<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	vegeta "github.com/tsenart/vegeta/lib"
)

const (
	encodingCSV  = "csv"
	encodingGob  = "gob"
	encodingJSON = "json"
)

type decoderFunc func(io.Reader) vegeta.Decoder
type encoderFunc func(io.Writer) vegeta.Encoder

func encodeCmd() command {
	var (
		fs     = flag.NewFlagSet("encode", flag.ExitOnError)
		from   = fs.String("from", "gob", "Input decoding [csv, gob, json]")
		to     = fs.String("to", "json", "Output encoding [csv, gob, json]")
		output = fs.String("output", "", "Output file")
	)

	fs.Usage = func() {
		fmt.Println("Usage: vegeta encode [flags] [<file>...]")
		fs.PrintDefaults()
	}

	return command{
		fs,
		func(args []string) error {
			fs.Parse(args)
			return encode(*from, *to, *output, args...)
		},
	}
}

func encode(from, to, output string, inputs ...string) error {
	var decFn decoderFunc

	switch from {
	case encodingCSV:
		decFn = vegeta.NewCSVDecoder
	case encodingJSON:
		decFn = vegeta.NewJSONDecoder
	default:
		// Gob is our default decode format to play nicely in pipes with attack.
		decFn = vegeta.NewDecoder
	}

	decs := []vegeta.Decoder{}

	if len(inputs) > 0 {
		for _, name := range inputs {
			f, err := os.Open(name)
			if err != nil {
				return err
			}
			defer f.Close()

			decs = append(decs, decFn(f))
		}
	} else {
		decs = append(decs, decFn(os.Stdin))
	}

	in := vegeta.NewRoundRobinDecoder(decs...)

	var encFn encoderFunc

	switch to {
	case encodingCSV:
		encFn = vegeta.NewCSVEncoder
	case encodingGob:
		encFn = vegeta.NewEncoder
	default:
		// JSON is our default encoding format to play nicely in pipes with auxiliary
		// tools like jq.
		encFn = vegeta.NewJSONEncoder
	}

	var out vegeta.Encoder

	if output != "" {
		f, err := os.Create(output)
		if err != nil {
			return err
		}
		defer f.Close()

		out = encFn(f)
	} else {
		out = encFn(os.Stdout)
	}

	for {
		var r vegeta.Result
		if err := in.Decode(&r); err != nil {
			if err == io.EOF {
				break
			}
			return err
		} else if err := out.Encode(&r); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>encode: Include "vegeta" in FlagSet name<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	vegeta "github.com/tsenart/vegeta/lib"
)

const (
	encodingCSV  = "csv"
	encodingGob  = "gob"
	encodingJSON = "json"
)

type decoderFunc func(io.Reader) vegeta.Decoder
type encoderFunc func(io.Writer) vegeta.Encoder

func encodeCmd() command {
	var (
		fs     = flag.NewFlagSet("vegeta encode", flag.ExitOnError)
		from   = fs.String("from", "gob", "Input decoding [csv, gob, json]")
		to     = fs.String("to", "json", "Output encoding [csv, gob, json]")
		output = fs.String("output", "", "Output file")
	)

	fs.Usage = func() {
		fmt.Println("Usage: vegeta encode [flags] [<file>...]")
		fs.PrintDefaults()
	}

	return command{
		fs,
		func(args []string) error {
			fs.Parse(args)
			return encode(*from, *to, *output, args...)
		},
	}
}

func encode(from, to, output string, inputs ...string) error {
	var decFn decoderFunc

	switch from {
	case encodingCSV:
		decFn = vegeta.NewCSVDecoder
	case encodingJSON:
		decFn = vegeta.NewJSONDecoder
	default:
		// Gob is our default decode format to play nicely in pipes with attack.
		decFn = vegeta.NewDecoder
	}

	decs := []vegeta.Decoder{}

	if len(inputs) > 0 {
		for _, name := range inputs {
			f, err := os.Open(name)
			if err != nil {
				return err
			}
			defer f.Close()

			decs = append(decs, decFn(f))
		}
	} else {
		decs = append(decs, decFn(os.Stdin))
	}

	in := vegeta.NewRoundRobinDecoder(decs...)

	var encFn encoderFunc

	switch to {
	case encodingCSV:
		encFn = vegeta.NewCSVEncoder
	case encodingGob:
		encFn = vegeta.NewEncoder
	default:
		// JSON is our default encoding format to play nicely in pipes with auxiliary
		// tools like jq.
		encFn = vegeta.NewJSONEncoder
	}

	var out vegeta.Encoder

	if output != "" {
		f, err := os.Create(output)
		if err != nil {
			return err
		}
		defer f.Close()

		out = encFn(f)
	} else {
		out = encFn(os.Stdout)
	}

	for {
		var r vegeta.Result
		if err := in.Decode(&r); err != nil {
			if err == io.EOF {
				break
			}
			return err
		} else if err := out.Encode(&r); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package service

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/newrelic/sidecar/output"
)

const (
	ALIVE     = iota
	TOMBSTONE = iota
	UNHEALTHY = iota
	UNKNOWN   = iota
)

type Port struct {
	Type        string
	Port        int64
	ServicePort int64
}

type Service struct {
	ID        string
	Name      string
	Image     string
	Created   time.Time
	Hostname  string
	Ports     []Port
	Updated   time.Time
	ProxyMode string
	Status    int
}

func (svc Service) Encode() ([]byte, error) {
	return json.Marshal(svc)
}

func (svc *Service) StatusString() string {
	return StatusString(svc.Status)
}

func (svc *Service) IsAlive() bool {
	return svc.Status == ALIVE
}

func (svc *Service) IsTombstone() bool {
	return svc.Status == TOMBSTONE
}

func (svc *Service) Invalidates(otherSvc *Service) bool {
	return otherSvc != nil && svc.Updated.After(otherSvc.Updated)
}

func (svc *Service) Format() string {
	var ports []string
	for _, port := range svc.Ports {
		ports = append(ports,
			fmt.Sprintf("%d->%d", port.ServicePort, port.Port),
		)
	}
	return fmt.Sprintf("      %s %-30s %-15s %-45s  %-15s %-9s\n",
		svc.ID,
		svc.Name,
		strings.Join(ports, ","),
		svc.Image,
		output.TimeAgo(svc.Updated, time.Now().UTC()),
		svc.StatusString(),
	)
}

func (svc *Service) Tombstone() {
	svc.Status = TOMBSTONE
	svc.Updated = time.Now().UTC()
}

// Look up a (usually Docker) mapped Port for a service by ServicePort
func (svc *Service) PortForServicePort(findPort int64, pType string) int64 {
	for _, port := range svc.Ports {
		if port.ServicePort == findPort && port.Type == pType {
			return port.Port
		}
	}

	log.Warnf("Unable to find ServicePort %d for service %s", findPort, svc.ID)
	return -1
}

func Decode(data []byte) *Service {
	var svc Service
	json.Unmarshal(data, &svc)

	return &svc
}

// Format an APIContainers struct into a more compact struct we
// can ship over the wire in a broadcast.
func ToService(container *docker.APIContainers) Service {
	var svc Service
	hostname, _ := os.Hostname()

	svc.ID = container.ID[0:12]   // Use short IDs
	svc.Name = container.Names[0] // Use the first name
	svc.Image = container.Image
	svc.Created = time.Unix(container.Created, 0).UTC()
	svc.Updated = time.Now().UTC()
	svc.Hostname = hostname
	svc.Status = ALIVE

	if _, ok := container.Labels["ProxyMode"]; ok {
		svc.ProxyMode = container.Labels["ProxyMode"]
	} else {
		svc.ProxyMode = "http"
	}

	svc.Ports = make([]Port, 0)

	for _, port := range container.Ports {
		if port.PublicPort != 0 {
			svc.Ports = append(svc.Ports, buildPortFor(&port, container))
		}
	}

	return svc
}

func StatusString(status int) {
	switch status {
	case ALIVE:
		return "Alive"
	case UNHEALTHY:
		return "Unhealthy"
	case UNKNOWN:
		return "Unknown"
	default:
		return "Tombstone"
	}
}

// Figure out the correct port configuration for a service
func buildPortFor(port *docker.APIPort, container *docker.APIContainers) Port {
	// We look up service port labels by convention in the format "ServicePort_80=8080"
	svcPortLabel := fmt.Sprintf("ServicePort_%d", port.PrivatePort)

	returnPort := Port{Port: port.PublicPort, Type: port.Type}

	if svcPort, ok := container.Labels[svcPortLabel]; ok {
		svcPortInt, err := strconv.Atoi(svcPort)
		if err != nil {
			log.Errorf("Error converting label value for %s to integer: %s",
				svcPortLabel,
				err.Error(),
			)
			return returnPort
		}

		// Everything was good, set the service port
		returnPort.ServicePort = int64(svcPortInt)
	}

	return returnPort
}
<commit_msg>Needs to return a string.<commit_after>package service

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/newrelic/sidecar/output"
)

const (
	ALIVE     = iota
	TOMBSTONE = iota
	UNHEALTHY = iota
	UNKNOWN   = iota
)

type Port struct {
	Type        string
	Port        int64
	ServicePort int64
}

type Service struct {
	ID        string
	Name      string
	Image     string
	Created   time.Time
	Hostname  string
	Ports     []Port
	Updated   time.Time
	ProxyMode string
	Status    int
}

func (svc Service) Encode() ([]byte, error) {
	return json.Marshal(svc)
}

func (svc *Service) StatusString() string {
	return StatusString(svc.Status)
}

func (svc *Service) IsAlive() bool {
	return svc.Status == ALIVE
}

func (svc *Service) IsTombstone() bool {
	return svc.Status == TOMBSTONE
}

func (svc *Service) Invalidates(otherSvc *Service) bool {
	return otherSvc != nil && svc.Updated.After(otherSvc.Updated)
}

func (svc *Service) Format() string {
	var ports []string
	for _, port := range svc.Ports {
		ports = append(ports,
			fmt.Sprintf("%d->%d", port.ServicePort, port.Port),
		)
	}
	return fmt.Sprintf("      %s %-30s %-15s %-45s  %-15s %-9s\n",
		svc.ID,
		svc.Name,
		strings.Join(ports, ","),
		svc.Image,
		output.TimeAgo(svc.Updated, time.Now().UTC()),
		svc.StatusString(),
	)
}

func (svc *Service) Tombstone() {
	svc.Status = TOMBSTONE
	svc.Updated = time.Now().UTC()
}

// Look up a (usually Docker) mapped Port for a service by ServicePort
func (svc *Service) PortForServicePort(findPort int64, pType string) int64 {
	for _, port := range svc.Ports {
		if port.ServicePort == findPort && port.Type == pType {
			return port.Port
		}
	}

	log.Warnf("Unable to find ServicePort %d for service %s", findPort, svc.ID)
	return -1
}

func Decode(data []byte) *Service {
	var svc Service
	json.Unmarshal(data, &svc)

	return &svc
}

// Format an APIContainers struct into a more compact struct we
// can ship over the wire in a broadcast.
func ToService(container *docker.APIContainers) Service {
	var svc Service
	hostname, _ := os.Hostname()

	svc.ID = container.ID[0:12]   // Use short IDs
	svc.Name = container.Names[0] // Use the first name
	svc.Image = container.Image
	svc.Created = time.Unix(container.Created, 0).UTC()
	svc.Updated = time.Now().UTC()
	svc.Hostname = hostname
	svc.Status = ALIVE

	if _, ok := container.Labels["ProxyMode"]; ok {
		svc.ProxyMode = container.Labels["ProxyMode"]
	} else {
		svc.ProxyMode = "http"
	}

	svc.Ports = make([]Port, 0)

	for _, port := range container.Ports {
		if port.PublicPort != 0 {
			svc.Ports = append(svc.Ports, buildPortFor(&port, container))
		}
	}

	return svc
}

func StatusString(status int) string {
	switch status {
	case ALIVE:
		return "Alive"
	case UNHEALTHY:
		return "Unhealthy"
	case UNKNOWN:
		return "Unknown"
	default:
		return "Tombstone"
	}
}

// Figure out the correct port configuration for a service
func buildPortFor(port *docker.APIPort, container *docker.APIContainers) Port {
	// We look up service port labels by convention in the format "ServicePort_80=8080"
	svcPortLabel := fmt.Sprintf("ServicePort_%d", port.PrivatePort)

	returnPort := Port{Port: port.PublicPort, Type: port.Type}

	if svcPort, ok := container.Labels[svcPortLabel]; ok {
		svcPortInt, err := strconv.Atoi(svcPort)
		if err != nil {
			log.Errorf("Error converting label value for %s to integer: %s",
				svcPortLabel,
				err.Error(),
			)
			return returnPort
		}

		// Everything was good, set the service port
		returnPort.ServicePort = int64(svcPortInt)
	}

	return returnPort
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	"strings"
)

// Error codes returned by failures
var (
	// ErrEmpty will returned if the string is empty.
	ErrEmpty = ErrInfo{
		Text:   errors.New("The string is empty."),
		Status: http.StatusBadRequest,
		Code:   "str_empty",
	}
)

// Service represents the operations of the service can do.
type Service interface {
	Uppercase(string) (string, error)
	Count(string) int
	CatchEvent(map[string]interface{}, map[string]string)
}

// Count counts the length of the string.
func (svc service) Count(s string) int {
	return len(s)
}

// Uppercase converts the string to uppercase.
func (svc service) Uppercase(s string) (string, error) {
	if s == "" {
		return "", Err{Message: ErrEmpty}
	}

	u := strings.ToUpper(s)

	svc.Store.CreateString(s, u)

	fmt.Println(svc.Store.GetLastString())

	return u, nil
}

func (svc service) CatchEvent(body map[string]interface{}, meta map[string]string) {
	fmt.Println(body["test"])
}
<commit_msg>Service with event publish, edited catchEvent func<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/jetbasrawi/go.geteventstore"
)

// Error codes returned by failures
var (
	// ErrEmpty is returned if the string is empty.
	ErrEmpty = ErrInfo{
		Text:   errors.New("The string is empty."),
		Status: http.StatusBadRequest,
		Code:   "string_empty",
	}
	// ErrEvent is returned if the error occurred while we're publishing the event.
	ErrEvent = ErrInfo{
		Text:   errors.New("Error occurred while publishing the event."),
		Status: http.StatusInternalServerError,
		Code:   "event_error",
	}
)

// Service represents the operations of the service can do.
type Service interface {
	Uppercase(string) (string, error)
	Count(string) int
	CatchUppercase(map[string]interface{}, map[string]interface{})
}

// Count counts the length of the string.
func (svc service) Count(s string) int {
	return len(s)
}

// Uppercase converts the string to uppercase.
func (svc service) Uppercase(s string) (string, error) {
	if s == "" {
		return "", Err{Message: ErrEmpty}
	}

	// Converts the string to uppercase.
	u := strings.ToUpper(s)

	// Create the string record in the database.
	svc.Store.CreateString(s, u)

	// Publish the uppercase event to the event store.
	if err := svc.ES.NewStreamWriter("uppercase").Append(nil, goes.NewEvent(
		"",
		"",
		H{"input": s, "output": u}.MapString(),
		H{}.MapString(),
	)); err != nil {
		return "", Err{Message: ErrEvent}
	}

	// Get the last record from the database.
	fmt.Println("Last record:")
	fmt.Println(svc.Store.GetLastString())

	return u, nil
}

// CatchUppercase catches the uppercase event, and print the event data.
func (svc service) CatchUppercase(body map[string]interface{}, meta map[string]interface{}) {
	fmt.Println("Event received:")
	fmt.Println("Input:" + body["input"].(string) + " | Output:" + body["output"].(string))
}
<|endoftext|>


<commit_before>package endly

import (
	"fmt"
	"github.com/viant/toolbox/cred"
	"github.com/viant/toolbox/ssh"
)

//NetworkServiceID represents java network service id
const NetworkServiceID = "network"

type networkService struct {
	*AbstractService
}

func (s *networkService) forward(context *Context, request *NetworkForwardRequest) (*NetworkForwardResponse, error) {
	var response = &NetworkForwardResponse{
		Forwards: make([]*NetworkForward, 0),
	}
	var target, err = context.ExpandResource(request.Target)
	var authConfig = &cred.Config{}
	hostname, port := getHostAndSSHPort(target)
	client, err := ssh.NewClient(hostname, port, authConfig)
	if err != nil {
		return nil, err
	}
	context.Deffer(func() {
		client.Close()
	})
	for _, forward := range request.Forwards {
		var local = context.Expand(forward.Local)
		var remote = context.Expand(forward.Remote)

		fmt.Printf(" %v -> %v\n", local, remote)
		err = client.Forward(local, remote)
		if err != nil {
			return nil, err
		}
		response.Forwards = append(response.Forwards, &NetworkForward{Local: local, Remote: remote})
	}
	return response, nil
}

func (s *networkService) Run(context *Context, request interface{}) *ServiceResponse {
	startEvent := s.Begin(context, request, Pairs("request", request))
	var response = &ServiceResponse{Status: "ok"}
	defer s.End(context)(startEvent, Pairs("response", response))
	var err error
	switch actualRequest := request.(type) {
	case *NetworkForwardRequest:
		response.Response, err = s.forward(context, actualRequest)
		if err != nil {
			response.Error = fmt.Sprintf("Failed to run forward: %v, %v", actualRequest.Target.URL, err)
		}
	default:
		response.Error = fmt.Sprintf("Unsupported request type: %T", request)
	}
	if response.Error != "" {
		response.Status = "err"
	}
	return response
}

//NewRequest creates a new request for an action (run).
func (s *networkService) NewRequest(action string) (interface{}, error) {
	switch action {
	case "forward":
		return &NetworkForwardRequest{}, nil
	}
	return s.AbstractService.NewRequest(action)
}

//NewNetworkService creates a new network service.
func NewNetworkService() Service {
	var result = &networkService{
		AbstractService: NewAbstractService(NetworkServiceID),
	}
	result.AbstractService.Service = result
	return result
}
<commit_msg>remvoed forward info<commit_after>package endly

import (
	"fmt"
	"github.com/viant/toolbox/cred"
	"github.com/viant/toolbox/ssh"
)

//NetworkServiceID represents java network service id
const NetworkServiceID = "network"

type networkService struct {
	*AbstractService
}

func (s *networkService) forward(context *Context, request *NetworkForwardRequest) (*NetworkForwardResponse, error) {
	var response = &NetworkForwardResponse{
		Forwards: make([]*NetworkForward, 0),
	}
	var target, err = context.ExpandResource(request.Target)
	var authConfig = &cred.Config{}
	hostname, port := getHostAndSSHPort(target)
	client, err := ssh.NewClient(hostname, port, authConfig)
	if err != nil {
		return nil, err
	}
	context.Deffer(func() {
		client.Close()
	})
	for _, forward := range request.Forwards {
		var local = context.Expand(forward.Local)
		var remote = context.Expand(forward.Remote)
		err = client.Forward(local, remote)
		if err != nil {
			return nil, err
		}
		response.Forwards = append(response.Forwards, &NetworkForward{Local: local, Remote: remote})
	}
	return response, nil
}

func (s *networkService) Run(context *Context, request interface{}) *ServiceResponse {
	startEvent := s.Begin(context, request, Pairs("request", request))
	var response = &ServiceResponse{Status: "ok"}
	defer s.End(context)(startEvent, Pairs("response", response))
	var err error
	switch actualRequest := request.(type) {
	case *NetworkForwardRequest:
		response.Response, err = s.forward(context, actualRequest)
		if err != nil {
			response.Error = fmt.Sprintf("Failed to run forward: %v, %v", actualRequest.Target.URL, err)
		}
	default:
		response.Error = fmt.Sprintf("Unsupported request type: %T", request)
	}
	if response.Error != "" {
		response.Status = "err"
	}
	return response
}

//NewRequest creates a new request for an action (run).
func (s *networkService) NewRequest(action string) (interface{}, error) {
	switch action {
	case "forward":
		return &NetworkForwardRequest{}, nil
	}
	return s.AbstractService.NewRequest(action)
}

//NewNetworkService creates a new network service.
func NewNetworkService() Service {
	var result = &networkService{
		AbstractService: NewAbstractService(NetworkServiceID),
	}
	result.AbstractService.Service = result
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/aerogo/http/client"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/mal"
	"github.com/fatih/color"
)

const imageWidthThreshold = 225

var (
	malDB           = arn.Node.Namespace("mal").RegisterTypes((*mal.Anime)(nil))
	characterFinder = arn.NewCharacterFinder("myanimelist/character")
)

func main() {
	color.Yellow("Syncing with MAL DB")

	defer color.Green("Finished.")
	defer arn.Node.Close()

	// Invoke via parameters
	if InvokeShellArgs() {
		return
	}

	// Sync the most important ones first
	allAnime := arn.AllAnime()
	arn.SortAnimeByQuality(allAnime)

	for _, anime := range allAnime {
		malID := anime.GetMapping("myanimelist/anime")

		if malID == "" {
			continue
		}

		sync(anime, malID)
	}
}

func sync(anime *arn.Anime, malID string) {
	obj, err := malDB.Get("Anime", malID)

	if err != nil {
		fmt.Println(err)
		return
	}

	malAnime := obj.(*mal.Anime)

	// Log title
	fmt.Printf("%s %s\n", color.CyanString(anime.Title.Canonical), malID)

	if len(anime.Genres) == 0 && len(malAnime.Genres) > 0 {
		fmt.Println("Genres:", malAnime.Genres)
		anime.Genres = malAnime.Genres
	}

	if anime.EpisodeCount == 0 && malAnime.EpisodeCount != 0 {
		fmt.Println("EpisodeCount:", malAnime.EpisodeCount)
		anime.EpisodeCount = malAnime.EpisodeCount
	}

	if anime.EpisodeLength == 0 && malAnime.EpisodeLength != 0 {
		fmt.Println("EpisodeLength:", malAnime.EpisodeLength)
		anime.EpisodeLength = malAnime.EpisodeLength
	}

	if anime.StartDate == "" && malAnime.StartDate != "" {
		fmt.Println("StartDate:", malAnime.StartDate)
		anime.StartDate = malAnime.StartDate
	}

	if anime.EndDate == "" && malAnime.EndDate != "" {
		fmt.Println("EndDate:", malAnime.EndDate)
		anime.EndDate = malAnime.EndDate
	}

	if anime.Source == "" && malAnime.Source != "" {
		fmt.Println("Source:", malAnime.Source)
		anime.Source = malAnime.Source
	}

	if anime.Title.Japanese == "" && malAnime.JapaneseTitle != "" {
		fmt.Println("JapaneseTitle:", malAnime.JapaneseTitle)
		anime.Title.Japanese = malAnime.JapaneseTitle
	}

	if anime.Title.English == "" && malAnime.EnglishTitle != "" {
		fmt.Println("EnglishTitle:", malAnime.EnglishTitle)
		anime.Title.English = malAnime.EnglishTitle
	}

	if (!anime.HasImage() || anime.Image.Width <= imageWidthThreshold) && malAnime.Image != "" {
		fmt.Println("Downloading image:", malAnime.Image)
		response, err := client.Get(malAnime.Image).End()

		if err == nil && response.StatusCode() == 200 {
			anime.SetImageBytes(response.Bytes())
		} else {
			color.Red("Error downloading image")
		}
	}

	// Check for existence of characters
	animeCharacters := anime.Characters()
	modifiedCharacters := false

	for _, malAnimeCharacter := range malAnime.Characters {
		// Make sure we have no invalid entries
		if malAnimeCharacter.ID == "" || malAnimeCharacter.Role == "" {
			fmt.Println("Skip:", malAnimeCharacter)
			continue
		}

		animeCharacter := animeCharacters.FindByMapping("myanimelist/character", malAnimeCharacter.ID)

		if animeCharacter != nil {
			continue
		}

		obj, err := malDB.Get("Character", malAnimeCharacter.ID)

		// If we don't have the MAL character in the DB,
		// we can't import anything here.
		if err != nil {
			continue
		}

		malCharacter := obj.(*mal.Character)

		if malCharacter.ID == "" || malCharacter.Name == "" || malCharacter.ImagePath == "" {
			fmt.Println("Skip character:", malAnimeCharacter.ID)
			continue
		}

		fmt.Println("Importing MAL AnimeCharacter:", malAnimeCharacter.ID, "as", malAnimeCharacter.Role)

		// Import character if needed
		character := characterFinder.GetCharacter(malAnimeCharacter.ID)

		if character == nil {
			character = importCharacter(malCharacter)
		} else {
			fmt.Println("Found existing character:", character)

			// Download image if missing
			if !character.HasImage() {
				fmt.Println("Downloading missing image for character:", character)
				character.DownloadImage(malCharacter.ImageLink())

				// Cancel import if that character has no image
				if err != nil {
					color.Red(err.Error())
					continue
				}
			}
		}

		// If import failed, continue
		if character == nil {
			continue
		}

		// Add to anime characters
		err = animeCharacters.Add(&arn.AnimeCharacter{
			CharacterID: character.ID,
			Role:        malAnimeCharacter.Role,
		})

		if err != nil {
			color.Red(err.Error())
		}

		modifiedCharacters = true
	}

	if modifiedCharacters {
		animeCharacters.Save()
	}

	anime.Save()
}

func importCharacter(malCharacter *mal.Character) *arn.Character {
	fmt.Println("Importing MAL Character:", malCharacter.ID, malCharacter.Name, malCharacter.ImagePath)

	character := arn.NewCharacter()
	character.Name.Canonical = malCharacter.Name

	// Cancel the import if image could not be fetched
	err := character.DownloadImage(malCharacter.ImageLink())

	if err != nil {
		color.Red(err.Error())
		return nil
	}

	// Add mapping
	character.SetMapping("myanimelist/character", malCharacter.ID)

	// Save character in DB
	character.Save()

	// Add to character finder so we don't create duplicates of this character
	characterFinder.Add(character)

	return character
}
<commit_msg>Simplified mal sync<commit_after>package main

import (
	"fmt"

	"github.com/aerogo/http/client"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/mal"
	"github.com/fatih/color"
)

const imageWidthThreshold = 225

var (
	malDB           = arn.Node.Namespace("mal").RegisterTypes((*mal.Anime)(nil))
	characterFinder = arn.NewCharacterFinder("myanimelist/character")
)

func main() {
	color.Yellow("Syncing with MAL DB")

	defer color.Green("Finished.")
	defer arn.Node.Close()

	// Invoke via parameters
	if InvokeShellArgs() {
		return
	}

	// Sync the most important ones first
	allAnime := arn.AllAnime()
	arn.SortAnimeByQuality(allAnime)

	for _, anime := range allAnime {
		malID := anime.GetMapping("myanimelist/anime")

		if malID == "" {
			continue
		}

		sync(anime, malID)
	}
}

func sync(anime *arn.Anime, malID string) {
	obj, err := malDB.Get("Anime", malID)

	if err != nil {
		fmt.Println(err)
		return
	}

	malAnime := obj.(*mal.Anime)

	// Log title
	fmt.Printf("%s %s\n", color.CyanString(anime.Title.Canonical), malID)

	if len(anime.Genres) == 0 && len(malAnime.Genres) > 0 {
		fmt.Println("Genres:", malAnime.Genres)
		anime.Genres = malAnime.Genres
	}

	if anime.EpisodeCount == 0 && malAnime.EpisodeCount != 0 {
		fmt.Println("EpisodeCount:", malAnime.EpisodeCount)
		anime.EpisodeCount = malAnime.EpisodeCount
	}

	if anime.EpisodeLength == 0 && malAnime.EpisodeLength != 0 {
		fmt.Println("EpisodeLength:", malAnime.EpisodeLength)
		anime.EpisodeLength = malAnime.EpisodeLength
	}

	if anime.StartDate == "" && malAnime.StartDate != "" {
		fmt.Println("StartDate:", malAnime.StartDate)
		anime.StartDate = malAnime.StartDate
	}

	if anime.EndDate == "" && malAnime.EndDate != "" {
		fmt.Println("EndDate:", malAnime.EndDate)
		anime.EndDate = malAnime.EndDate
	}

	if anime.Source == "" && malAnime.Source != "" {
		fmt.Println("Source:", malAnime.Source)
		anime.Source = malAnime.Source
	}

	if anime.Title.Japanese == "" && malAnime.JapaneseTitle != "" {
		fmt.Println("JapaneseTitle:", malAnime.JapaneseTitle)
		anime.Title.Japanese = malAnime.JapaneseTitle
	}

	if anime.Title.English == "" && malAnime.EnglishTitle != "" {
		fmt.Println("EnglishTitle:", malAnime.EnglishTitle)
		anime.Title.English = malAnime.EnglishTitle
	}

	if (!anime.HasImage() || anime.Image.Width <= imageWidthThreshold) && malAnime.Image != "" {
		fmt.Println("Downloading image:", malAnime.Image)
		response, err := client.Get(malAnime.Image).End()

		if err == nil && response.StatusCode() == 200 {
			anime.SetImageBytes(response.Bytes())
		} else {
			color.Red("Error downloading image")
		}
	}

	// Characters
	syncCharacters(anime, malAnime)

	// Save in database
	anime.Save()
}

func syncCharacters(anime *arn.Anime, malAnime *mal.Anime) {
	// Check for existence of characters
	animeCharacters := anime.Characters()
	modifiedCharacters := false

	for _, malAnimeCharacter := range malAnime.Characters {
		// Make sure we have no invalid entries
		if malAnimeCharacter.ID == "" || malAnimeCharacter.Role == "" {
			fmt.Println("Skip:", malAnimeCharacter)
			continue
		}

		animeCharacter := animeCharacters.FindByMapping("myanimelist/character", malAnimeCharacter.ID)

		if animeCharacter != nil {
			continue
		}

		obj, err := malDB.Get("Character", malAnimeCharacter.ID)

		// If we don't have the MAL character in the DB,
		// we can't import anything here.
		if err != nil {
			continue
		}

		malCharacter := obj.(*mal.Character)

		if malCharacter.ID == "" || malCharacter.Name == "" || malCharacter.ImagePath == "" {
			fmt.Println("Skip character:", malAnimeCharacter.ID)
			continue
		}

		fmt.Println("Importing MAL AnimeCharacter:", malAnimeCharacter.ID, "as", malAnimeCharacter.Role)

		// Import character if needed
		character := characterFinder.GetCharacter(malAnimeCharacter.ID)

		if character == nil {
			character = importCharacter(malCharacter)
		} else {
			fmt.Println("Found existing character:", character)

			// Download image if missing
			if !character.HasImage() {
				fmt.Println("Downloading missing image for character:", character)
				character.DownloadImage(malCharacter.ImageLink())

				// Cancel import if that character has no image
				if err != nil {
					color.Red(err.Error())
					continue
				}
			}
		}

		// If import failed, continue
		if character == nil {
			continue
		}

		// Add to anime characters
		err = animeCharacters.Add(&arn.AnimeCharacter{
			CharacterID: character.ID,
			Role:        malAnimeCharacter.Role,
		})

		if err != nil {
			color.Red(err.Error())
		}

		modifiedCharacters = true
	}

	if modifiedCharacters {
		animeCharacters.Save()
	}
}

func importCharacter(malCharacter *mal.Character) *arn.Character {
	fmt.Println("Importing MAL Character:", malCharacter.ID, malCharacter.Name, malCharacter.ImagePath)

	character := arn.NewCharacter()
	character.Name.Canonical = malCharacter.Name

	// Cancel the import if image could not be fetched
	err := character.DownloadImage(malCharacter.ImageLink())

	if err != nil {
		color.Red(err.Error())
		return nil
	}

	// Add mapping
	character.SetMapping("myanimelist/character", malCharacter.ID)

	// Save character in DB
	character.Save()

	// Add to character finder so we don't create duplicates of this character
	characterFinder.Add(character)

	return character
}
<|endoftext|>


<commit_before>package metrics

import (
	"github.com/DataDog/datadog-go/statsd"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
)

type DatadogStatsdMetricsClient struct {
	client *statsd.Client
}

func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
	if !conf.IsSet("metrics.dogstatsd.address") {
		return errors.Errorf("Unable to initialize DatadogMetricsClient: metrics.dogstatsd.address must be set in the config.")
	}

	addr := conf.GetString("metrics.dogstatsd.address")
	client, err := statsd.New(addr)
	if err != nil {
		return err
	}

	// Set global namespace if set in config.
	if conf.IsSet("metrics.dogstatsd.namespace") {
		client.Namespace = conf.GetString("metrics.dogstatsd.namespace")
	}

	// Set global tags if set in config.
	if conf.IsSet("metrics.dogstatsd.tags") {
		tags := conf.GetStringSlice("metrics.dogstatsd.tags")
		for _, tag := range tags {
			client.Tags = append(client.Tags, tag)
		}
	}

	dd.client = client

	return nil
}

func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []string, rate float64) error {
	return dd.client.Decr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []string, rate float64) error {
	return dd.client.Incr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Histogram(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Distribution(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, tags []string, rate float64) error {
	return dd.client.Set(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Event(e event) error {
	se := statsd.NewEvent(e.Title, e.Text)
	se.Tags = e.Tags
	return dd.client.Event(se)
}
<commit_msg>init (#270)<commit_after>package metrics

import (
	"fmt"
	"github.com/DataDog/datadog-go/statsd"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
)

type DatadogStatsdMetricsClient struct {
	client *statsd.Client
}

func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
	if !conf.IsSet("metrics.dogstatsd.address") {
		return errors.Errorf("Unable to initialize DatadogMetricsClient: metrics.dogstatsd.address must be set in the config.")
	}

	addr := conf.GetString("metrics.dogstatsd.address")
	client, err := statsd.New(addr)
	if err != nil {
		return err
	}

	// Set global namespace if set in config.
	if conf.IsSet("metrics.dogstatsd.namespace") {
		client.Namespace = conf.GetString("metrics.dogstatsd.namespace")
	}

	// Set global tags if set in config.
	if conf.IsSet("metrics.dogstatsd.tags") {
		tags := conf.GetStringMapString("metrics.dogstatsd.tags")
		for k, v := range tags {
			client.Tags = append(client.Tags, fmt.Sprintf("%s=%s", k, v))
		}
	}

	dd.client = client

	return nil
}

func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []string, rate float64) error {
	return dd.client.Decr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []string, rate float64) error {
	return dd.client.Incr(string(name), tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Histogram(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value float64, tags []string, rate float64) error {
	return dd.client.Distribution(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, tags []string, rate float64) error {
	return dd.client.Set(string(name), value, tags, rate)
}

func (dd *DatadogStatsdMetricsClient) Event(e event) error {
	se := statsd.NewEvent(e.Title, e.Text)
	se.Tags = e.Tags
	return dd.client.Event(se)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package errors

import (
	"fmt"
)

// AutoscalerErrorType describes a high-level category of a given error
type AutoscalerErrorType string

// AutoscalerError contains information about Autoscaler errors
type AutoscalerError interface {
	// Error implements golang error interface
	Error() string

	// Type returns the type of AutoscalerError
	Type() AutoscalerErrorType

	// AddPrefix adds a prefix to error message.
	// Returns the error it's called for convenient inline use.
	// Example:
	// if err := DoSomething(myObject); err != nil {
	//	return err.AddPrefix("can't do something with %v: ", myObject)
	// }
	AddPrefix(msg string, args ...interface{}) AutoscalerError
}

type autoscalerErrorImpl struct {
	errorType AutoscalerErrorType
	msg       string
}

const (
	// CloudProviderError is an error related to underlying infrastructure
	CloudProviderError AutoscalerErrorType = "cloudProviderError"
	// ApiCallError is an error related to communication with k8s API server
	ApiCallError AutoscalerErrorType = "apiCallError"
	// InternalError is an error inside Cluster Autoscaler
	InternalError AutoscalerErrorType = "internalError"
	// TransientError is an error that causes us to skip a single loop, but
	// does not require any additional action.
	TransientError AutoscalerErrorType = "transientError"
)

// NewAutoscalerError returns new autoscaler error with a message constructed from format string
func NewAutoscalerError(errorType AutoscalerErrorType, msg string, args ...interface{}) AutoscalerError {
	return autoscalerErrorImpl{
		errorType: errorType,
		msg:       fmt.Sprintf(msg, args...),
	}
}

// ToAutoscalerError converts an error to AutoscalerError with given type,
// unless it already is an AutoscalerError (in which case it's not modified).
func ToAutoscalerError(defaultType AutoscalerErrorType, err error) AutoscalerError {
	if e, ok := err.(AutoscalerError); ok {
		return e
	}
	return NewAutoscalerError(defaultType, err.Error())
}

// Error implements golang error interface
func (e autoscalerErrorImpl) Error() string {
	return e.msg
}

// Type returns the type of AutoscalerError
func (e autoscalerErrorImpl) Type() AutoscalerErrorType {
	return e.errorType
}

// AddPrefix adds a prefix to error message.
// Returns the error it's called for convenient inline use.
// Example:
// if err := DoSomething(myObject); err != nil {
//	return err.AddPrefix("can't do something with %v: ", myObject)
// }
func (e autoscalerErrorImpl) AddPrefix(msg string, args ...interface{}) AutoscalerError {
	e.msg = fmt.Sprintf(msg, args...) + e.msg
	return e
}
<commit_msg>Fix potential panic<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package errors

import (
	"fmt"
)

// AutoscalerErrorType describes a high-level category of a given error
type AutoscalerErrorType string

// AutoscalerError contains information about Autoscaler errors
type AutoscalerError interface {
	// Error implements golang error interface
	Error() string

	// Type returns the type of AutoscalerError
	Type() AutoscalerErrorType

	// AddPrefix adds a prefix to error message.
	// Returns the error it's called for convenient inline use.
	// Example:
	// if err := DoSomething(myObject); err != nil {
	//	return err.AddPrefix("can't do something with %v: ", myObject)
	// }
	AddPrefix(msg string, args ...interface{}) AutoscalerError
}

type autoscalerErrorImpl struct {
	errorType AutoscalerErrorType
	msg       string
}

const (
	// CloudProviderError is an error related to underlying infrastructure
	CloudProviderError AutoscalerErrorType = "cloudProviderError"
	// ApiCallError is an error related to communication with k8s API server
	ApiCallError AutoscalerErrorType = "apiCallError"
	// InternalError is an error inside Cluster Autoscaler
	InternalError AutoscalerErrorType = "internalError"
	// TransientError is an error that causes us to skip a single loop, but
	// does not require any additional action.
	TransientError AutoscalerErrorType = "transientError"
)

// NewAutoscalerError returns new autoscaler error with a message constructed from format string
func NewAutoscalerError(errorType AutoscalerErrorType, msg string, args ...interface{}) AutoscalerError {
	return autoscalerErrorImpl{
		errorType: errorType,
		msg:       fmt.Sprintf(msg, args...),
	}
}

// ToAutoscalerError converts an error to AutoscalerError with given type,
// unless it already is an AutoscalerError (in which case it's not modified).
func ToAutoscalerError(defaultType AutoscalerErrorType, err error) AutoscalerError {
	if e, ok := err.(AutoscalerError); ok {
		return e
	}
	return NewAutoscalerError(defaultType, "%v", err)
}

// Error implements golang error interface
func (e autoscalerErrorImpl) Error() string {
	return e.msg
}

// Type returns the type of AutoscalerError
func (e autoscalerErrorImpl) Type() AutoscalerErrorType {
	return e.errorType
}

// AddPrefix adds a prefix to error message.
// Returns the error it's called for convenient inline use.
// Example:
// if err := DoSomething(myObject); err != nil {
//	return err.AddPrefix("can't do something with %v: ", myObject)
// }
func (e autoscalerErrorImpl) AddPrefix(msg string, args ...interface{}) AutoscalerError {
	e.msg = fmt.Sprintf(msg, args...) + e.msg
	return e
}
<|endoftext|>


<commit_before>package collectors

import (
	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/wmi"
)

func init() {
	collectors = append(collectors, c_physical_disk_windows)
	collectors = append(collectors, c_diskspace_windows)
}

const PHYSICAL_DISK_QUERY = `
	SELECT Name, AvgDisksecPerRead, AvgDisksecPerWrite,
		AvgDiskReadQueueLength, AvgDiskWriteQueueLength,
		DiskReadBytesPersec, DiskReadsPersec,
		DiskWriteBytesPersec, DiskWritesPersec,
		SplitIOPerSec, PercentDiskReadTime, PercentDiskWriteTime
	FROM Win32_PerfRawData_PerfDisk_PhysicalDisk
	WHERE Name <> '_Total'
`

//Similar Breakdowns exist as to physical, but for now just using this for the space utilization
const DISKSPACE_QUERY = `
	SELECT Name, FreeMegaBytes, PercentFreeSpace
	FROM Win32_PerfRawData_PerfDisk_LogicalDisk
	WHERE Name <> '_Total'
`

func c_diskspace_windows() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfDisk_LogicalDisk
	err := wmi.Query(DISKSPACE_QUERY, &dst)
	if err != nil {
		l.Println("diskpace:", err)
		return nil
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "disk.logical.free_bytes", v.FreeMegabytes*1048576, opentsdb.TagSet{"partition": v.Name})
		Add(&md, "disk.logical.percent_free", v.PercentFreeSpace, opentsdb.TagSet{"partition": v.Name})
	}
	return md
}

func c_physical_disk_windows() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	err := wmi.Query(PHYSICAL_DISK_QUERY, &dst)
	if err != nil {
		l.Println("disk_physical:", err)
		return nil
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "disk.physical.duration", v.AvgDiskSecPerRead, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.duration", v.AvgDiskSecPerWrite, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.queue", v.AvgDiskReadQueueLength, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.queue", v.AvgDiskWriteQueueLength, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.percenttime", v.PercentDiskReadTime, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.percenttime", v.PercentDiskWriteTime, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name})
	}
	return md
}
<commit_msg>cmd/scollector: Cleanup comment<commit_after>package collectors

import (
	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/wmi"
)

func init() {
	collectors = append(collectors, c_physical_disk_windows)
	collectors = append(collectors, c_diskspace_windows)
}

const PHYSICAL_DISK_QUERY = `
	SELECT Name, AvgDisksecPerRead, AvgDisksecPerWrite,
		AvgDiskReadQueueLength, AvgDiskWriteQueueLength,
		DiskReadBytesPersec, DiskReadsPersec,
		DiskWriteBytesPersec, DiskWritesPersec,
		SplitIOPerSec, PercentDiskReadTime, PercentDiskWriteTime
	FROM Win32_PerfRawData_PerfDisk_PhysicalDisk
	WHERE Name <> '_Total'
`

// Similar breakdowns exist as to physical, but for now just using this for the
// space utilization.
const DISKSPACE_QUERY = `
	SELECT Name, FreeMegaBytes, PercentFreeSpace
	FROM Win32_PerfRawData_PerfDisk_LogicalDisk
	WHERE Name <> '_Total'
`

func c_diskspace_windows() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfDisk_LogicalDisk
	err := wmi.Query(DISKSPACE_QUERY, &dst)
	if err != nil {
		l.Println("diskpace:", err)
		return nil
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "disk.logical.free_bytes", v.FreeMegabytes*1048576, opentsdb.TagSet{"partition": v.Name})
		Add(&md, "disk.logical.percent_free", v.PercentFreeSpace, opentsdb.TagSet{"partition": v.Name})
	}
	return md
}

func c_physical_disk_windows() opentsdb.MultiDataPoint {
	var dst []Win32_PerfRawData_PerfDisk_PhysicalDisk
	err := wmi.Query(PHYSICAL_DISK_QUERY, &dst)
	if err != nil {
		l.Println("disk_physical:", err)
		return nil
	}
	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		Add(&md, "disk.physical.duration", v.AvgDiskSecPerRead, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.duration", v.AvgDiskSecPerWrite, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.queue", v.AvgDiskReadQueueLength, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.queue", v.AvgDiskWriteQueueLength, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.ops", v.DiskReadsPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.ops", v.DiskWritesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.bytes", v.DiskReadBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.bytes", v.DiskWriteBytesPerSec, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.percenttime", v.PercentDiskReadTime, opentsdb.TagSet{"disk": v.Name, "type": "read"})
		Add(&md, "disk.physical.percenttime", v.PercentDiskWriteTime, opentsdb.TagSet{"disk": v.Name, "type": "write"})
		Add(&md, "disk.physical.spltio", v.SplitIOPerSec, opentsdb.TagSet{"disk": v.Name})
	}
	return md
}
<|endoftext|>


<commit_before>package collectors

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/util"
)

func init() {
	registerInit(func(c *conf.Conf) {
		if c.IfaceExpr != "" {
			ifstatRE = regexp.MustCompile(fmt.Sprintf("(%s):(.*)", c.IfaceExpr))
		}

		collectors = append(collectors, &IntervalCollector{F: c_ifstat_linux})
		collectors = append(collectors, &IntervalCollector{F: c_ipcount_linux})
		collectors = append(collectors, &IntervalCollector{F: c_if_team_linux})
		collectors = append(collectors, &IntervalCollector{F: c_if_bond_linux})
	})
}

var netFields = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
}{
	{"bytes", metadata.Counter, metadata.Bytes},
	{"packets", metadata.Counter, metadata.Count},
	{"errs", metadata.Counter, metadata.Count},
	{"dropped", metadata.Counter, metadata.Count},
	{"fifo.errs", metadata.Counter, metadata.Count},
	{"frame.errs", metadata.Counter, metadata.Count},
	{"compressed", metadata.Counter, metadata.Count},
	{"multicast", metadata.Counter, metadata.Count},
	{"bytes", metadata.Counter, metadata.Bytes},
	{"packets", metadata.Counter, metadata.Count},
	{"errs", metadata.Counter, metadata.Count},
	{"dropped", metadata.Counter, metadata.Count},
	{"fifo.errs", metadata.Counter, metadata.Count},
	{"collisions", metadata.Counter, metadata.Count},
	{"carrier.errs", metadata.Counter, metadata.Count},
	{"compressed", metadata.Counter, metadata.Count},
}

var ifstatRE = regexp.MustCompile(`\s+(eth\d+|em\d+_\d+/\d+|em\d+_\d+|em\d+|` +
	`bond\d+|team\d+|` + `p\d+p\d+_\d+/\d+|p\d+p\d+_\d+|p\d+p\d+):(.*)`)

func c_ipcount_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	v4c := 0
	v6c := 0
	err := util.ReadCommand(func(line string) error {
		tl := strings.TrimSpace(line)
		if strings.HasPrefix(tl, "inet ") {
			v4c++
		}
		if strings.HasPrefix(tl, "inet6 ") {
			v6c++
		}
		return nil
	}, "ip", "addr", "list")
	if err != nil {
		return md, err
	}
	Add(&md, "linux.net.ip_count", v4c, opentsdb.TagSet{"version": "4"}, metadata.Gauge, "IP_Addresses", "")
	Add(&md, "linux.net.ip_count", v6c, opentsdb.TagSet{"version": "6"}, metadata.Gauge, "IP_Addresses", "")
	return md, nil
}

func c_ifstat_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	direction := func(i int) string {
		if i >= 8 {
			return "out"
		} else {
			return "in"
		}
	}
	err := readLine("/proc/net/dev", func(s string) error {
		m := ifstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		intf := m[1]
		stats := strings.Fields(m[2])
		tags := opentsdb.TagSet{"iface": intf}
		var bond_string string
		if strings.HasPrefix(intf, "bond") || strings.HasPrefix(intf, "team") {
			bond_string = "bond."
		}
		// Detect speed of the interface in question
		_ = readLine("/sys/class/net/"+intf+"/speed", func(speed string) error {
			Add(&md, "linux.net."+bond_string+"ifspeed", speed, tags, metadata.Gauge, metadata.Megabit, "")
			Add(&md, "os.net."+bond_string+"ifspeed", speed, tags, metadata.Gauge, metadata.Megabit, "")
			return nil
		})
		for i, v := range stats {
			Add(&md, "linux.net."+bond_string+strings.Replace(netFields[i].key, ".", "_", -1), v, opentsdb.TagSet{
				"iface":     intf,
				"direction": direction(i),
			}, netFields[i].rate, netFields[i].unit, "")
			if i < 4 || (i >= 8 && i < 12) {
				Add(&md, "os.net."+bond_string+strings.Replace(netFields[i].key, ".", "_", -1), v, opentsdb.TagSet{
					"iface":     intf,
					"direction": direction(i),
				}, netFields[i].rate, netFields[i].unit, "")

			}
		}
		return nil
	})
	return md, err
}

const (
	linuxNetBondSlaveIsUpDesc = "The status of the bonded or teamed interface."
	linuxNetBondSlaveCount    = "The number of slaves on the bonded or teamed interface."
)

func c_if_bond_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	const bondingPath = "/proc/net/bonding"
	bondDevices, err := ioutil.ReadDir(bondingPath)
	if err != nil {
		return md, nil
	}
	for _, fi := range bondDevices {
		var iface string
		var slaveCount int
		if err := readLine(filepath.Join(bondingPath, fi.Name()), func(s string) error {
			f := strings.SplitN(s, ":", 2)
			if len(f) != 2 {
				return nil
			}
			f[0] = strings.TrimSpace(f[0])
			f[1] = strings.TrimSpace(f[1])
			if f[0] == "Slave Interface" {
				iface = f[1]
				slaveCount++
			}
			// TODO: This will probably need to be updated for other types of bonding beside LACP, but I have no examples available to work with at the moment
			if f[0] == "MII Status" && iface != "" {
				var status int
				if f[1] == "up" {
					status = 1
				}
				Add(&md, "linux.net.bond.slave.is_up", status, opentsdb.TagSet{"slave": iface, "bond": fi.Name()}, metadata.Gauge, metadata.Bool, linuxNetBondSlaveIsUpDesc)
			}
			return nil
		}); err != nil {
			return md, err
		}
		Add(&md, "linux.net.bond.slave.count", slaveCount, opentsdb.TagSet{"bond": fi.Name()}, metadata.Gauge, metadata.Count, linuxNetBondSlaveCount)
	}
	return md, nil
}

func c_if_team_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	getState := func(iname string) (TeamState, error) {
		var ts TeamState
		reader, err := util.Command(time.Second*5, nil, "teamdctl", iname, "state", "dump")
		if err != nil {
			return ts, err
		}
		err = json.NewDecoder(reader).Decode(&ts)
		if err != nil {
			return ts, err
		}
		return ts, nil
	}
	teamdFiles, err := ioutil.ReadDir("/var/run/teamd")
	if err != nil {
		return md, nil
	}
	for _, f := range teamdFiles {
		name := f.Name()
		if strings.HasSuffix(name, ".pid") {
			name = strings.TrimSuffix(name, ".pid")
			ts, err := getState(name)
			if err != nil {
				return md, err
			}
			var slaveCount int
			var speed int64
			for portName, port := range ts.TeamPorts {
				slaveCount++
				speed += int64(port.Link.Speed)
				metadata.AddMeta("", opentsdb.TagSet{"iface": portName}, "master", name, true)
				Add(&md, "linux.net.bond.slave.is_up", port.Link.Up, opentsdb.TagSet{"slave": portName, "bond": name}, metadata.Gauge, metadata.Bool, linuxNetBondSlaveIsUpDesc)
			}
			Add(&md, "os.net.bond.ifspeed", speed, opentsdb.TagSet{"bond": name}, metadata.Gauge, metadata.Megabit, osNetIfSpeedDesc)
			Add(&md, "linux.net.bond.slave.count", slaveCount, opentsdb.TagSet{"bond": name}, metadata.Gauge, metadata.Count, linuxNetBondSlaveCount)
		}
	}
	return md, nil
}

type TeamState struct {
	TeamPorts map[string]TeamPort `json:"ports"`
	Runner    struct {
		Active       bool    `json:"active"`
		FastRate     bool    `json:"fast_rate"`
		SelectPolicy string  `json:"select_policy"`
		SysPrio      float64 `json:"sys_prio"`
	} `json:"runner"`
	Setup struct {
		Daemonized         bool    `json:"daemonized"`
		DbusEnabled        bool    `json:"dbus_enabled"`
		DebugLevel         float64 `json:"debug_level"`
		KernelTeamModeName string  `json:"kernel_team_mode_name"`
		Pid                float64 `json:"pid"`
		PidFile            string  `json:"pid_file"`
		RunnerName         string  `json:"runner_name"`
		ZmqEnabled         bool    `json:"zmq_enabled"`
	} `json:"setup"`
	TeamDevice struct {
		Ifinfo struct {
			DevAddr    string  `json:"dev_addr"`
			DevAddrLen float64 `json:"dev_addr_len"`
			Ifindex    float64 `json:"ifindex"`
			Ifname     string  `json:"ifname"`
		} `json:"ifinfo"`
	} `json:"team_device"`
}

type TeamPort struct {
	Ifinfo struct {
		DevAddr    string  `json:"dev_addr"`
		DevAddrLen float64 `json:"dev_addr_len"`
		Ifindex    float64 `json:"ifindex"`
		Ifname     string  `json:"ifname"`
	}
	Link struct {
		Duplex string  `json:"duplex"`
		Speed  float64 `json:"speed"`
		Up     bool    `json:"up"`
	} `json:"link"`
	LinkWatches struct {
		List struct {
			LinkWatch0 struct {
				DelayDown float64 `json:"delay_down"`
				DelayUp   float64 `json:"delay_up"`
				Name      string  `json:"name"`
				Up        bool    `json:"up"`
			} `json:"link_watch_0"`
		} `json:"list"`
		Up bool `json:"up"`
	} `json:"link_watches"`
	Runner struct {
		ActorLacpduInfo struct {
			Key            float64 `json:"key"`
			Port           float64 `json:"port"`
			PortPriority   float64 `json:"port_priority"`
			State          float64 `json:"state"`
			System         string  `json:"system"`
			SystemPriority float64 `json:"system_priority"`
		} `json:"actor_lacpdu_info"`
		Aggregator struct {
			ID       float64 `json:"id"`
			Selected bool    `json:"selected"`
		} `json:"aggregator"`
		Key               float64 `json:"key"`
		PartnerLacpduInfo struct {
			Key            float64 `json:"key"`
			Port           float64 `json:"port"`
			PortPriority   float64 `json:"port_priority"`
			State          float64 `json:"state"`
			System         string  `json:"system"`
			SystemPriority float64 `json:"system_priority"`
		} `json:"partner_lacpdu_info"`
		Prio     float64 `json:"prio"`
		Selected bool    `json:"selected"`
		State    string  `json:"state"`
	} `json:"runner"`
}
<commit_msg>cmd/scollector: add systemd physical "predictable" linux net interface names (#1985)<commit_after>package collectors

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/util"
)

func init() {
	registerInit(func(c *conf.Conf) {
		if c.IfaceExpr != "" {
			ifstatRE = regexp.MustCompile(fmt.Sprintf("(%s):(.*)", c.IfaceExpr))
		}

		collectors = append(collectors, &IntervalCollector{F: c_ifstat_linux})
		collectors = append(collectors, &IntervalCollector{F: c_ipcount_linux})
		collectors = append(collectors, &IntervalCollector{F: c_if_team_linux})
		collectors = append(collectors, &IntervalCollector{F: c_if_bond_linux})
	})
}

var netFields = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
}{
	{"bytes", metadata.Counter, metadata.Bytes},
	{"packets", metadata.Counter, metadata.Count},
	{"errs", metadata.Counter, metadata.Count},
	{"dropped", metadata.Counter, metadata.Count},
	{"fifo.errs", metadata.Counter, metadata.Count},
	{"frame.errs", metadata.Counter, metadata.Count},
	{"compressed", metadata.Counter, metadata.Count},
	{"multicast", metadata.Counter, metadata.Count},
	{"bytes", metadata.Counter, metadata.Bytes},
	{"packets", metadata.Counter, metadata.Count},
	{"errs", metadata.Counter, metadata.Count},
	{"dropped", metadata.Counter, metadata.Count},
	{"fifo.errs", metadata.Counter, metadata.Count},
	{"collisions", metadata.Counter, metadata.Count},
	{"carrier.errs", metadata.Counter, metadata.Count},
	{"compressed", metadata.Counter, metadata.Count},
}

var ifstatRE = regexp.MustCompile(`\s*(eth\d+|em\d+_\d+/\d+|em\d+_\d+|em\d+|` +
	`bond\d+|team\d+|` +
	`p\d+p\d+_\d+/\d+|p\d+p\d+_\d+|p\d+p\d+|` +
	`en[[:alnum:]]+|wl[[:alnum:]]+|ww[[:alnum:]]+` + // Systemd predictable interface names
	`):(.*)`)

func c_ipcount_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	v4c := 0
	v6c := 0
	err := util.ReadCommand(func(line string) error {
		tl := strings.TrimSpace(line)
		if strings.HasPrefix(tl, "inet ") {
			v4c++
		}
		if strings.HasPrefix(tl, "inet6 ") {
			v6c++
		}
		return nil
	}, "ip", "addr", "list")
	if err != nil {
		return md, err
	}
	Add(&md, "linux.net.ip_count", v4c, opentsdb.TagSet{"version": "4"}, metadata.Gauge, "IP_Addresses", "")
	Add(&md, "linux.net.ip_count", v6c, opentsdb.TagSet{"version": "6"}, metadata.Gauge, "IP_Addresses", "")
	return md, nil
}

func c_ifstat_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	direction := func(i int) string {
		if i >= 8 {
			return "out"
		} else {
			return "in"
		}
	}
	err := readLine("/proc/net/dev", func(s string) error {
		m := ifstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		intf := m[1]
		stats := strings.Fields(m[2])
		tags := opentsdb.TagSet{"iface": intf}
		var bond_string string
		if strings.HasPrefix(intf, "bond") || strings.HasPrefix(intf, "team") {
			bond_string = "bond."
		}
		// Detect speed of the interface in question
		_ = readLine("/sys/class/net/"+intf+"/speed", func(speed string) error {
			Add(&md, "linux.net."+bond_string+"ifspeed", speed, tags, metadata.Gauge, metadata.Megabit, "")
			Add(&md, "os.net."+bond_string+"ifspeed", speed, tags, metadata.Gauge, metadata.Megabit, "")
			return nil
		})
		for i, v := range stats {
			Add(&md, "linux.net."+bond_string+strings.Replace(netFields[i].key, ".", "_", -1), v, opentsdb.TagSet{
				"iface":     intf,
				"direction": direction(i),
			}, netFields[i].rate, netFields[i].unit, "")
			if i < 4 || (i >= 8 && i < 12) {
				Add(&md, "os.net."+bond_string+strings.Replace(netFields[i].key, ".", "_", -1), v, opentsdb.TagSet{
					"iface":     intf,
					"direction": direction(i),
				}, netFields[i].rate, netFields[i].unit, "")

			}
		}
		return nil
	})
	return md, err
}

const (
	linuxNetBondSlaveIsUpDesc = "The status of the bonded or teamed interface."
	linuxNetBondSlaveCount    = "The number of slaves on the bonded or teamed interface."
)

func c_if_bond_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	const bondingPath = "/proc/net/bonding"
	bondDevices, err := ioutil.ReadDir(bondingPath)
	if err != nil {
		return md, nil
	}
	for _, fi := range bondDevices {
		var iface string
		var slaveCount int
		if err := readLine(filepath.Join(bondingPath, fi.Name()), func(s string) error {
			f := strings.SplitN(s, ":", 2)
			if len(f) != 2 {
				return nil
			}
			f[0] = strings.TrimSpace(f[0])
			f[1] = strings.TrimSpace(f[1])
			if f[0] == "Slave Interface" {
				iface = f[1]
				slaveCount++
			}
			// TODO: This will probably need to be updated for other types of bonding beside LACP, but I have no examples available to work with at the moment
			if f[0] == "MII Status" && iface != "" {
				var status int
				if f[1] == "up" {
					status = 1
				}
				Add(&md, "linux.net.bond.slave.is_up", status, opentsdb.TagSet{"slave": iface, "bond": fi.Name()}, metadata.Gauge, metadata.Bool, linuxNetBondSlaveIsUpDesc)
			}
			return nil
		}); err != nil {
			return md, err
		}
		Add(&md, "linux.net.bond.slave.count", slaveCount, opentsdb.TagSet{"bond": fi.Name()}, metadata.Gauge, metadata.Count, linuxNetBondSlaveCount)
	}
	return md, nil
}

func c_if_team_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	getState := func(iname string) (TeamState, error) {
		var ts TeamState
		reader, err := util.Command(time.Second*5, nil, "teamdctl", iname, "state", "dump")
		if err != nil {
			return ts, err
		}
		err = json.NewDecoder(reader).Decode(&ts)
		if err != nil {
			return ts, err
		}
		return ts, nil
	}
	teamdFiles, err := ioutil.ReadDir("/var/run/teamd")
	if err != nil {
		return md, nil
	}
	for _, f := range teamdFiles {
		name := f.Name()
		if strings.HasSuffix(name, ".pid") {
			name = strings.TrimSuffix(name, ".pid")
			ts, err := getState(name)
			if err != nil {
				return md, err
			}
			var slaveCount int
			var speed int64
			for portName, port := range ts.TeamPorts {
				slaveCount++
				speed += int64(port.Link.Speed)
				metadata.AddMeta("", opentsdb.TagSet{"iface": portName}, "master", name, true)
				Add(&md, "linux.net.bond.slave.is_up", port.Link.Up, opentsdb.TagSet{"slave": portName, "bond": name}, metadata.Gauge, metadata.Bool, linuxNetBondSlaveIsUpDesc)
			}
			Add(&md, "os.net.bond.ifspeed", speed, opentsdb.TagSet{"bond": name}, metadata.Gauge, metadata.Megabit, osNetIfSpeedDesc)
			Add(&md, "linux.net.bond.slave.count", slaveCount, opentsdb.TagSet{"bond": name}, metadata.Gauge, metadata.Count, linuxNetBondSlaveCount)
		}
	}
	return md, nil
}

type TeamState struct {
	TeamPorts map[string]TeamPort `json:"ports"`
	Runner    struct {
		Active       bool    `json:"active"`
		FastRate     bool    `json:"fast_rate"`
		SelectPolicy string  `json:"select_policy"`
		SysPrio      float64 `json:"sys_prio"`
	} `json:"runner"`
	Setup struct {
		Daemonized         bool    `json:"daemonized"`
		DbusEnabled        bool    `json:"dbus_enabled"`
		DebugLevel         float64 `json:"debug_level"`
		KernelTeamModeName string  `json:"kernel_team_mode_name"`
		Pid                float64 `json:"pid"`
		PidFile            string  `json:"pid_file"`
		RunnerName         string  `json:"runner_name"`
		ZmqEnabled         bool    `json:"zmq_enabled"`
	} `json:"setup"`
	TeamDevice struct {
		Ifinfo struct {
			DevAddr    string  `json:"dev_addr"`
			DevAddrLen float64 `json:"dev_addr_len"`
			Ifindex    float64 `json:"ifindex"`
			Ifname     string  `json:"ifname"`
		} `json:"ifinfo"`
	} `json:"team_device"`
}

type TeamPort struct {
	Ifinfo struct {
		DevAddr    string  `json:"dev_addr"`
		DevAddrLen float64 `json:"dev_addr_len"`
		Ifindex    float64 `json:"ifindex"`
		Ifname     string  `json:"ifname"`
	}
	Link struct {
		Duplex string  `json:"duplex"`
		Speed  float64 `json:"speed"`
		Up     bool    `json:"up"`
	} `json:"link"`
	LinkWatches struct {
		List struct {
			LinkWatch0 struct {
				DelayDown float64 `json:"delay_down"`
				DelayUp   float64 `json:"delay_up"`
				Name      string  `json:"name"`
				Up        bool    `json:"up"`
			} `json:"link_watch_0"`
		} `json:"list"`
		Up bool `json:"up"`
	} `json:"link_watches"`
	Runner struct {
		ActorLacpduInfo struct {
			Key            float64 `json:"key"`
			Port           float64 `json:"port"`
			PortPriority   float64 `json:"port_priority"`
			State          float64 `json:"state"`
			System         string  `json:"system"`
			SystemPriority float64 `json:"system_priority"`
		} `json:"actor_lacpdu_info"`
		Aggregator struct {
			ID       float64 `json:"id"`
			Selected bool    `json:"selected"`
		} `json:"aggregator"`
		Key               float64 `json:"key"`
		PartnerLacpduInfo struct {
			Key            float64 `json:"key"`
			Port           float64 `json:"port"`
			PortPriority   float64 `json:"port_priority"`
			State          float64 `json:"state"`
			System         string  `json:"system"`
			SystemPriority float64 `json:"system_priority"`
		} `json:"partner_lacpdu_info"`
		Prio     float64 `json:"prio"`
		Selected bool    `json:"selected"`
		State    string  `json:"state"`
	} `json:"runner"`
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/9seconds/ah/app/environments"
	"github.com/9seconds/ah/app/historyentries"
	"github.com/9seconds/ah/app/utils"
)

const teeDelta = 1

// Tee implements t (trace, tee) command.
func Tee(input string, interactive bool, pseudoTTY bool, env *environments.Environment) {
	output, err := ioutil.TempFile(env.GetTmpDir(), "ah")
	if err != nil {
		utils.Logger.Panic("Cannot create temporary file")
	}

	bufferedOutput := bufio.NewWriter(output)
	gzippedWrapper := utils.NewSynchronizedWriter(gzip.NewWriter(bufferedOutput))
	combinedStdout := io.MultiWriter(os.Stdout, gzippedWrapper)
	combinedStderr := io.MultiWriter(os.Stderr, gzippedWrapper)

	var commandError *exec.ExitError
	defer func() {
		// defer here because command may cause a panic but we do not want to lose any output
		gzippedWrapper.Close()
		bufferedOutput.Flush()
		output.Close()

		if hash, err := getPreciseHash(input, env); err == nil {
			err = os.Rename(output.Name(), env.GetTraceFileName(hash))
			if err != nil {
				utils.Logger.Errorf("Cannot save trace: %v. Get it here: %s", err, output.Name())
			} else {
				os.Remove(output.Name())
			}
		} else {
			utils.Logger.Errorf("Error occured on fetching command number: %v", err)
		}

		if commandError != nil {
			os.Exit(utils.GetStatusCode(commandError))
		}
	}()

	commandError = utils.Exec(input,
		string(env.GetShell()), interactive, pseudoTTY,
		os.Stdin, combinedStdout, combinedStderr)
}

func getPreciseHash(cmd string, env *environments.Environment) (hash string, err error) {
	commands, err := historyentries.GetCommands(historyentries.GetCommandsAll, nil, env)
	if err != nil {
		err = fmt.Errorf("Cannot fetch commands list: %v", err)
		return
	}
	commandList := commands.Result().([]historyentries.HistoryEntry)

	if len(commandList) == 0 {
		err = errors.New("Command list is empty")
		return
	}

	// Why do I need such complicated logic here? The reason is trivial:
	// one may use substitutions in a command like
	// ah t -- `which python` script $(docker images -a -q)
	// basically it manages these situations as precise as possible.

	// Candidates here just means that several command may be executed
	// simultaneously (e.g. with tmuxinator) so timestamp is not precise
	// identifier here.

	candidates, err := getCandidates(commandList)
	if err != nil {
		err = fmt.Errorf("Cannot detect proper command: %v", err)
		return
	}

	if len(candidates) == 1 {
		hash = candidates[0].GetTraceName()
		return
	}

	suitable := getSuitable(candidates, cmd)
	hash = (*suitable).GetTraceName()

	return
}

func getCandidates(commands []historyentries.HistoryEntry) (candidates []historyentries.HistoryEntry, err error) {
	start := -1
	finish := -1

	for idx := len(commands) - 1; idx >= 0; idx-- {
		timestamp := commands[idx].GetTimestamp()
		switch {
		case timestamp < environments.CreatedAt-teeDelta:
			break
		case timestamp > environments.CreatedAt+teeDelta:
			continue
		default:
			if finish == -1 {
				finish = idx + 1
			}
			start = idx
		}
	}

	if start == -1 || finish == -1 {
		err = errors.New("Cannot find anything in the time range")
	} else {
		candidates = commands[start:finish]
	}

	return
}

func getSuitable(candidates []historyentries.HistoryEntry, cmd string) *historyentries.HistoryEntry {
	chunks := strings.Split(cmd, " ")
	maxHaveElements := -1
	maxIdx := 0

	for idx := 0; idx < len(candidates); idx++ {
		haveElements := countElements(candidates[idx].GetCommand(), chunks)
		if haveElements == len(chunks) {
			maxIdx = idx
			break
		}
		if haveElements > maxHaveElements {
			maxHaveElements = haveElements
			maxIdx = idx
		}
	}

	return &candidates[maxIdx]
}

func countElements(cmd string, chunks []string) (count int) {
	for idx := 0; idx < len(chunks); idx++ {
		if strings.Contains(cmd, chunks[idx]) {
			count++
		}
	}

	return
}
<commit_msg>Fix for #10<commit_after>package commands

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/9seconds/ah/app/environments"
	"github.com/9seconds/ah/app/historyentries"
	"github.com/9seconds/ah/app/utils"
)

const teeDelta = 1

// Tee implements t (trace, tee) command.
func Tee(input string, interactive bool, pseudoTTY bool, env *environments.Environment) {
	output, err := ioutil.TempFile(env.GetTmpDir(), "ah")
	if err != nil {
		utils.Logger.Panic("Cannot create temporary file")
	}

	bufferedOutput := bufio.NewWriter(output)
	gzippedWrapper := utils.NewSynchronizedWriter(gzip.NewWriter(bufferedOutput))
	combinedStdout := io.MultiWriter(os.Stdout, gzippedWrapper)
	combinedStderr := io.MultiWriter(os.Stderr, gzippedWrapper)

	var commandError *exec.ExitError
	defer func() {
		// defer here because command may cause a panic but we do not want to lose any output
		gzippedWrapper.Close()
		bufferedOutput.Flush()
		output.Close()

		if hash, err := getPreciseHash(input, env); err == nil {
			err = os.Rename(output.Name(), env.GetTraceFileName(hash))
			if err != nil {
				utils.Logger.Errorf("Cannot save trace: %v. Get it here: %s", err, output.Name())
			} else {
				os.Remove(output.Name())
			}
		} else {
			utils.Logger.Errorf("Error occured on fetching command number: %v", err)
		}

		if commandError != nil {
			os.Exit(utils.GetStatusCode(commandError))
		}
	}()

	commandError = utils.Exec(input,
		string(env.GetShell()), interactive, pseudoTTY,
		os.Stdin, combinedStdout, combinedStderr)
}

func getPreciseHash(cmd string, env *environments.Environment) (hash string, err error) {
	commands, err := historyentries.GetCommands(historyentries.GetCommandsAll, nil, env)
	if err != nil {
		err = fmt.Errorf("Cannot fetch commands list: %v", err)
		return
	}
	commandList := commands.Result().([]historyentries.HistoryEntry)

	if len(commandList) == 0 {
		err = errors.New("Command list is empty")
		return
	}

	// Why do I need such complicated logic here? The reason is trivial:
	// one may use substitutions in a command like
	// ah t -- `which python` script $(docker images -a -q)
	// basically it manages these situations as precise as possible.

	// Candidates here just means that several command may be executed
	// simultaneously (e.g. with tmuxinator) so timestamp is not precise
	// identifier here.

	candidates, err := getCandidates(commandList)
	if err != nil {
		err = fmt.Errorf("Cannot detect proper command: %v", err)
		return
	}

	if len(candidates) == 1 {
		hash = candidates[0].GetTraceName()
		return
	}

	suitable := getSuitable(candidates, cmd)
	hash = (*suitable).GetTraceName()

	return
}

func getCandidates(commands []historyentries.HistoryEntry) (candidates []historyentries.HistoryEntry, err error) {
	start := -1
	finish := -1

	for idx := len(commands) - 1; idx >= 0; idx-- {
		timestamp := commands[idx].GetTimestamp()
		switch {
		case timestamp < environments.CreatedAt-teeDelta:
			break
		case timestamp > environments.CreatedAt+teeDelta:
			continue
		default:
			if finish == -1 {
				finish = idx + 1
			}
			start = idx
		}
	}

	if start == -1 || finish == -1 {
		utils.Logger.Warn("Cannot find anything on the time range " +
			"so assume that deduplication works. Take the latest.")
		candidates = commands[len(commands)-1:]
	} else {
		candidates = commands[start:finish]
	}

	return
}

func getSuitable(candidates []historyentries.HistoryEntry, cmd string) *historyentries.HistoryEntry {
	chunks := strings.Split(cmd, " ")
	maxHaveElements := -1
	maxIdx := 0

	for idx := 0; idx < len(candidates); idx++ {
		haveElements := countElements(candidates[idx].GetCommand(), chunks)
		if haveElements == len(chunks) {
			maxIdx = idx
			break
		}
		if haveElements > maxHaveElements {
			maxHaveElements = haveElements
			maxIdx = idx
		}
	}

	return &candidates[maxIdx]
}

func countElements(cmd string, chunks []string) (count int) {
	for idx := 0; idx < len(chunks); idx++ {
		if strings.Contains(cmd, chunks[idx]) {
			count++
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"testing"

	"github.com/coreos/rkt/tests/testutils"
)

func queryValue(query string, tx *sql.Tx) (int, error) {
	var value int
	rows, err := tx.Query(query)
	if err != nil {
		return -1, err
	}
	defer rows.Close()

	if !rows.Next() {
		return -1, fmt.Errorf("no result of %q", query)
	}
	if err := rows.Scan(&value); err != nil {
		return -1, err
	}
	return value, nil
}

func insertValue(db *DB) error {
	return db.Do(func(tx *sql.Tx) error {
		// Get the current count.
		count, err := queryValue("SELECT count(*) FROM rkt_db_test", tx)
		if err != nil {
			return err
		}
		// Increase the count.
		_, err = tx.Exec(fmt.Sprintf("INSERT INTO rkt_db_test VALUES (%d)", count+1))
		return err
	})
}

func getMaxCount(db *DB, t *testing.T) int {
	var maxCount int
	var err error
	if err := db.Do(func(tx *sql.Tx) error {
		// Get the maximum count.
		maxCount, err = queryValue("SELECT max(counts) FROM rkt_db_test", tx)
		return err
	}); err != nil {
		t.Fatalf("Failed to get the maximum count: %v", err)
	}
	return maxCount
}

func createTable(db *DB, t *testing.T) {
	if err := db.Do(func(tx *sql.Tx) error {
		_, err := tx.Exec("CREATE TABLE rkt_db_test (counts int)")
		return err
	}); err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
}

func TestDBRace(t *testing.T) {
	// TODO(sgotti) this will not find concurrenct accesses to ql db from
	// multiple processes using multiple goroutines. A test that spawns at
	// least two processes using multiple goroutines is needed.
	oldGoMaxProcs := runtime.GOMAXPROCS(runtime.NumCPU())
	defer runtime.GOMAXPROCS(oldGoMaxProcs)

	dir, err := ioutil.TempDir("", "rkt_db_test")
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	defer os.RemoveAll(dir)

	db, err := NewDB(dir)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Create the table.
	createTable(db, t)

	// Insert values concurrently.
	ga := testutils.NewGoroutineAssistant(t)
	runs := 100
	ga.Add(runs)
	for i := 0; i < runs; i++ {
		go func() {
			if err := insertValue(db); err != nil {
				ga.Fatalf("Failed to insert value: %v", err)
			}
			ga.Done()
		}()
	}
	ga.Wait()

	// Check the final values.
	maxCount := getMaxCount(db, t)
	if maxCount != runs {
		t.Errorf("Expected: %v, saw: %v", runs, maxCount)
	}
}
<commit_msg>store: fix comment + add link in db test<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"testing"

	"github.com/coreos/rkt/tests/testutils"
)

func queryValue(query string, tx *sql.Tx) (int, error) {
	var value int
	rows, err := tx.Query(query)
	if err != nil {
		return -1, err
	}
	defer rows.Close()

	if !rows.Next() {
		return -1, fmt.Errorf("no result of %q", query)
	}
	if err := rows.Scan(&value); err != nil {
		return -1, err
	}
	return value, nil
}

func insertValue(db *DB) error {
	return db.Do(func(tx *sql.Tx) error {
		// Get the current count.
		count, err := queryValue("SELECT count(*) FROM rkt_db_test", tx)
		if err != nil {
			return err
		}
		// Increase the count.
		_, err = tx.Exec(fmt.Sprintf("INSERT INTO rkt_db_test VALUES (%d)", count+1))
		return err
	})
}

func getMaxCount(db *DB, t *testing.T) int {
	var maxCount int
	var err error
	if err := db.Do(func(tx *sql.Tx) error {
		// Get the maximum count.
		maxCount, err = queryValue("SELECT max(counts) FROM rkt_db_test", tx)
		return err
	}); err != nil {
		t.Fatalf("Failed to get the maximum count: %v", err)
	}
	return maxCount
}

func createTable(db *DB, t *testing.T) {
	if err := db.Do(func(tx *sql.Tx) error {
		_, err := tx.Exec("CREATE TABLE rkt_db_test (counts int)")
		return err
	}); err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
}

func TestDBRace(t *testing.T) {
	// TODO(sgotti) this will not find concurrent accesses to ql db from
	// multiple processes using multiple goroutines. A test that spawns at
	// least two processes using multiple goroutines is needed.
	// See https://github.com/coreos/rkt/pull/2391
	oldGoMaxProcs := runtime.GOMAXPROCS(runtime.NumCPU())
	defer runtime.GOMAXPROCS(oldGoMaxProcs)

	dir, err := ioutil.TempDir("", "rkt_db_test")
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	defer os.RemoveAll(dir)

	db, err := NewDB(dir)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	// Create the table.
	createTable(db, t)

	// Insert values concurrently.
	ga := testutils.NewGoroutineAssistant(t)
	runs := 100
	ga.Add(runs)
	for i := 0; i < runs; i++ {
		go func() {
			if err := insertValue(db); err != nil {
				ga.Fatalf("Failed to insert value: %v", err)
			}
			ga.Done()
		}()
	}
	ga.Wait()

	// Check the final values.
	maxCount := getMaxCount(db, t)
	if maxCount != runs {
		t.Errorf("Expected: %v, saw: %v", runs, maxCount)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package riegeli

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"github.com/DataDog/zstd"
	"github.com/google/brotli/go/cbrotli"
)

// A decompressor decodes a compressed Riegeli block.
type decompressor interface {
	byteReader
	io.Closer
}

func newDecompressor(r byteReader, c compressionType) (decompressor, error) {
	if c == noCompression {
		return &nopDecompressorClose{r}, nil
	}

	if _, err := binary.ReadUvarint(r); err != nil {
		return nil, fmt.Errorf("bad varint prefix for compressed block: %v", err)
	}
	switch c {
	case brotliCompression:
		return &byteReadCloser{cbrotli.NewReader(r)}, nil
	case zstdCompression:
		return &byteReadCloser{zstd.NewReader(r)}, nil
	default:
		return nil, fmt.Errorf("unsupported compression_type: '%s'", []byte{byte(c)})
	}
}

// A byteReadCloser trivially implements io.ByteReader for a io.ReadCloser.
type byteReadCloser struct{ io.ReadCloser }

// ReadByte implements the io.ByteReader interface.
func (b byteReadCloser) ReadByte() (byte, error) {
	var buf [1]byte
	_, err := b.Read(buf[:])
	return buf[0], err
}

// A compressor builds a Riegeli compressed block.
type compressor interface {
	writerTo
	io.Closer
}

func newCompressor(opts *WriterOptions) (compressor, error) {
	buf := bytes.NewBuffer(nil)
	switch opts.compressionType() {
	case noCompression:
		return &nopCompressorClose{buf}, nil
	case brotliCompression:
		brotliOpts := cbrotli.WriterOptions{Quality: opts.compressionLevel()}
		w := cbrotli.NewWriter(buf, brotliOpts)
		return &sizePrefixedWriterTo{buf: buf, WriteCloser: w}, nil
	case zstdCompression:
		w := zstd.NewWriterLevel(buf, opts.compressionLevel())
		return &sizePrefixedWriterTo{buf: buf, WriteCloser: w}, nil
	default:
		return nil, fmt.Errorf("unsupported compression_type: '%s'", []byte{byte(opts.compressionType())})
	}
}

type sizePrefixedWriterTo struct {
	buf *bytes.Buffer
	io.WriteCloser
	prefix []byte
}

// Close implements part of the compressor interface.
func (w *sizePrefixedWriterTo) Close() error {
	if err := w.WriteCloser.Close(); err != nil {
		return err
	}

	w.prefix = make([]byte, binary.MaxVarintLen64)
	n := int64(binary.PutUvarint(w.prefix[:], uint64(w.buf.Len())))
	w.prefix = w.prefix[:n]

	return nil
}

// WriteTo implements part of the compressor interface.
func (w *sizePrefixedWriterTo) WriteTo(out io.Writer) (int64, error) {
	if n, err := out.Write(w.prefix); err != nil {
		return int64(n), err
	}
	n, err := w.buf.WriteTo(out)
	n += int64(len(w.prefix))
	return n, err
}

// Len implements part of the compressor interface.
func (w *sizePrefixedWriterTo) Len() int { return len(w.prefix) + w.buf.Len() }

type byteReader interface {
	io.Reader
	io.ByteReader
}

type writerTo interface {
	io.WriterTo
	io.Writer

	// Len returns the total data that will be written by WriteTo.
	Len() int
}

// A nopDecompressorClose trivially implements io.Closer for a byteReader.
type nopDecompressorClose struct{ byteReader }

// Close implements the io.Closer interface.
func (nopDecompressorClose) Close() error { return nil }

// A nopCompressorClose trivially implements io.Closer for a writerTo.
type nopCompressorClose struct{ writerTo }

// Close implements the io.Closer interface.
func (nopCompressorClose) Close() error { return nil }
<commit_msg>Riegeli: properly attempt to read byte until an error (#2805)<commit_after>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package riegeli

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"github.com/DataDog/zstd"
	"github.com/google/brotli/go/cbrotli"
)

// A decompressor decodes a compressed Riegeli block.
type decompressor interface {
	byteReader
	io.Closer
}

func newDecompressor(r byteReader, c compressionType) (decompressor, error) {
	if c == noCompression {
		return &nopDecompressorClose{r}, nil
	}

	if _, err := binary.ReadUvarint(r); err != nil {
		return nil, fmt.Errorf("bad varint prefix for compressed block: %v", err)
	}
	switch c {
	case brotliCompression:
		return &byteReadCloser{cbrotli.NewReader(r)}, nil
	case zstdCompression:
		return &byteReadCloser{zstd.NewReader(r)}, nil
	default:
		return nil, fmt.Errorf("unsupported compression_type: '%s'", []byte{byte(c)})
	}
}

// A byteReadCloser trivially implements io.ByteReader for a io.ReadCloser.
type byteReadCloser struct{ io.ReadCloser }

// ReadByte implements the io.ByteReader interface.
func (b byteReadCloser) ReadByte() (byte, error) {
	var buf [1]byte
	_, err := io.ReadFull(b.ReadCloser, buf[:])
	return buf[0], err
}

// A compressor builds a Riegeli compressed block.
type compressor interface {
	writerTo
	io.Closer
}

func newCompressor(opts *WriterOptions) (compressor, error) {
	buf := bytes.NewBuffer(nil)
	switch opts.compressionType() {
	case noCompression:
		return &nopCompressorClose{buf}, nil
	case brotliCompression:
		brotliOpts := cbrotli.WriterOptions{Quality: opts.compressionLevel()}
		w := cbrotli.NewWriter(buf, brotliOpts)
		return &sizePrefixedWriterTo{buf: buf, WriteCloser: w}, nil
	case zstdCompression:
		w := zstd.NewWriterLevel(buf, opts.compressionLevel())
		return &sizePrefixedWriterTo{buf: buf, WriteCloser: w}, nil
	default:
		return nil, fmt.Errorf("unsupported compression_type: '%s'", []byte{byte(opts.compressionType())})
	}
}

type sizePrefixedWriterTo struct {
	buf *bytes.Buffer
	io.WriteCloser
	prefix []byte
}

// Close implements part of the compressor interface.
func (w *sizePrefixedWriterTo) Close() error {
	if err := w.WriteCloser.Close(); err != nil {
		return err
	}

	w.prefix = make([]byte, binary.MaxVarintLen64)
	n := int64(binary.PutUvarint(w.prefix[:], uint64(w.buf.Len())))
	w.prefix = w.prefix[:n]

	return nil
}

// WriteTo implements part of the compressor interface.
func (w *sizePrefixedWriterTo) WriteTo(out io.Writer) (int64, error) {
	if n, err := out.Write(w.prefix); err != nil {
		return int64(n), err
	}
	n, err := w.buf.WriteTo(out)
	n += int64(len(w.prefix))
	return n, err
}

// Len implements part of the compressor interface.
func (w *sizePrefixedWriterTo) Len() int { return len(w.prefix) + w.buf.Len() }

type byteReader interface {
	io.Reader
	io.ByteReader
}

type writerTo interface {
	io.WriterTo
	io.Writer

	// Len returns the total data that will be written by WriteTo.
	Len() int
}

// A nopDecompressorClose trivially implements io.Closer for a byteReader.
type nopDecompressorClose struct{ byteReader }

// Close implements the io.Closer interface.
func (nopDecompressorClose) Close() error { return nil }

// A nopCompressorClose trivially implements io.Closer for a writerTo.
type nopCompressorClose struct{ writerTo }

// Close implements the io.Closer interface.
func (nopCompressorClose) Close() error { return nil }
<|endoftext|>


<commit_before>package assettube

import (
	"bytes"
	"html/template"
	"net/http"
	"net/http/httptest"
	"testing"

	_ "github.com/bom-d-van/sidekick"
)

func TestManager(t *testing.T) {
	for _, c := range []struct {
		cname       int
		path        string
		fingerprint bool
		pathMap     [][2]string
		getURL      string
	}{
		{
			cname:       1,
			path:        "/Users/bom_d_van/Code/go/workspace/src/github.com/theplant/assettube/test",
			fingerprint: true,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css"},
				{"js/file.js", "/js/file.bf5a6a7119046d97ee509d017080c6aa.js"},
			},
			getURL: "http://example.com/js/file.bf5a6a7119046d97ee509d017080c6aa.js",
		},
		{
			cname:       2,
			path:        "test",
			fingerprint: true,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css"},
				{"js/file.js", "/js/file.bf5a6a7119046d97ee509d017080c6aa.js"},
			},
			getURL: "http://example.com/js/file.bf5a6a7119046d97ee509d017080c6aa.js",
		},
		{
			cname:       3,
			path:        "test",
			fingerprint: false,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.css"},
				{"js/file.js", "/js/file.js"},
			},
			getURL: "http://example.com/js/file.js",
		},
	} {
		t.Logf("case %d", c.cname)

		m, err := NewManager(Config{Fingerprint: c.fingerprint}, c.path)
		if err != nil {
			t.Fatal(err)
		}

		for _, c := range c.pathMap {
			if got, want := m.AssetPath(c[0]), c[1]; got != want {
				t.Errorf("m.AssetPath(%s) = %s; want %s", c[0], got, want)
			}
		}

		req, err := http.NewRequest("GET", c.getURL, nil)
		if err != nil {
			t.Fatal(err)
		}
		w := httptest.NewRecorder()
		var body bytes.Buffer
		w.Body = &body
		m.ServeHTTP(w, req)
		if got, want := body.String(), "var code = 'test';\n"; got != want {
			t.Errorf("body.String() = %s; want %s", got, want)
		}
	}
}

func TestHostname(t *testing.T) {
	m, _ := NewManager(Config{Hostname: "https://cdn.com"}, "test")
	if got, want := m.AssetPath("js/file.js"), "https://cdn.com/js/file.js"; got != want {
		t.Errorf("m.AssetPath(js/file.js) = %s; want %s", got, want)
	}
}

func TestScriptAndLink(t *testing.T) {
	m, err := NewManager(Config{Fingerprint: true}, "test")
	if err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript" attr="val<tag>"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
	if got, want := m.Link("css/file.css"), template.HTML(`<link href="/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css" rel="stylesheet" type="text/css"></link>`); got != want {
		t.Errorf("m.Link(css/file.css) = %s; want %s", got, want)
	}

	if err := m.SetConfig(Config{Fingerprint: true, SubresourceIntegrity: true}); err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript" attr="val<tag>" integrity="sha384-ikdSg6BDd7ZQH0wpe7EtsWSf4DDnkWmgulB70NrXja4doy1lTsql2ajoHay1xkiu"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}

	if err := m.SetConfig(Config{SubresourceIntegrity: true}); err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.js" type="text/javascript" attr="val<tag>" integrity=""></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
}
<commit_msg>Add TestIntegrity<commit_after>package assettube

import (
	"bytes"
	"html/template"
	"net/http"
	"net/http/httptest"
	"testing"

	_ "github.com/bom-d-van/sidekick"
)

func TestNewAndAssetPath(t *testing.T) {
	for _, c := range []struct {
		cname       int
		path        string
		fingerprint bool
		pathMap     [][2]string
		getURL      string
	}{
		{
			cname:       1,
			path:        "/Users/bom_d_van/Code/go/workspace/src/github.com/theplant/assettube/test",
			fingerprint: true,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css"},
				{"js/file.js", "/js/file.bf5a6a7119046d97ee509d017080c6aa.js"},
			},
			getURL: "http://example.com/js/file.bf5a6a7119046d97ee509d017080c6aa.js",
		},
		{
			cname:       2,
			path:        "test",
			fingerprint: true,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css"},
				{"js/file.js", "/js/file.bf5a6a7119046d97ee509d017080c6aa.js"},
			},
			getURL: "http://example.com/js/file.bf5a6a7119046d97ee509d017080c6aa.js",
		},
		{
			cname:       3,
			path:        "test",
			fingerprint: false,
			pathMap: [][2]string{
				{"css/file.css", "/css/file.css"},
				{"js/file.js", "/js/file.js"},
			},
			getURL: "http://example.com/js/file.js",
		},
	} {
		t.Logf("case %d", c.cname)

		m, err := NewManager(Config{Fingerprint: c.fingerprint}, c.path)
		if err != nil {
			t.Fatal(err)
		}

		for _, c := range c.pathMap {
			if got, want := m.AssetPath(c[0]), c[1]; got != want {
				t.Errorf("m.AssetPath(%s) = %s; want %s", c[0], got, want)
			}
		}

		req, err := http.NewRequest("GET", c.getURL, nil)
		if err != nil {
			t.Fatal(err)
		}
		w := httptest.NewRecorder()
		var body bytes.Buffer
		w.Body = &body
		m.ServeHTTP(w, req)
		if got, want := body.String(), "var code = 'test';\n"; got != want {
			t.Errorf("body.String() = %s; want %s", got, want)
		}
	}
}

func TestHostname(t *testing.T) {
	m, _ := NewManager(Config{Hostname: "https://cdn.com"}, "test")
	if got, want := m.AssetPath("js/file.js"), "https://cdn.com/js/file.js"; got != want {
		t.Errorf("m.AssetPath(js/file.js) = %s; want %s", got, want)
	}
}

func TestIntegrity(t *testing.T) {
	m, err := NewManager(Config{SubresourceIntegrity: true, Fingerprint: true}, "test")
	if err != nil {
		t.Fatal(err)
	}
	if got, want := m.Integrity("js/file.js"), "sha384-ikdSg6BDd7ZQH0wpe7EtsWSf4DDnkWmgulB70NrXja4doy1lTsql2ajoHay1xkiu"; got != want {
		t.Errorf(`m.Integrity("js/file.js") = %s; want %s`, got, want)
	}
	if err := m.SetConfig(Config{SubresourceIntegrity: true, Fingerprint: true, HashType: HTSHA512}); err != nil {
		t.Fatal(err)
	}
	if got, want := m.Integrity("js/file.js"), "sha512-ju5jHaaN+e9x7kaWXjRO8fgoYCzKsw7lAzY3uzpjSAF3FJsKoIYAhvZ6Plxp5hgFyu0ho7a7U6mAWxcvKrC+Dw"; got != want {
		t.Errorf(`m.Integrity("js/file.js") = %s; want %s`, got, want)
	}
}

func TestScriptAndLink(t *testing.T) {
	m, err := NewManager(Config{Fingerprint: true}, "test")
	if err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript" attr="val<tag>"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
	if got, want := m.Link("css/file.css"), template.HTML(`<link href="/css/file.0bc77612dba2d5253636e9f0b0d3e6cc.css" rel="stylesheet" type="text/css"></link>`); got != want {
		t.Errorf("m.Link(css/file.css) = %s; want %s", got, want)
	}

	if err := m.SetConfig(Config{Fingerprint: true, SubresourceIntegrity: true}); err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.bf5a6a7119046d97ee509d017080c6aa.js" type="text/javascript" attr="val<tag>" integrity="sha384-ikdSg6BDd7ZQH0wpe7EtsWSf4DDnkWmgulB70NrXja4doy1lTsql2ajoHay1xkiu"></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}

	if err := m.SetConfig(Config{SubresourceIntegrity: true}); err != nil {
		t.Error(err)
	}
	if got, want := m.Script("js/file.js", "attr", "val<tag>"), template.HTML(`<script src="/js/file.js" type="text/javascript" attr="val<tag>" integrity=""></script>`); got != want {
		t.Errorf("m.Script(js/file.js) = %s; want %s", got, want)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

const CANT_CREATE_SITE_ERROR = `Can't create.
Try again in a few seconds or contact codumentary.com@gmail.com for help`

// ApiError stores response error messages and HTTP status code.
type ApiError struct {
	Message    string
	StatusCode int
}

// NewApiError returns an ApiError with the given status code and a message made from
// the format string.
func NewApiError(statusCode int, format string, a ...interface{}) error {
	return &ApiError{
		Message:    fmt.Sprintf(format, a...),
		StatusCode: statusCode,
	}
}

// ApiError.Error returns error message.
func (error *ApiError) Error() string {
	return error.Message
}

func InvalidDelayError(delay string) error {
	return NewApiError(http.StatusBadRequest,
		"Oopsie daisy! Could not convert delay <%s> to float.", delay)
}

func DelayIsTooBigError(delayInSeconds float64) error {
	return NewApiError(http.StatusBadRequest,
		"Oopsie daisy! Delay can't be greater then %d seconds, got delay %.0f seconds.",
		MAX_DELAY, delayInSeconds)
}

// ChangeBuiltinSiteError returns an ApiError when somebody tries to
// change a built-in site.
func ChangeBuiltinSiteError() error {
	return NewApiError(http.StatusForbidden, "Oopsie daisy! You can't change builtin sites.")
}

func UnknownSiteError(site string) error {
	return NewApiError(http.StatusNotFound, "Oopsie daisy! Site <%s> doesn't exist.", site)
}

func CantCreateSiteError() error {
	return NewApiError(http.StatusInternalServerError, CANT_CREATE_SITE_ERROR)
}
<commit_msg>Fix typo in comment<commit_after>package main

import (
	"fmt"
	"net/http"
)

const CANT_CREATE_SITE_ERROR = `Can't create.
Try again in a few seconds or contact codumentary.com@gmail.com for help`

// ApiError stores response error message and HTTP status code.
type ApiError struct {
	Message    string
	StatusCode int
}

// NewApiError returns an ApiError with the given status code and a message made from
// the format string.
func NewApiError(statusCode int, format string, a ...interface{}) error {
	return &ApiError{
		Message:    fmt.Sprintf(format, a...),
		StatusCode: statusCode,
	}
}

// ApiError.Error returns error message.
func (error *ApiError) Error() string {
	return error.Message
}

func InvalidDelayError(delay string) error {
	return NewApiError(http.StatusBadRequest,
		"Oopsie daisy! Could not convert delay <%s> to float.", delay)
}

func DelayIsTooBigError(delayInSeconds float64) error {
	return NewApiError(http.StatusBadRequest,
		"Oopsie daisy! Delay can't be greater then %d seconds, got delay %.0f seconds.",
		MAX_DELAY, delayInSeconds)
}

// ChangeBuiltinSiteError returns an ApiError when somebody tries to
// change a built-in site.
func ChangeBuiltinSiteError() error {
	return NewApiError(http.StatusForbidden, "Oopsie daisy! You can't change builtin sites.")
}

func UnknownSiteError(site string) error {
	return NewApiError(http.StatusNotFound, "Oopsie daisy! Site <%s> doesn't exist.", site)
}

func CantCreateSiteError() error {
	return NewApiError(http.StatusInternalServerError, CANT_CREATE_SITE_ERROR)
}
<|endoftext|>


<commit_before>package incoming

import (
	"encoding/json"
	"net/http"
	"net/url"
)

type IncomingConf struct {
	WebHookUrl string
}

// https://api.slack.com/docs/attachments

type Field struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool `json:"short"`
}

type Attachment struct {
	Fallback string `json:"fallback"`
	Pretext  string `json:"pretext"`
	Color    string `json:"color"`
	Fields   []*Field `json:"fields"`
}

type Payload struct{
	Attachments []*Attachment `json:"attachments"`
}

func Post(conf IncomingConf, payload Payload) error {
	p, err := json.Marshal(&payload)
	if err != nil {
		return err
	}

	_, err = http.PostForm(conf.WebHookUrl, url.Values{
		"payload": []string{string(p)},
	})

	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fixed<commit_after>package incoming

import (
	"encoding/json"
	"net/http"
	"net/url"
)

// Config struct
type Config struct {
	WebHookURL string
}

// https://api.slack.com/docs/attachments

// Field struct
type Field struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}

// Attachment struct
type Attachment struct {
	Fallback string  `json:"fallback"`
	Pretext  string  `json:"pretext"`
	Color    string  `json:"color"`
	Fields   []Field `json:"fields"`
}

// Payload struct
type Payload struct {
	Attachments []Attachment `json:"attachments"`
}

// Post method
func Post(conf Config, payload Payload) error {
	p, err := json.Marshal(&payload)
	if err != nil {
		return err
	}

	_, err = http.PostForm(conf.WebHookURL, url.Values{
		"payload": []string{string(p)},
	})

	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>