<commit_before>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package azure

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/compose-spec/compose-go/cli"
	"github.com/compose-spec/compose-go/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/docker/api/azure/convert"
	"github.com/docker/api/azure/login"
	"github.com/docker/api/backend"
	"github.com/docker/api/compose"
	"github.com/docker/api/containers"
	apicontext "github.com/docker/api/context"
	"github.com/docker/api/context/cloud"
	"github.com/docker/api/context/store"
	"github.com/docker/api/errdefs"
)

const (
	singleContainerTag        = "docker-single-container"
	composeContainerTag       = "docker-compose-application"
	composeContainerSeparator = "_"
	statusUnknown             = "Unknown"
)

// ErrNoSuchContainer is returned when the mentioned container does not exist
var ErrNoSuchContainer = errors.New("no such container")

func init() {
	backend.Register("aci", "aci", service, getCloudService)
}

func service(ctx context.Context) (backend.Service, error) {
	contextStore := store.ContextStore(ctx)
	currentContext := apicontext.CurrentContext(ctx)
	var aciContext store.AciContext

	if err := contextStore.GetEndpoint(currentContext, &aciContext); err != nil {
		return nil, err
	}

	return getAciAPIService(aciContext), nil
}

func getCloudService() (cloud.Service, error) {
	service, err := login.NewAzureLoginService()
	if err != nil {
		return nil, err
	}
	return &aciCloudService{
		loginService: service,
	}, nil
}

func getAciAPIService(aciCtx store.AciContext) *aciAPIService {
	return &aciAPIService{
		aciContainerService: &aciContainerService{
			ctx: aciCtx,
		},
		aciComposeService: &aciComposeService{
			ctx: aciCtx,
		},
	}
}

type aciAPIService struct {
	*aciContainerService
	*aciComposeService
}

func (a *aciAPIService) ContainerService() containers.Service {
	return a.aciContainerService
}

func (a *aciAPIService) ComposeService() compose.Service {
	return a.aciComposeService
}

type aciContainerService struct {
	ctx store.AciContext
}

func (cs *aciContainerService) List(ctx context.Context, _ bool) ([]containers.Container, error) {
	groupsClient, err := getContainerGroupsClient(cs.ctx.SubscriptionID)
	if err != nil {
		return nil, err
	}
	var containerGroups []containerinstance.ContainerGroup
	result, err := groupsClient.ListByResourceGroup(ctx, cs.ctx.ResourceGroup)
	if err != nil {
		return []containers.Container{}, err
	}

	for result.NotDone() {
		containerGroups = append(containerGroups, result.Values()...)
		if err := result.NextWithContext(ctx); err != nil {
			return []containers.Container{}, err
		}
	}

	var res []containers.Container
	for _, containerGroup := range containerGroups {
		group, err := groupsClient.Get(ctx, cs.ctx.ResourceGroup, *containerGroup.Name)
		if err != nil {
			return []containers.Container{}, err
		}

		if _, ok := group.Tags[singleContainerTag]; ok {
			if group.Containers == nil || len(*group.Containers) < 1 {
				return []containers.Container{}, fmt.Errorf("no containers to run")
			}
			container := (*group.Containers)[0]
			c := getContainer(*containerGroup.Name, group.IPAddress, container)
			res = append(res, c)
			continue
		}

		for _, container := range *group.Containers {
			var containerID string
			// don't list sidecar container
			if *container.Name == convert.ComposeDNSSidecarName {
				continue
			}
			containerID = *containerGroup.Name + composeContainerSeparator + *container.Name
			c := getContainer(containerID, group.IPAddress, container)
			res = append(res, c)
		}
	}

	return res, nil
}

func getContainer(containerID string, ipAddress *containerinstance.IPAddress, container containerinstance.Container) containers.Container {
	status := statusUnknown
	if container.InstanceView != nil && container.InstanceView.CurrentState != nil {
		status = *container.InstanceView.CurrentState.State
	}
	return containers.Container{
		ID:     containerID,
		Image:  *container.Image,
		Status: status,
		Ports:  convert.ToPorts(ipAddress, *container.Ports),
	}
}

func (cs *aciContainerService) Run(ctx context.Context, r containers.ContainerConfig) error {
	if strings.Contains(r.ID, composeContainerSeparator) {
		return errors.New(fmt.Sprintf("invalid container name. ACI container name cannot include %q", composeContainerSeparator))
	}

	project, err := convert.ContainerToComposeProject(r)
	if err != nil {
		return err
	}

	logrus.Debugf("Running container %q with name %q\n", r.Image, r.ID)
	groupDefinition, err := convert.ToContainerGroup(cs.ctx, project)
	if err != nil {
		return err
	}
	addTag(groupDefinition, singleContainerTag)

	return createACIContainers(ctx, cs.ctx, groupDefinition)
}

func addTag(groupDefinition containerinstance.ContainerGroup, tagName string) {
	if groupDefinition.Tags == nil {
		groupDefinition.Tags = make(map[string]*string, 1)
	}
	groupDefinition.Tags[tagName] = to.StringPtr("")
}

func (cs *aciContainerService) Stop(ctx context.Context, containerName string, timeout *uint32) error {
	return errdefs.ErrNotImplemented
}

func getGroupAndContainerName(containerID string) (string, string) {
	tokens := strings.Split(containerID, composeContainerSeparator)
	groupName := tokens[0]
	containerName := groupName
	if len(tokens) > 1 {
		containerName = tokens[len(tokens)-1]
		groupName = containerID[:len(containerID)-(len(containerName)+1)]
	}
	return groupName, containerName
}

func (cs *aciContainerService) Exec(ctx context.Context, name string, request containers.ExecRequest) error {
	err := verifyExecCommand(request.Command)
	if err != nil {
		return err
	}
	groupName, containerAciName := getGroupAndContainerName(name)
	containerExecResponse, err := execACIContainer(ctx, cs.ctx, request.Command, groupName, containerAciName)
	if err != nil {
		return err
	}

	return exec(
		context.Background(),
		*containerExecResponse.WebSocketURI,
		*containerExecResponse.Password,
		request,
	)
}

func verifyExecCommand(command string) error {
	tokens := strings.Split(command, " ")
	if len(tokens) > 1 {
		return errors.New("ACI exec command does not accept arguments to the command. " +
			"Only the binary should be specified")
	}
	return nil
}

func (cs *aciContainerService) Logs(ctx context.Context, containerName string, req containers.LogsRequest) error {
	groupName, containerAciName := getGroupAndContainerName(containerName)
	var tail *int32

	if req.Follow {
		return streamLogs(ctx, cs.ctx, groupName, containerAciName, req)
	}

	if req.Tail != "all" {
		reqTail, err := strconv.Atoi(req.Tail)
		if err != nil {
			return err
		}
		i32 := int32(reqTail)
		tail = &i32
	}

	logs, err := getACIContainerLogs(ctx, cs.ctx, groupName, containerAciName, tail)
	if err != nil {
		return err
	}

	_, err = fmt.Fprint(req.Writer, logs)
	return err
}

func (cs *aciContainerService) Delete(ctx context.Context, containerID string, _ bool) error {
	groupName, containerName := getGroupAndContainerName(containerID)
	if groupName != containerID {
		msg := "cannot delete service %q from compose application %q, you can delete the entire compose app with docker compose down --project-name %s"
		return errors.New(fmt.Sprintf(msg, containerName, groupName, groupName))
	}
	cg, err := deleteACIContainerGroup(ctx, cs.ctx, groupName)
	if err != nil {
		return err
	}
	if cg.StatusCode == http.StatusNoContent {
		return ErrNoSuchContainer
	}

	return err
}

func (cs *aciContainerService) Inspect(ctx context.Context, containerID string) (containers.Container, error) {
	groupName, containerName := getGroupAndContainerName(containerID)

	cg, err := getACIContainerGroup(ctx, cs.ctx, groupName)
	if err != nil {
		return containers.Container{}, err
	}
	if cg.StatusCode == http.StatusNoContent {
		return containers.Container{}, ErrNoSuchContainer
	}

	var cc containerinstance.Container
	var found = false
	for _, c := range *cg.Containers {
		if to.String(c.Name) == containerName {
			cc = c
			found = true
			break
		}
	}
	if !found {
		return containers.Container{}, ErrNoSuchContainer
	}

	return convert.ContainerGroupToContainer(containerID, cg, cc)
}

type aciComposeService struct {
	ctx store.AciContext
}

func (cs *aciComposeService) Up(ctx context.Context, opts cli.ProjectOptions) error {
	project, err := cli.ProjectFromOptions(&opts)
	if err != nil {
		return err
	}
	logrus.Debugf("Up on project with name %q\n", project.Name)
	groupDefinition, err := convert.ToContainerGroup(cs.ctx, *project)
	addTag(groupDefinition, composeContainerTag)

	if err != nil {
		return err
	}
	return createOrUpdateACIContainers(ctx, cs.ctx, groupDefinition)
}

func (cs *aciComposeService) Down(ctx context.Context, opts cli.ProjectOptions) error {
	var project types.Project

	if opts.Name != "" {
		project = types.Project{Name: opts.Name}
	} else {
		fullProject, err := cli.ProjectFromOptions(&opts)
		if err != nil {
			return err
		}
		project = *fullProject
	}
	logrus.Debugf("Down on project with name %q\n", project.Name)

	cg, err := deleteACIContainerGroup(ctx, cs.ctx, project.Name)
	if err != nil {
		return err
	}
	if cg.StatusCode == http.StatusNoContent {
		return ErrNoSuchContainer
	}

	return err
}

type aciCloudService struct {
	loginService login.AzureLoginService
}

func (cs *aciCloudService) Login(ctx context.Context, params map[string]string) error {
	return cs.loginService.Login(ctx, params[login.TenantIDLoginParam])
}

func (cs *aciCloudService) Logout(ctx context.Context) error {
	return cs.loginService.Logout(ctx)
}

func (cs *aciCloudService) CreateContextData(ctx context.Context, params map[string]string) (interface{}, string, error) {
	contextHelper := newContextCreateHelper()
	return contextHelper.createContextData(ctx, params)
}
<commit_msg>Fixed wrong error message<commit_after>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package azure

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance"
	"github.com/Azure/go-autorest/autorest/to"
	"github.com/compose-spec/compose-go/cli"
	"github.com/compose-spec/compose-go/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/docker/api/azure/convert"
	"github.com/docker/api/azure/login"
	"github.com/docker/api/backend"
	"github.com/docker/api/compose"
	"github.com/docker/api/containers"
	apicontext "github.com/docker/api/context"
	"github.com/docker/api/context/cloud"
	"github.com/docker/api/context/store"
	"github.com/docker/api/errdefs"
)

const (
	singleContainerTag        = "docker-single-container"
	composeContainerTag       = "docker-compose-application"
	composeContainerSeparator = "_"
	statusUnknown             = "Unknown"
)

// ErrNoSuchContainer is returned when the mentioned container does not exist
var ErrNoSuchContainer = errors.New("no such container")

func init() {
	backend.Register("aci", "aci", service, getCloudService)
}

func service(ctx context.Context) (backend.Service, error) {
	contextStore := store.ContextStore(ctx)
	currentContext := apicontext.CurrentContext(ctx)
	var aciContext store.AciContext

	if err := contextStore.GetEndpoint(currentContext, &aciContext); err != nil {
		return nil, err
	}

	return getAciAPIService(aciContext), nil
}

func getCloudService() (cloud.Service, error) {
	service, err := login.NewAzureLoginService()
	if err != nil {
		return nil, err
	}
	return &aciCloudService{
		loginService: service,
	}, nil
}

func getAciAPIService(aciCtx store.AciContext) *aciAPIService {
	return &aciAPIService{
		aciContainerService: &aciContainerService{
			ctx: aciCtx,
		},
		aciComposeService: &aciComposeService{
			ctx: aciCtx,
		},
	}
}

type aciAPIService struct {
	*aciContainerService
	*aciComposeService
}

func (a *aciAPIService) ContainerService() containers.Service {
	return a.aciContainerService
}

func (a *aciAPIService) ComposeService() compose.Service {
	return a.aciComposeService
}

type aciContainerService struct {
	ctx store.AciContext
}

func (cs *aciContainerService) List(ctx context.Context, _ bool) ([]containers.Container, error) {
	groupsClient, err := getContainerGroupsClient(cs.ctx.SubscriptionID)
	if err != nil {
		return nil, err
	}
	var containerGroups []containerinstance.ContainerGroup
	result, err := groupsClient.ListByResourceGroup(ctx, cs.ctx.ResourceGroup)
	if err != nil {
		return []containers.Container{}, err
	}

	for result.NotDone() {
		containerGroups = append(containerGroups, result.Values()...)
		if err := result.NextWithContext(ctx); err != nil {
			return []containers.Container{}, err
		}
	}

	var res []containers.Container
	for _, containerGroup := range containerGroups {
		group, err := groupsClient.Get(ctx, cs.ctx.ResourceGroup, *containerGroup.Name)
		if err != nil {
			return []containers.Container{}, err
		}

		if _, ok := group.Tags[singleContainerTag]; ok {
			if group.Containers == nil || len(*group.Containers) < 1 {
				return []containers.Container{}, fmt.Errorf("found no containers in ACI container group %s", *containerGroup.Name)
			}
			container := (*group.Containers)[0]
			c := getContainer(*containerGroup.Name, group.IPAddress, container)
			res = append(res, c)
			continue
		}

		for _, container := range *group.Containers {
			var containerID string
			// don't list sidecar container
			if *container.Name == convert.ComposeDNSSidecarName {
				continue
			}
			containerID = *containerGroup.Name + composeContainerSeparator + *container.Name
			c := getContainer(containerID, group.IPAddress, container)
			res = append(res, c)
		}
	}

	return res, nil
}

func getContainer(containerID string, ipAddress *containerinstance.IPAddress, container containerinstance.Container) containers.Container {
	status := statusUnknown
	if container.InstanceView != nil && container.InstanceView.CurrentState != nil {
		status = *container.InstanceView.CurrentState.State
	}
	return containers.Container{
		ID:     containerID,
		Image:  *container.Image,
		Status: status,
		Ports:  convert.ToPorts(ipAddress, *container.Ports),
	}
}

func (cs *aciContainerService) Run(ctx context.Context, r containers.ContainerConfig) error {
	if strings.Contains(r.ID, composeContainerSeparator) {
		return errors.New(fmt.Sprintf("invalid container name. ACI container name cannot include %q", composeContainerSeparator))
	}

	project, err := convert.ContainerToComposeProject(r)
	if err != nil {
		return err
	}

	logrus.Debugf("Running container %q with name %q\n", r.Image, r.ID)
	groupDefinition, err := convert.ToContainerGroup(cs.ctx, project)
	if err != nil {
		return err
	}
	addTag(groupDefinition, singleContainerTag)

	return createACIContainers(ctx, cs.ctx, groupDefinition)
}

func addTag(groupDefinition containerinstance.ContainerGroup, tagName string) {
	if groupDefinition.Tags == nil {
		groupDefinition.Tags = make(map[string]*string, 1)
	}
	groupDefinition.Tags[tagName] = to.StringPtr("")
}

func (cs *aciContainerService) Stop(ctx context.Context, containerName string, timeout *uint32) error {
	return errdefs.ErrNotImplemented
}

func getGroupAndContainerName(containerID string) (string, string) {
	tokens := strings.Split(containerID, composeContainerSeparator)
	groupName := tokens[0]
	containerName := groupName
	if len(tokens) > 1 {
		containerName = tokens[len(tokens)-1]
		groupName = containerID[:len(containerID)-(len(containerName)+1)]
	}
	return groupName, containerName
}

func (cs *aciContainerService) Exec(ctx context.Context, name string, request containers.ExecRequest) error {
	err := verifyExecCommand(request.Command)
	if err != nil {
		return err
	}
	groupName, containerAciName := getGroupAndContainerName(name)
	containerExecResponse, err := execACIContainer(ctx, cs.ctx, request.Command, groupName, containerAciName)
	if err != nil {
		return err
	}

	return exec(
		context.Background(),
		*containerExecResponse.WebSocketURI,
		*containerExecResponse.Password,
		request,
	)
}

func verifyExecCommand(command string) error {
	tokens := strings.Split(command, " ")
	if len(tokens) > 1 {
		return errors.New("ACI exec command does not accept arguments to the command. " +
			"Only the binary should be specified")
	}
	return nil
}

func (cs *aciContainerService) Logs(ctx context.Context, containerName string, req containers.LogsRequest) error {
	groupName, containerAciName := getGroupAndContainerName(containerName)
	var tail *int32

	if req.Follow {
		return streamLogs(ctx, cs.ctx, groupName, containerAciName, req)
	}

	if req.Tail != "all" {
		reqTail, err := strconv.Atoi(req.Tail)
		if err != nil {
			return err
		}
		i32 := int32(reqTail)
		tail = &i32
	}

	logs, err := getACIContainerLogs(ctx, cs.ctx, groupName, containerAciName, tail)
	if err != nil {
		return err
	}

	_, err = fmt.Fprint(req.Writer, logs)
	return err
}

func (cs *aciContainerService) Delete(ctx context.Context, containerID string, _ bool) error {
	groupName, containerName := getGroupAndContainerName(containerID)
	if groupName != containerID {
		msg := "cannot delete service %q from compose application %q, you can delete the entire compose app with docker compose down --project-name %s"
		return errors.New(fmt.Sprintf(msg, containerName, groupName, groupName))
	}
	cg, err := deleteACIContainerGroup(ctx, cs.ctx, groupName)
	if err != nil {
		return err
	}
	if cg.StatusCode == http.StatusNoContent {
		return ErrNoSuchContainer
	}

	return err
}

func (cs *aciContainerService) Inspect(ctx context.Context, containerID string) (containers.Container, error) {
	groupName, containerName := getGroupAndContainerName(containerID)

	cg, err := getACIContainerGroup(ctx, cs.ctx, groupName)
	if err != nil {
		return containers.Container{}, err
	}
	if cg.StatusCode == http.StatusNoContent {
		return containers.Container{}, ErrNoSuchContainer
	}

	var cc containerinstance.Container
	var found = false
	for _, c := range *cg.Containers {
		if to.String(c.Name) == containerName {
			cc = c
			found = true
			break
		}
	}
	if !found {
		return containers.Container{}, ErrNoSuchContainer
	}

	return convert.ContainerGroupToContainer(containerID, cg, cc)
}

type aciComposeService struct {
	ctx store.AciContext
}

func (cs *aciComposeService) Up(ctx context.Context, opts cli.ProjectOptions) error {
	project, err := cli.ProjectFromOptions(&opts)
	if err != nil {
		return err
	}
	logrus.Debugf("Up on project with name %q\n", project.Name)
	groupDefinition, err := convert.ToContainerGroup(cs.ctx, *project)
	addTag(groupDefinition, composeContainerTag)

	if err != nil {
		return err
	}
	return createOrUpdateACIContainers(ctx, cs.ctx, groupDefinition)
}

func (cs *aciComposeService) Down(ctx context.Context, opts cli.ProjectOptions) error {
	var project types.Project

	if opts.Name != "" {
		project = types.Project{Name: opts.Name}
	} else {
		fullProject, err := cli.ProjectFromOptions(&opts)
		if err != nil {
			return err
		}
		project = *fullProject
	}
	logrus.Debugf("Down on project with name %q\n", project.Name)

	cg, err := deleteACIContainerGroup(ctx, cs.ctx, project.Name)
	if err != nil {
		return err
	}
	if cg.StatusCode == http.StatusNoContent {
		return ErrNoSuchContainer
	}

	return err
}

type aciCloudService struct {
	loginService login.AzureLoginService
}

func (cs *aciCloudService) Login(ctx context.Context, params map[string]string) error {
	return cs.loginService.Login(ctx, params[login.TenantIDLoginParam])
}

func (cs *aciCloudService) Logout(ctx context.Context) error {
	return cs.loginService.Logout(ctx)
}

func (cs *aciCloudService) CreateContextData(ctx context.Context, params map[string]string) (interface{}, string, error) {
	contextHelper := newContextCreateHelper()
	return contextHelper.createContextData(ctx, params)
}
<|endoftext|>


<commit_before>package recycleme

import (
	"io/ioutil"
	"log"
	"strings"
	"testing"
)

var blacklistJSON = `{
  "Blacklist": [
    "http://www.upcitemdb.com/upc/3057640136573"
  ]
}`

func TestBlacklist(t *testing.T) {
	url := "http://www.upcitemdb.com/upc/3057640136573"
	logger := log.New(ioutil.Discard, "", 0)
	LoadBlacklistJSON(strings.NewReader(blacklistJSON), logger)
	if !Blacklist.Contains(url) {
		t.Fatalf("%v not in blacklist", url)
	}
	_, err := UpcItemDbFetcher.Fetch("3057640136573")
	if err == nil {
		t.Fatalf("%v not blacklisted", url)
	}
	if err.(*ProductError).err != errBlacklisted {
		t.Fatalf("not a blacklist error: %v", err)
	}
}

func TestAmazonFetcher(t *testing.T) {
	amazonFetcher, err := newAmazonURLFetcher()

	if amazonFetcher.SecretKey == "" || amazonFetcher.AccessKey == "" || amazonFetcher.AssociateTag == "" {
		t.Log("Missing either AccessKey, SecretKey or AssociateTag. AmazonFetcher will not be tested")
		return
	}
	_, err = amazonFetcher.Fetch("4006381333634")
	if err != nil {
		if err.(*ProductError).err != errTooManyProducts {
			t.Fatal(err)
		}
	}
	p, err := amazonFetcher.Fetch("5021991938818")
	if err != nil {
		t.Fatal(err)
	}
	if p.Name != "Clipper Thé Vert Biologique 20 infusettes" ||
		p.EAN != "5021991938818" ||
		p.URL != "webservices.amazon.fr" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/517qE9owUDL.jpg" ||
		p.WebsiteURL != "http://www.amazon.fr/Clipper-Th%C3%A9-Vert-Biologique-infusettes/dp/B011C4L3S0%3FSubscriptionId%3DAKIAIOSACIYSSJVD3IQA%26tag%3Dhowtorecme-21%26linkCode%3Dxm2%26camp%3D2025%26creative%3D165953%26creativeASIN%3DB011C4L3S0" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetchers(t *testing.T) {
	p, err := UpcItemDbFetcher.Fetch("5029053038896")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Kleenex tissues in a Christmas House box" ||
		p.EAN != "5029053038896" ||
		p.URL != "http://www.upcitemdb.com/upc/5029053038896" ||
		p.ImageURL != "http://www.staples.co.uk/content/images/product/428056_1_xnl.jpg" ||
		p.WebsiteURL != "http://www.upcitemdb.com/upc/5029053038896" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = UpcItemDbFetcher.Fetch("4006381333634")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Stabilo Boss Original Highlighter Blue" ||
		p.EAN != "4006381333634" ||
		p.URL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.WebsiteURL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/41SfgGjtcpL._SL160_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = OpenFoodFactsFetcher.Fetch("7613034383808")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Four à Pierre Royale" || p.EAN != "7613034383808" ||
		p.URL != "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json" ||
		p.WebsiteURL != "http://fr.openfoodfacts.org/produit/7613034383808/" ||
		p.ImageURL != "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = IsbnSearchFetcher.Fetch("9782501104265")
	if err != nil {
		t.Error(err)
	} else if p.Name != "le rugby c'est pas sorcier" || p.EAN != "9782501104265" ||
		p.URL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.WebsiteURL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/51V4iimUfML._SL194_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetcher(t *testing.T) {
	fetcher, _ := NewDefaultFetcher()
	_, err := fetcher.Fetch("5029053038896")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7613034383808")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7640140337517")
	if err == nil {
		t.Fatal(err)
	}
}
<commit_msg>Fix test with url update<commit_after>package recycleme

import (
	"io/ioutil"
	"log"
	"strings"
	"testing"
)

var blacklistJSON = `{
  "Blacklist": [
    "http://www.upcitemdb.com/upc/3057640136573"
  ]
}`

func TestBlacklist(t *testing.T) {
	url := "http://www.upcitemdb.com/upc/3057640136573"
	logger := log.New(ioutil.Discard, "", 0)
	LoadBlacklistJSON(strings.NewReader(blacklistJSON), logger)
	if !Blacklist.Contains(url) {
		t.Fatalf("%v not in blacklist", url)
	}
	_, err := UpcItemDbFetcher.Fetch("3057640136573")
	if err == nil {
		t.Fatalf("%v not blacklisted", url)
	}
	if err.(*ProductError).err != errBlacklisted {
		t.Fatalf("not a blacklist error: %v", err)
	}
}

func TestAmazonFetcher(t *testing.T) {
	amazonFetcher, err := newAmazonURLFetcher()

	if amazonFetcher.SecretKey == "" || amazonFetcher.AccessKey == "" || amazonFetcher.AssociateTag == "" {
		t.Log("Missing either AccessKey, SecretKey or AssociateTag. AmazonFetcher will not be tested")
		return
	}
	_, err = amazonFetcher.Fetch("4006381333634")
	if err != nil {
		if err.(*ProductError).err != errTooManyProducts {
			t.Fatal(err)
		}
	}
	p, err := amazonFetcher.Fetch("5021991938818")
	if err != nil {
		t.Fatal(err)
	}
	if p.Name != "Clipper Thé Vert Biologique 20 infusettes" ||
		p.EAN != "5021991938818" ||
		p.URL != "webservices.amazon.fr/5021991938818" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/517qE9owUDL.jpg" ||
		p.WebsiteURL != "http://www.amazon.fr/Clipper-Th%C3%A9-Vert-Biologique-infusettes/dp/B011C4L3S0%3FSubscriptionId%3DAKIAIOSACIYSSJVD3IQA%26tag%3Dhowtorecme-21%26linkCode%3Dxm2%26camp%3D2025%26creative%3D165953%26creativeASIN%3DB011C4L3S0" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetchers(t *testing.T) {
	p, err := UpcItemDbFetcher.Fetch("5029053038896")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Kleenex tissues in a Christmas House box" ||
		p.EAN != "5029053038896" ||
		p.URL != "http://www.upcitemdb.com/upc/5029053038896" ||
		p.ImageURL != "http://www.staples.co.uk/content/images/product/428056_1_xnl.jpg" ||
		p.WebsiteURL != "http://www.upcitemdb.com/upc/5029053038896" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = UpcItemDbFetcher.Fetch("4006381333634")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Stabilo Boss Original Highlighter Blue" ||
		p.EAN != "4006381333634" ||
		p.URL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.WebsiteURL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/41SfgGjtcpL._SL160_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = OpenFoodFactsFetcher.Fetch("7613034383808")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Four à Pierre Royale" || p.EAN != "7613034383808" ||
		p.URL != "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json" ||
		p.WebsiteURL != "http://fr.openfoodfacts.org/produit/7613034383808/" ||
		p.ImageURL != "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = IsbnSearchFetcher.Fetch("9782501104265")
	if err != nil {
		t.Error(err)
	} else if p.Name != "le rugby c'est pas sorcier" || p.EAN != "9782501104265" ||
		p.URL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.WebsiteURL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/51V4iimUfML._SL194_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetcher(t *testing.T) {
	fetcher, _ := NewDefaultFetcher()
	_, err := fetcher.Fetch("5029053038896")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7613034383808")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7640140337517")
	if err == nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The ACH Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package ach

import (
	"fmt"
	"strings"
	"time"
)

// Errors specific to a File Header Record
var (
	msgRecordType       = "received expecting %d"
	msgRecordSize       = "is not 094"
	msgBlockingFactor   = "is not 10"
	msgFormatCode       = "is not 1"
	msgFileCreationDate = "was created before " + time.Now().String()
)

// FileHeader is a Record designating physical file characteristics and identify
// the origin (sending point) and destination (receiving point) of the entries
// contained in the file. The file header also includes creation date and time
// fields which can be used to uniquely identify a file.
type FileHeader struct {
	// RecordType defines the type of record in the block. headerPos
	recordType string

	// PriorityCode consists of the numerals 01
	priorityCode string

	// ImmediateDestination contains the Routing Number of the ACH Operator or receiving
	// point to which the file is being sent. The 10 character field begins with
	// a blank in the first position, followed by the four digit Federal Reserve
	// Routing Symbol, the four digit ABA Institution Identifier, and the Check
	// Digit (bTTTTAAAAC).
	ImmediateDestination int

	// ImmediateOrigin contains the Routing Number of the ACH Operator or sending
	// point that is sending the file. The 10 character field begins with
	// a blank in the first position, followed by the four digit Federal Reserve
	// Routing Symbol, the four digit ABA Institution Identifier, and the Check
	// Digit (bTTTTAAAAC).
	ImmediateOrigin int

	// FileCreationDate is expressed in a "YYMMDD" format. The File Creation
	// Date is the date on which the file is prepared by an ODFI (ACH input files)
	// or the date (exchange date) on which a file is transmitted from ACH Operator
	// to ACH Operator, or from ACH Operator to RDFIs (ACH output files).
	FileCreationDate time.Time

	// FileCreationTime is expressed ina n "HHMM" (24 hour clock) format.
	// The system time when the ACH file was created
	FileCreationTime time.Time

	// This field should start at zero and increment by 1 (up to 9) and then go to
	// letters starting at A through Z for each subsequent file that is created for
	// a single system date. (34-34) 1 numeric 0-9 or uppercase alpha A-Z.
	// I have yet to see this ID not A
	FileIDModifier string

	// RecordSize indicates the number of characters contained in each
	// record. At this time, the value "094" must be used.
	recordSize string

	// BlockingFactor defines the number of physical records within a block
	// (a block is 940 characters). For all files moving between a DFI and an ACH
	// Operator (either way), the value "10" must be used. If the number of records
	// within the file is not a multiple of ten, the remainder of the block must
	// be nine-filled.
	blockingFactor string

	// FormatCode a code to allow for future format variations. As
	// currently defined, this field will contain a value of "1".
	formatCode string

	// ImmediateDestinationName us the name of the ACH or receiving point for which that
	// file is destined. Name corresponding to the ImmediateDestination
	ImmediateDestinationName string

	// ImmediateOriginName is the name of the ACH operator or sending point that is
	// sending the file. Name corresponding to the ImmediateOrigin
	ImmediateOriginName string

	// ReferenceCode is reserved for information pertinent to the Originator.
	ReferenceCode string
	// validator is composed for data validation
	validator
	// converters is composed for ACH to GoLang Converters
	converters
}

// NewFileHeader returns a new FileHeader with default values for none exported fields
func NewFileHeader() FileHeader {
	fh := FileHeader{
		recordType:     "1",
		priorityCode:   "01",
		FileIDModifier: "A",
		recordSize:     "094",
		blockingFactor: "10",
		formatCode:     "1",
	}
	return fh
}

// Parse takes the input record string and parses the FileHeader values
func (fh *FileHeader) Parse(record string) {
	// (character position 1-1) Always "1"
	fh.recordType = "1"
	// (2-3) Always "01"
	fh.priorityCode = "01"
	// (4-13) A blank space followed by your ODFI's routing number. For example: " 121140399"
	fh.ImmediateDestination = fh.parseNumField(record[3:13])
	// (14-23) A 10-digit number assigned to you by the ODFI once they approve you to originate ACH files through them
	fh.ImmediateOrigin = fh.parseNumField(record[13:23])
	// 24-29 Today's date in YYMMDD format
	// must be after todays date.
	fh.FileCreationDate = fh.parseSimpleDate(record[23:29])
	// 30-33 The current time in HHMM format
	fh.FileCreationTime = fh.parseSimpleTime(record[29:33])
	// 35-37 Always "A"
	fh.FileIDModifier = record[33:34]
	// 35-37 always "094"
	fh.recordSize = "094"
	//38-39 always "10"
	fh.blockingFactor = "10"
	//40 always "1"
	fh.formatCode = "1"
	//41-63 The name of the ODFI. example "SILICON VALLEY BANK    "
	fh.ImmediateDestinationName = strings.TrimSpace(record[40:63])
	//64-86 ACH operator or sending point that is sending the file
	fh.ImmediateOriginName = strings.TrimSpace(record[63:86])
	//97-94 Optional field that may be used to describe the ACH file for internal accounting purposes
	fh.ReferenceCode = strings.TrimSpace(record[86:94])
}

// String writes the FileHeader struct to a 94 character string.
func (fh *FileHeader) String() string {
	return fmt.Sprintf("%v%v%v%v%v%v%v%v%v%v%v%v%v",
		fh.recordType,
		fh.priorityCode,
		fh.ImmediateDestinationField(),
		fh.ImmediateOriginField(),
		fh.FileCreationDateField(),
		fh.FileCreationTimeField(),
		fh.FileIDModifier,
		fh.recordSize,
		fh.blockingFactor,
		fh.formatCode,
		fh.ImmediateDestinationNameField(),
		fh.ImmediateOriginNameField(),
		fh.ReferenceCodeField(),
	)

}

// Validate performs NACHA format rule checks on the record and returns an error if not Validated
// The first error encountered is returned and stops the parsing.
func (fh *FileHeader) Validate() error {

	if err := fh.fieldInclusion(); err != nil {
		return err
	}
	if fh.recordType != "1" {
		msg := fmt.Sprintf(msgRecordType, 1)
		return &FieldError{FieldName: "recordType", Value: fh.recordType, Msg: msg}
	}
	if err := fh.isUpperAlphanumeric(fh.FileIDModifier); err != nil {
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: err.Error()}
	}
	if len(fh.FileIDModifier) != 1 {
		msg := fmt.Sprintf(msgValidFieldLength, 1)
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: msg}
	}
	if fh.recordSize != "094" {
		return &FieldError{FieldName: "recordSize", Value: fh.recordSize, Msg: msgRecordSize}
	}
	if fh.blockingFactor != "10" {
		return &FieldError{FieldName: "blockingFactor", Value: fh.blockingFactor, Msg: msgBlockingFactor}
	}
	if fh.formatCode != "1" {
		return &FieldError{FieldName: "formatCode", Value: fh.formatCode, Msg: msgFormatCode}
	}
	if err := fh.isAlphanumeric(fh.ImmediateDestinationName); err != nil {
		return &FieldError{FieldName: "ImmediateDestinationName", Value: fh.ImmediateDestinationName, Msg: err.Error()}
	}
	if err := fh.isAlphanumeric(fh.ImmediateOriginName); err != nil {
		return &FieldError{FieldName: "ImmediateOriginName", Value: fh.ImmediateOriginName, Msg: err.Error()}
	}
	if err := fh.isAlphanumeric(fh.ReferenceCode); err != nil {
		return &FieldError{FieldName: "ReferenceCode", Value: fh.ReferenceCode, Msg: err.Error()}
	}

	// todo: handle test cases for before date
	/*
		if fh.fileCreationDate.Before(time.Now()) {
			return false, ErrFileCreationDate
		}
	*/
	return nil
}

// fieldInclusion validate mandatory fields are not default values. If fields are
// invalid the ACH transfer will be returned.
func (fh *FileHeader) fieldInclusion() error {
	if fh.recordType == "" {
		return &FieldError{FieldName: "recordType", Value: fh.recordType, Msg: msgFieldInclusion}
	}
	if fh.ImmediateDestination == 0 {
		return &FieldError{FieldName: "ImmediateDestination", Value: fh.ImmediateDestinationField(), Msg: msgFieldInclusion}
	}
	if fh.ImmediateOrigin == 0 {
		return &FieldError{FieldName: "ImmediateOrigin", Value: fh.ImmediateOriginField(), Msg: msgFieldInclusion}
	}
	if fh.FileCreationDate.IsZero() {
		return &FieldError{FieldName: "FileCreationDate", Value: fh.FileCreationDate.String(), Msg: msgFieldInclusion}
	}
	if fh.FileIDModifier == "" {
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: msgFieldInclusion}
	}
	if fh.recordSize == "" {
		return &FieldError{FieldName: "recordSize", Value: fh.recordSize, Msg: msgFieldInclusion}
	}
	if fh.blockingFactor == "" {
		return &FieldError{FieldName: "blockingFactor", Value: fh.blockingFactor, Msg: msgFieldInclusion}
	}
	if fh.formatCode == "" {
		return &FieldError{FieldName: "formatCode", Value: fh.formatCode, Msg: msgFieldInclusion}
	}
	return nil
}

// ImmediateDestinationField gets the immediate destination number with zero padding
func (fh *FileHeader) ImmediateDestinationField() string {
	return " " + fh.numericField(fh.ImmediateDestination, 9)
}

// ImmediateOriginField gets the immediate origin number with 0 padding
func (fh *FileHeader) ImmediateOriginField() string {
	return " " + fh.numericField(fh.ImmediateOrigin, 9)
}

// FileCreationDateField gets the file creation date in YYMMDD format
func (fh *FileHeader) FileCreationDateField() string {
	return fh.formatSimpleDate(fh.FileCreationDate)
}

// FileCreationTimeField gets the file creation time in HHMM format
func (fh *FileHeader) FileCreationTimeField() string {
	return fh.formatSimpleTime(fh.FileCreationTime)
}

// ImmediateDestinationNameField gets the ImmediateDestinationName field padded
func (fh *FileHeader) ImmediateDestinationNameField() string {
	return fh.alphaField(fh.ImmediateDestinationName, 23)
}

// ImmediateOriginNameField gets the ImmImmediateOriginName field padded
func (fh *FileHeader) ImmediateOriginNameField() string {
	return fh.alphaField(fh.ImmediateOriginName, 23)
}

// ReferenceCodeField gets the ReferenceCode field padded
func (fh *FileHeader) ReferenceCodeField() string {
	return fh.alphaField(fh.ReferenceCode, 8)
}
<commit_msg>var msgFileCreationDate is unused<commit_after>// Copyright 2016 The ACH Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package ach

import (
	"fmt"
	"strings"
	"time"
)

// Errors specific to a File Header Record
var (
	msgRecordType     = "received expecting %d"
	msgRecordSize     = "is not 094"
	msgBlockingFactor = "is not 10"
	msgFormatCode     = "is not 1"
)

// FileHeader is a Record designating physical file characteristics and identify
// the origin (sending point) and destination (receiving point) of the entries
// contained in the file. The file header also includes creation date and time
// fields which can be used to uniquely identify a file.
type FileHeader struct {
	// RecordType defines the type of record in the block. headerPos
	recordType string

	// PriorityCode consists of the numerals 01
	priorityCode string

	// ImmediateDestination contains the Routing Number of the ACH Operator or receiving
	// point to which the file is being sent. The 10 character field begins with
	// a blank in the first position, followed by the four digit Federal Reserve
	// Routing Symbol, the four digit ABA Institution Identifier, and the Check
	// Digit (bTTTTAAAAC).
	ImmediateDestination int

	// ImmediateOrigin contains the Routing Number of the ACH Operator or sending
	// point that is sending the file. The 10 character field begins with
	// a blank in the first position, followed by the four digit Federal Reserve
	// Routing Symbol, the four digit ABA Institution Identifier, and the Check
	// Digit (bTTTTAAAAC).
	ImmediateOrigin int

	// FileCreationDate is expressed in a "YYMMDD" format. The File Creation
	// Date is the date on which the file is prepared by an ODFI (ACH input files)
	// or the date (exchange date) on which a file is transmitted from ACH Operator
	// to ACH Operator, or from ACH Operator to RDFIs (ACH output files).
	FileCreationDate time.Time

	// FileCreationTime is expressed ina n "HHMM" (24 hour clock) format.
	// The system time when the ACH file was created
	FileCreationTime time.Time

	// This field should start at zero and increment by 1 (up to 9) and then go to
	// letters starting at A through Z for each subsequent file that is created for
	// a single system date. (34-34) 1 numeric 0-9 or uppercase alpha A-Z.
	// I have yet to see this ID not A
	FileIDModifier string

	// RecordSize indicates the number of characters contained in each
	// record. At this time, the value "094" must be used.
	recordSize string

	// BlockingFactor defines the number of physical records within a block
	// (a block is 940 characters). For all files moving between a DFI and an ACH
	// Operator (either way), the value "10" must be used. If the number of records
	// within the file is not a multiple of ten, the remainder of the block must
	// be nine-filled.
	blockingFactor string

	// FormatCode a code to allow for future format variations. As
	// currently defined, this field will contain a value of "1".
	formatCode string

	// ImmediateDestinationName us the name of the ACH or receiving point for which that
	// file is destined. Name corresponding to the ImmediateDestination
	ImmediateDestinationName string

	// ImmediateOriginName is the name of the ACH operator or sending point that is
	// sending the file. Name corresponding to the ImmediateOrigin
	ImmediateOriginName string

	// ReferenceCode is reserved for information pertinent to the Originator.
	ReferenceCode string
	// validator is composed for data validation
	validator
	// converters is composed for ACH to GoLang Converters
	converters
}

// NewFileHeader returns a new FileHeader with default values for none exported fields
func NewFileHeader() FileHeader {
	fh := FileHeader{
		recordType:     "1",
		priorityCode:   "01",
		FileIDModifier: "A",
		recordSize:     "094",
		blockingFactor: "10",
		formatCode:     "1",
	}
	return fh
}

// Parse takes the input record string and parses the FileHeader values
func (fh *FileHeader) Parse(record string) {
	// (character position 1-1) Always "1"
	fh.recordType = "1"
	// (2-3) Always "01"
	fh.priorityCode = "01"
	// (4-13) A blank space followed by your ODFI's routing number. For example: " 121140399"
	fh.ImmediateDestination = fh.parseNumField(record[3:13])
	// (14-23) A 10-digit number assigned to you by the ODFI once they approve you to originate ACH files through them
	fh.ImmediateOrigin = fh.parseNumField(record[13:23])
	// 24-29 Today's date in YYMMDD format
	// must be after todays date.
	fh.FileCreationDate = fh.parseSimpleDate(record[23:29])
	// 30-33 The current time in HHMM format
	fh.FileCreationTime = fh.parseSimpleTime(record[29:33])
	// 35-37 Always "A"
	fh.FileIDModifier = record[33:34]
	// 35-37 always "094"
	fh.recordSize = "094"
	//38-39 always "10"
	fh.blockingFactor = "10"
	//40 always "1"
	fh.formatCode = "1"
	//41-63 The name of the ODFI. example "SILICON VALLEY BANK    "
	fh.ImmediateDestinationName = strings.TrimSpace(record[40:63])
	//64-86 ACH operator or sending point that is sending the file
	fh.ImmediateOriginName = strings.TrimSpace(record[63:86])
	//97-94 Optional field that may be used to describe the ACH file for internal accounting purposes
	fh.ReferenceCode = strings.TrimSpace(record[86:94])
}

// String writes the FileHeader struct to a 94 character string.
func (fh *FileHeader) String() string {
	return fmt.Sprintf("%v%v%v%v%v%v%v%v%v%v%v%v%v",
		fh.recordType,
		fh.priorityCode,
		fh.ImmediateDestinationField(),
		fh.ImmediateOriginField(),
		fh.FileCreationDateField(),
		fh.FileCreationTimeField(),
		fh.FileIDModifier,
		fh.recordSize,
		fh.blockingFactor,
		fh.formatCode,
		fh.ImmediateDestinationNameField(),
		fh.ImmediateOriginNameField(),
		fh.ReferenceCodeField(),
	)

}

// Validate performs NACHA format rule checks on the record and returns an error if not Validated
// The first error encountered is returned and stops the parsing.
func (fh *FileHeader) Validate() error {

	if err := fh.fieldInclusion(); err != nil {
		return err
	}
	if fh.recordType != "1" {
		msg := fmt.Sprintf(msgRecordType, 1)
		return &FieldError{FieldName: "recordType", Value: fh.recordType, Msg: msg}
	}
	if err := fh.isUpperAlphanumeric(fh.FileIDModifier); err != nil {
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: err.Error()}
	}
	if len(fh.FileIDModifier) != 1 {
		msg := fmt.Sprintf(msgValidFieldLength, 1)
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: msg}
	}
	if fh.recordSize != "094" {
		return &FieldError{FieldName: "recordSize", Value: fh.recordSize, Msg: msgRecordSize}
	}
	if fh.blockingFactor != "10" {
		return &FieldError{FieldName: "blockingFactor", Value: fh.blockingFactor, Msg: msgBlockingFactor}
	}
	if fh.formatCode != "1" {
		return &FieldError{FieldName: "formatCode", Value: fh.formatCode, Msg: msgFormatCode}
	}
	if err := fh.isAlphanumeric(fh.ImmediateDestinationName); err != nil {
		return &FieldError{FieldName: "ImmediateDestinationName", Value: fh.ImmediateDestinationName, Msg: err.Error()}
	}
	if err := fh.isAlphanumeric(fh.ImmediateOriginName); err != nil {
		return &FieldError{FieldName: "ImmediateOriginName", Value: fh.ImmediateOriginName, Msg: err.Error()}
	}
	if err := fh.isAlphanumeric(fh.ReferenceCode); err != nil {
		return &FieldError{FieldName: "ReferenceCode", Value: fh.ReferenceCode, Msg: err.Error()}
	}

	// todo: handle test cases for before date
	/*
		if fh.fileCreationDate.Before(time.Now()) {
			return false, ErrFileCreationDate
		}
	*/
	return nil
}

// fieldInclusion validate mandatory fields are not default values. If fields are
// invalid the ACH transfer will be returned.
func (fh *FileHeader) fieldInclusion() error {
	if fh.recordType == "" {
		return &FieldError{FieldName: "recordType", Value: fh.recordType, Msg: msgFieldInclusion}
	}
	if fh.ImmediateDestination == 0 {
		return &FieldError{FieldName: "ImmediateDestination", Value: fh.ImmediateDestinationField(), Msg: msgFieldInclusion}
	}
	if fh.ImmediateOrigin == 0 {
		return &FieldError{FieldName: "ImmediateOrigin", Value: fh.ImmediateOriginField(), Msg: msgFieldInclusion}
	}
	if fh.FileCreationDate.IsZero() {
		return &FieldError{FieldName: "FileCreationDate", Value: fh.FileCreationDate.String(), Msg: msgFieldInclusion}
	}
	if fh.FileIDModifier == "" {
		return &FieldError{FieldName: "FileIDModifier", Value: fh.FileIDModifier, Msg: msgFieldInclusion}
	}
	if fh.recordSize == "" {
		return &FieldError{FieldName: "recordSize", Value: fh.recordSize, Msg: msgFieldInclusion}
	}
	if fh.blockingFactor == "" {
		return &FieldError{FieldName: "blockingFactor", Value: fh.blockingFactor, Msg: msgFieldInclusion}
	}
	if fh.formatCode == "" {
		return &FieldError{FieldName: "formatCode", Value: fh.formatCode, Msg: msgFieldInclusion}
	}
	return nil
}

// ImmediateDestinationField gets the immediate destination number with zero padding
func (fh *FileHeader) ImmediateDestinationField() string {
	return " " + fh.numericField(fh.ImmediateDestination, 9)
}

// ImmediateOriginField gets the immediate origin number with 0 padding
func (fh *FileHeader) ImmediateOriginField() string {
	return " " + fh.numericField(fh.ImmediateOrigin, 9)
}

// FileCreationDateField gets the file creation date in YYMMDD format
func (fh *FileHeader) FileCreationDateField() string {
	return fh.formatSimpleDate(fh.FileCreationDate)
}

// FileCreationTimeField gets the file creation time in HHMM format
func (fh *FileHeader) FileCreationTimeField() string {
	return fh.formatSimpleTime(fh.FileCreationTime)
}

// ImmediateDestinationNameField gets the ImmediateDestinationName field padded
func (fh *FileHeader) ImmediateDestinationNameField() string {
	return fh.alphaField(fh.ImmediateDestinationName, 23)
}

// ImmediateOriginNameField gets the ImmImmediateOriginName field padded
func (fh *FileHeader) ImmediateOriginNameField() string {
	return fh.alphaField(fh.ImmediateOriginName, 23)
}

// ReferenceCodeField gets the ReferenceCode field padded
func (fh *FileHeader) ReferenceCodeField() string {
	return fh.alphaField(fh.ReferenceCode, 8)
}
<|endoftext|>


<commit_before>package charmap

import "testing"

func TestLanguageOf(t *testing.T) {

	inArray := []rune{'ಅ', 'ḥ', 'ɭ', '\u0d0a'}
	outArray := []string{"kn_IN", "ISO15919", "IPA", "ml_IN"}

	for index, value := range inArray {
		if x, output := LanguageOf(value), outArray[index]; x != output {
			t.Errorf("LanguageOf(%v) = %s we need %s", value, x, output)
		}
	}

}

func TestCharCompare(t *testing.T) {
	inArray := []rune{'ँ', 'అ'}
	outArray := []rune{'ಁ', 'അ'}

	// multiByteIn := []string {`aː`, `ಆ` }
	// multiByteOut := []string {`ആ`, `ā`}

	for index, value := range inArray {
		if x := CharCompare(value, outArray[index]); !x {
			t.Errorf("CharCompare(%v, %v) = %v we need %v", value, outArray[index], x, true)
		}
	}

	// for index, value := range multiByteIn {
	// 	if x := CharCompare(value, multiByteOut[index]); !x {
	// 		t.Errorf("CharCompare(%v, %v) = %v we need %v", value, multiByteOut[index], x, true)
	// 	}
	// }
}
<commit_msg>Updated test cases.<commit_after>package charmap

import "testing"

func TestLanguageOf(t *testing.T) {

	inArray := []string{`ಅ`, `ḥ`, `uː`, `ഊ`}
	outArray := []string{"kn_IN", "ISO15919", "IPA", "ml_IN"}

	for index, value := range inArray {
		if x, output := LanguageOf(value), outArray[index]; x != output {
			t.Errorf("LanguageOf(%v) = %s we need %s", value, x, output)
		}
	}

}

func TestCharCompare(t *testing.T) {
	inArray := []string{`ँ`, `అ`, `aː`, `ಆ`}
	outArray := []string{`ಁ`, `അ`, `ആ`, `ā`}

	for index, value := range inArray {
		if x := CharCompare(value, outArray[index]); !x {
			t.Errorf("CharCompare(%v, %v) = %v we need %v", value, outArray[index], x, true)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"embed"
	"fmt"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
	"io"
	"io/fs"
	"math/rand"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/chrislusf/seaweedfs/weed/command"
	"github.com/chrislusf/seaweedfs/weed/glog"
)

var IsDebug *bool

var commands = command.Commands

var exitStatus = 0
var exitMu sync.Mutex

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

//go:embed static
var static embed.FS

func init() {
	weed_server.StaticFS, _ = fs.Sub(static, "static")

	flag.Var(&util.ConfigurationFileDirectory, "conf_dir", "directory with toml configuration files")
}

func main() {
	glog.MaxSize = 1024 * 1024 * 32
	rand.Seed(time.Now().UnixNano())
	flag.Usage = usage

	if command.AutocompleteMain(commands) {
		return
	}

	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		for _, cmd := range commands {
			if len(args) >= 2 && cmd.Name() == args[1] && cmd.Run != nil {
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
		}
		return
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			if !cmd.Run(cmd, args) {
				fmt.Fprintf(os.Stderr, "\n")
				cmd.Flag.Usage()
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
			exit()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "weed: unknown subcommand %q\nRun 'weed help' for usage.\n", args[0])
	setExitStatus(2)
	exit()
}

var usageTemplate = `
SeaweedFS: store billions of files and serve them fast!

Usage:

	weed command [arguments]

The commands are:
{{range .}}{{if .Runnable}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}

Use "weed help [command]" for more information about a command.

`

var helpTemplate = `{{if .Runnable}}Usage: weed {{.UsageLine}}
{{end}}
  {{.Long}}
`

// tmpl executes the given template text on data, writing the result to w.
func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize})
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

func capitalize(s string) string {
	if s == "" {
		return s
	}
	r, n := utf8.DecodeRuneInString(s)
	return string(unicode.ToTitle(r)) + s[n:]
}

func printUsage(w io.Writer) {
	tmpl(w, usageTemplate, commands)
}

func usage() {
	printUsage(os.Stderr)
	fmt.Fprintf(os.Stderr, "For Logging, use \"weed [logging_options] [command]\". The logging options are:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

// help implements the 'help' command.
func help(args []string) {
	if len(args) == 0 {
		printUsage(os.Stdout)
		// not exit 2: succeeded at 'weed help'.
		return
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: weed help command\n\nToo many arguments given.\n")
		os.Exit(2) // failed at 'weed help'
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			tmpl(os.Stdout, helpTemplate, cmd)
			// not exit 2: succeeded at 'weed help cmd'.
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'weed help'.\n", arg)
	os.Exit(2) // failed at 'weed help cmd'
}

var atexitFuncs []func()

func atexit(f func()) {
	atexitFuncs = append(atexitFuncs, f)
}

func exit() {
	for _, f := range atexitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}
<commit_msg>change to config_dir, to be consistent with "weed scaffold -config"<commit_after>package main

import (
	"embed"
	"fmt"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
	"io"
	"io/fs"
	"math/rand"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/chrislusf/seaweedfs/weed/command"
	"github.com/chrislusf/seaweedfs/weed/glog"
)

var IsDebug *bool

var commands = command.Commands

var exitStatus = 0
var exitMu sync.Mutex

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

//go:embed static
var static embed.FS

func init() {
	weed_server.StaticFS, _ = fs.Sub(static, "static")

	flag.Var(&util.ConfigurationFileDirectory, "config_dir", "directory with toml configuration files")
}

func main() {
	glog.MaxSize = 1024 * 1024 * 32
	rand.Seed(time.Now().UnixNano())
	flag.Usage = usage

	if command.AutocompleteMain(commands) {
		return
	}

	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		for _, cmd := range commands {
			if len(args) >= 2 && cmd.Name() == args[1] && cmd.Run != nil {
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
		}
		return
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			if !cmd.Run(cmd, args) {
				fmt.Fprintf(os.Stderr, "\n")
				cmd.Flag.Usage()
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
			exit()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "weed: unknown subcommand %q\nRun 'weed help' for usage.\n", args[0])
	setExitStatus(2)
	exit()
}

var usageTemplate = `
SeaweedFS: store billions of files and serve them fast!

Usage:

	weed command [arguments]

The commands are:
{{range .}}{{if .Runnable}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}

Use "weed help [command]" for more information about a command.

`

var helpTemplate = `{{if .Runnable}}Usage: weed {{.UsageLine}}
{{end}}
  {{.Long}}
`

// tmpl executes the given template text on data, writing the result to w.
func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize})
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

func capitalize(s string) string {
	if s == "" {
		return s
	}
	r, n := utf8.DecodeRuneInString(s)
	return string(unicode.ToTitle(r)) + s[n:]
}

func printUsage(w io.Writer) {
	tmpl(w, usageTemplate, commands)
}

func usage() {
	printUsage(os.Stderr)
	fmt.Fprintf(os.Stderr, "For Logging, use \"weed [logging_options] [command]\". The logging options are:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

// help implements the 'help' command.
func help(args []string) {
	if len(args) == 0 {
		printUsage(os.Stdout)
		// not exit 2: succeeded at 'weed help'.
		return
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: weed help command\n\nToo many arguments given.\n")
		os.Exit(2) // failed at 'weed help'
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			tmpl(os.Stdout, helpTemplate, cmd)
			// not exit 2: succeeded at 'weed help cmd'.
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'weed help'.\n", arg)
	os.Exit(2) // failed at 'weed help cmd'
}

var atexitFuncs []func()

func atexit(f func()) {
	atexitFuncs = append(atexitFuncs, f)
}

func exit() {
	for _, f := range atexitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}
<|endoftext|>


<commit_before>package files

import (
	"blacksync/lib/blacksync/storage"
	"blacksync/lib/blacksync/auth"
	"blacksync/lib/blacksync/config"
	"blacksync/lib/blacksync/storage/storageproxy"
	"encoding/json"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

func Stat() http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		log.WithFields(log.Fields{
			"frontend": "api",
			"app":      "files",
			"action":   "stat",
		}).Debug("files app stat resource")

		cfg, err := config.Get()
		if err != nil {
			log.Error(err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}

		authRes := auth.GetAuthResourceFromRequest(r)

		storageProxy, err := storageproxy.NewStorageProxy(cfg.Storage, authRes)
		if err != nil {
			log.Error(err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}

		var children bool
		queryChildren := r.URL.Query().Get("children")
		if queryChildren != "" {
			children, err = strconv.ParseBool(queryChildren)
			if err != nil {
				children = false
			}
		}

		if err != nil {
			log.Warn(err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		resource := filepath.Join("/", strings.TrimPrefix(r.URL.Path, "/api/v1/core/apps/files/stat"))

		meta, err := storageProxy.Stat(resource, children)
		if err != nil {
			if storage.IsNotExistError(err){
				log.Error(err)
				http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
				return
			}
			log.Error(err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}

		metaJSON, err := json.Marshal(meta)
		if err != nil {
			log.Error(err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Write(metaJSON)
		return
	}
	return http.HandlerFunc(fn)
}
<commit_msg>API Files stat<commit_after>package files

import (
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	"github.com/sanato/sanato-lib/storage"
	"net/http"
	"strconv"
)

func (api *API) stat(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	_, err := api.tokenAuth(r, true)
	if err != nil {
		logrus.Error(err)
		http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
		return
	}
	resource := p.ByName("path")
	if resource == "" {
		resource = "/"
	}
	var children bool
	queryChildren := r.URL.Query().Get("children")
	if queryChildren != "" {
		children, err = strconv.ParseBool(queryChildren)
		if err != nil {
			children = false
		}
	}
	if err != nil {
		logrus.Warn(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	meta, err := api.storageProvider.Stat(resource, children)
	if err != nil {
		if storage.IsNotExistError(err) {
			logrus.Error(err)
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		logrus.Error(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	metaJSON, err := json.Marshal(meta)
	if err != nil {
		logrus.Error(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	w.Write(metaJSON)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
)

// TODO: make this customizable
var ignoredDirs = []string{
	".git",
	".cvs",
	".svn",
}

func filterDirs(fileInfo os.FileInfo) bool {
	name := fileInfo.Name()
	for _, dir := range ignoredDirs {
		if strings.HasPrefix(name, dir) {
			return false
		}
	}

	return true
}

func tlcStats(container *tlc.Container) string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}

func diff(target string, source string, recipe string, brotliQuality int) {
	startTime := time.Now()

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if target == "/dev/null" {
		targetContainer = &tlc.Container{}
	} else {
		targetInfo, err := os.Lstat(target)
		must(err)

		if targetInfo.IsDir() {
			comm.Opf("Hashing %s", target)
			targetContainer, err = tlc.Walk(target, filterDirs)
			must(err)

			comm.StartProgress()
			targetSignature, err = pwr.ComputeSignature(targetContainer, target, comm.NewStateConsumer())
			comm.EndProgress()
			must(err)

			{
				prettySize := humanize.Bytes(uint64(targetContainer.Size))
				perSecond := humanize.Bytes(uint64(float64(targetContainer.Size) / time.Since(startTime).Seconds()))
				comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(targetContainer), perSecond)
			}
		} else {
			comm.Opf("Reading signature from %s", target)
			signatureReader, err := os.Open(target)
			must(err)
			targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
			must(err)
			must(signatureReader.Close())
		}

	}

	startTime = time.Now()

	sourceContainer, err := tlc.Walk(source, filterDirs)
	must(err)

	recipeWriter, err := os.Create(recipe)
	must(err)
	defer recipeWriter.Close()

	signaturePath := recipe + ".sig"
	signatureWriter, err := os.Create(signaturePath)
	must(err)
	defer signatureWriter.Close()

	recipeCounter := counter.NewWriter(recipeWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	dctx := &pwr.DiffContext{
		SourceContainer: sourceContainer,
		SourcePath:      source,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: comm.NewStateConsumer(),
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(*diffArgs.quality),
		},
	}

	comm.Opf("Diffing %s", source)
	comm.StartProgress()
	must(dctx.WriteRecipe(recipeCounter, signatureCounter))
	comm.EndProgress()

	{
		prettySize := humanize.Bytes(uint64(sourceContainer.Size))
		perSecond := humanize.Bytes(uint64(float64(sourceContainer.Size) / time.Since(startTime).Seconds()))
		comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(sourceContainer), perSecond)
	}

	if *diffArgs.verify {
		tmpDir, err := ioutil.TempDir(os.TempDir(), "pwr")
		must(err)
		defer os.RemoveAll(tmpDir)

		apply(recipe, target, tmpDir)

		verify(signaturePath, tmpDir)
	}

	{
		prettyRecipeSize := humanize.Bytes(uint64(recipeCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(recipeCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

		comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		comm.Statf("%s recipe (%.2f%% of the full size)", prettyRecipeSize, relToNew)
	}
}

func apply(recipe string, target string, output string) {
	comm.Opf("Patching %s", output)
	startTime := time.Now()

	recipeReader, err := os.Open(recipe)
	must(err)

	actx := &pwr.ApplyContext{
		TargetPath: target,
		OutputPath: output,

		Consumer: comm.NewStateConsumer(),
	}

	comm.StartProgress()
	must(actx.ApplyRecipe(recipeReader))
	comm.EndProgress()

	container := actx.SourceContainer
	prettySize := humanize.Bytes(uint64(container.Size))
	perSecond := humanize.Bytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s\n", prettySize, tlcStats(container), perSecond)
}

func sign(output string, signature string) {
	comm.Opf("Creating signature for %s", output)
	startTime := time.Now()

	container, err := tlc.Walk(output, nil)
	must(err)

	signatureWriter, err := os.Create(signature)
	must(err)

	compression := pwr.CompressionDefault()

	rawSigWire := wire.NewWriteContext(signatureWriter)
	rawSigWire.WriteMagic(pwr.SignatureMagic)

	rawSigWire.WriteMessage(&pwr.SignatureHeader{
		Compression: compression,
	})

	sigWire, err := pwr.CompressWire(rawSigWire, compression)
	must(err)
	sigWire.WriteMessage(container)

	comm.StartProgress()
	err = pwr.ComputeSignatureToWriter(container, output, comm.NewStateConsumer(), func(hash sync.BlockHash) error {
		return sigWire.WriteMessage(&pwr.BlockHash{
			WeakHash:   hash.WeakHash,
			StrongHash: hash.StrongHash,
		})
	})
	comm.EndProgress()
	must(err)

	must(sigWire.Close())

	prettySize := humanize.Bytes(uint64(container.Size))
	perSecond := humanize.Bytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(container), perSecond)
}

func verify(signature string, output string) {
	comm.Opf("Verifying %s", output)
	startTime := time.Now()

	signatureReader, err := os.Open(signature)
	must(err)
	defer signatureReader.Close()

	refContainer, refHashes, err := pwr.ReadSignature(signatureReader)
	must(err)

	comm.StartProgress()
	hashes, err := pwr.ComputeSignature(refContainer, output, comm.NewStateConsumer())
	comm.EndProgress()
	must(err)

	success := true

	if len(hashes) != len(refHashes) {
		must(fmt.Errorf("Expected %d blocks, got %d.", len(refHashes), len(hashes)))
	}

	for i, refHash := range refHashes {
		hash := hashes[i]

		if refHash.WeakHash != hash.WeakHash {
			success = false
			comm.Logf("At block %d, expected weak hash %x, got %x", i, refHash.WeakHash, hash.WeakHash)
		}

		if !bytes.Equal(refHash.StrongHash, hash.StrongHash) {
			success = false
			comm.Logf("At block %d, expected strong hash %x, got %x", i, refHash.StrongHash, hash.StrongHash)
		}
	}

	if !success {
		comm.Dief("Some checks failed after checking %d block.", len(refHashes))
	}

	prettySize := humanize.Bytes(uint64(refContainer.Size))
	perSecond := humanize.Bytes(uint64(float64(refContainer.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(refContainer), perSecond)
}
<commit_msg>Better verify output<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
)

// TODO: make this customizable
var ignoredDirs = []string{
	".git",
	".cvs",
	".svn",
}

func filterDirs(fileInfo os.FileInfo) bool {
	name := fileInfo.Name()
	for _, dir := range ignoredDirs {
		if strings.HasPrefix(name, dir) {
			return false
		}
	}

	return true
}

func tlcStats(container *tlc.Container) string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}

func diff(target string, source string, recipe string, brotliQuality int) {
	startTime := time.Now()

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if target == "/dev/null" {
		targetContainer = &tlc.Container{}
	} else {
		targetInfo, err := os.Lstat(target)
		must(err)

		if targetInfo.IsDir() {
			comm.Opf("Hashing %s", target)
			targetContainer, err = tlc.Walk(target, filterDirs)
			must(err)

			comm.StartProgress()
			targetSignature, err = pwr.ComputeSignature(targetContainer, target, comm.NewStateConsumer())
			comm.EndProgress()
			must(err)

			{
				prettySize := humanize.Bytes(uint64(targetContainer.Size))
				perSecond := humanize.Bytes(uint64(float64(targetContainer.Size) / time.Since(startTime).Seconds()))
				comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(targetContainer), perSecond)
			}
		} else {
			comm.Opf("Reading signature from %s", target)
			signatureReader, err := os.Open(target)
			must(err)
			targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
			must(err)
			must(signatureReader.Close())
		}

	}

	startTime = time.Now()

	sourceContainer, err := tlc.Walk(source, filterDirs)
	must(err)

	recipeWriter, err := os.Create(recipe)
	must(err)
	defer recipeWriter.Close()

	signaturePath := recipe + ".sig"
	signatureWriter, err := os.Create(signaturePath)
	must(err)
	defer signatureWriter.Close()

	recipeCounter := counter.NewWriter(recipeWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	dctx := &pwr.DiffContext{
		SourceContainer: sourceContainer,
		SourcePath:      source,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: comm.NewStateConsumer(),
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(*diffArgs.quality),
		},
	}

	comm.Opf("Diffing %s", source)
	comm.StartProgress()
	must(dctx.WriteRecipe(recipeCounter, signatureCounter))
	comm.EndProgress()

	{
		prettySize := humanize.Bytes(uint64(sourceContainer.Size))
		perSecond := humanize.Bytes(uint64(float64(sourceContainer.Size) / time.Since(startTime).Seconds()))
		comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(sourceContainer), perSecond)
	}

	if *diffArgs.verify {
		tmpDir, err := ioutil.TempDir(os.TempDir(), "pwr")
		must(err)
		defer os.RemoveAll(tmpDir)

		apply(recipe, target, tmpDir)

		verify(signaturePath, tmpDir)
	}

	{
		prettyRecipeSize := humanize.Bytes(uint64(recipeCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(recipeCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

		comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		comm.Statf("%s recipe (%.2f%% of the full size)", prettyRecipeSize, relToNew)
	}
}

func apply(recipe string, target string, output string) {
	comm.Opf("Patching %s", output)
	startTime := time.Now()

	recipeReader, err := os.Open(recipe)
	must(err)

	actx := &pwr.ApplyContext{
		TargetPath: target,
		OutputPath: output,

		Consumer: comm.NewStateConsumer(),
	}

	comm.StartProgress()
	must(actx.ApplyRecipe(recipeReader))
	comm.EndProgress()

	container := actx.SourceContainer
	prettySize := humanize.Bytes(uint64(container.Size))
	perSecond := humanize.Bytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s\n", prettySize, tlcStats(container), perSecond)
}

func sign(output string, signature string) {
	comm.Opf("Creating signature for %s", output)
	startTime := time.Now()

	container, err := tlc.Walk(output, nil)
	must(err)

	signatureWriter, err := os.Create(signature)
	must(err)

	compression := pwr.CompressionDefault()

	rawSigWire := wire.NewWriteContext(signatureWriter)
	rawSigWire.WriteMagic(pwr.SignatureMagic)

	rawSigWire.WriteMessage(&pwr.SignatureHeader{
		Compression: compression,
	})

	sigWire, err := pwr.CompressWire(rawSigWire, compression)
	must(err)
	sigWire.WriteMessage(container)

	comm.StartProgress()
	err = pwr.ComputeSignatureToWriter(container, output, comm.NewStateConsumer(), func(hash sync.BlockHash) error {
		return sigWire.WriteMessage(&pwr.BlockHash{
			WeakHash:   hash.WeakHash,
			StrongHash: hash.StrongHash,
		})
	})
	comm.EndProgress()
	must(err)

	must(sigWire.Close())

	prettySize := humanize.Bytes(uint64(container.Size))
	perSecond := humanize.Bytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(container), perSecond)
}

func verify(signature string, output string) {
	comm.Opf("Verifying %s", output)
	startTime := time.Now()

	signatureReader, err := os.Open(signature)
	must(err)
	defer signatureReader.Close()

	refContainer, refHashes, err := pwr.ReadSignature(signatureReader)
	must(err)

	comm.StartProgress()
	hashes, err := pwr.ComputeSignature(refContainer, output, comm.NewStateConsumer())
	comm.EndProgress()
	must(err)

	success := true

	if len(hashes) != len(refHashes) {
		must(fmt.Errorf("Expected %d blocks, got %d.", len(refHashes), len(hashes)))
	}

	for i, refHash := range refHashes {
		hash := hashes[i]

		if refHash.WeakHash != hash.WeakHash {
			success = false
			comm.Logf("At block %d / %d, expected weak hash %x, got %x", i, len(refHashes), refHash.WeakHash, hash.WeakHash)
		}

		if !bytes.Equal(refHash.StrongHash, hash.StrongHash) {
			success = false
			comm.Logf("At block %d / %d, expected strong hash %x, got %x", i, len(refHashes), refHash.WeakHash, hash.WeakHash)
		}
	}

	if !success {
		comm.Dief("Some checks failed after checking %d block.", len(refHashes))
	}

	prettySize := humanize.Bytes(uint64(refContainer.Size))
	perSecond := humanize.Bytes(uint64(float64(refContainer.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, tlcStats(refContainer), perSecond)
}
<|endoftext|>


<commit_before>package mantle

import (
	"github.com/youtube/vitess/go/pools"
	"time"
)

type PoolSettings struct {
	HostAndPorts []string
	Capacity     int
	MaxCapacity  int
	Timeout      time.Duration
}

func (rp *ResourcePool) GetConn() (pools.Resource, error) {
	resource, err := rp.pool.Get()
	if err != nil {
		return nil, err
	}
	return resource, nil
}

func (rp *ResourcePool) PutConn(conn pools.Resource) {
	rp.pool.Put(conn)
}

type ResourcePool struct {
	pool *pools.ResourcePool
}

//The connect callback that is passed by a backend
type dialAndConnect func(hostNPorts []string) (pools.Resource, error)

func NewPool(connect dialAndConnect, settings PoolSettings) *ResourcePool {
	return &ResourcePool{
		pools.NewResourcePool(newRedisFactory(connect, settings.HostAndPorts),
			settings.Capacity, settings.MaxCapacity, settings.Timeout)}
}

func newRedisFactory(connect dialAndConnect, HostAndPorts []string) pools.Factory {
	return func() (pools.Resource, error) {
		return connect(HostAndPorts)
	}
}
<commit_msg>lets pass options to settings<commit_after>package mantle

import (
	"github.com/youtube/vitess/go/pools"
	"time"
)

type PoolSettings struct {
	HostAndPorts []string
	Capacity     int
	MaxCapacity  int
	Options      map[string]string
	Timeout      time.Duration
}

func (rp *ResourcePool) GetConn() (pools.Resource, error) {
	resource, err := rp.pool.Get()
	if err != nil {
		return nil, err
	}
	return resource, nil
}

func (rp *ResourcePool) PutConn(conn pools.Resource) {
	rp.pool.Put(conn)
}

type ResourcePool struct {
	pool *pools.ResourcePool
}

//The connect callback that is passed by a backend
type dialAndConnect func(instance interface{}) (pools.Resource, error)

func NewPool(connect dialAndConnect, instance interface{}, settings PoolSettings) *ResourcePool {
	return &ResourcePool{
		pools.NewResourcePool(newRedisFactory(connect, instance),
			settings.Capacity, settings.MaxCapacity, settings.Timeout)}
}

func newRedisFactory(connect dialAndConnect, instance interface{}) pools.Factory {
	return func() (pools.Resource, error) {
		return connect(instance)
	}
}
<|endoftext|>


<commit_before>// +build new

package gototo

import (
	"encoding/json"
	zmq "github.com/JeremyOT/gozmq"
	"log"
	"runtime"
)

var logger *log.Logger

func SetLogger(l *log.Logger) {
	logger = l
}

func writeLog(message ...interface{}) {
	if logger != nil {
		logger.Println(message)
	} else {
		println(message)
	}
}

var registeredWorkerFunctions map[string]WorkerFunction = make(map[string]WorkerFunction)
var activeTimeout = 1
var passiveTimeout = 100

type WorkerFunction func(interface{}) interface{}

func RunRouter(routerAddress, dealerAddress string, routerBind, dealerBind bool) error {
	context, _ := zmq.NewContext()
	defer context.Close()
	router, _ := context.NewSocket(zmq.ROUTER)
	defer router.Close()
	if routerBind {
		router.Bind(routerAddress)
	} else {
		router.Connect(routerAddress)
	}
	dealer, _ := context.NewSocket(zmq.DEALER)
	defer dealer.Close()
	if dealerBind {
		dealer.Bind(dealerAddress)
	} else {
		dealer.Connect(dealerAddress)
	}
	return zmq.Device(zmq.QUEUE, router, dealer)
}

func RegisterWorkerFunction(name string, workerFunction WorkerFunction) {
	registeredWorkerFunctions[name] = workerFunction
}

func callworker(responseChannel chan [][]byte, message [][]byte, parameters interface{}, workerFunction WorkerFunction) {
	response := workerFunction(parameters)
	responseData, _ := json.Marshal(response)
	message[len(message)-1] = responseData
	responseChannel <- message
}

func RunWorker(address string, numWorkers int, quit chan int, wait chan int) {
	defer func() { close(wait) }()
	context, _ := zmq.NewContext()
	defer context.Close()
	socket, _ := context.NewSocket(zmq.ROUTER)
	defer socket.Close()
	socket.SetSockOptInt(zmq.RCVTIMEO, passiveTimeout)
	socket.Bind(address)
	runningWorkers := 0
	responseChannel := make(chan [][]byte)
	sendResponse := func(response [][]byte) {
		runningWorkers -= 1
		socket.SendMultipart(response, 0)
		if runningWorkers == 0 {
			socket.SetSockOptInt(zmq.RCVTIMEO, passiveTimeout)
		}
	}
	for {
		if runningWorkers == numWorkers {
			select {
			case response := <-responseChannel:
				sendResponse(response)
			case <-quit:
				return
			}
		}
		select {
		case <-quit:
			return
		case response := <-responseChannel:
			sendResponse(response)
			break
		default:
			message, err := socket.RecvMultipart(0)
			if err != nil {
				// Yield to other goroutines. The 1.2 pre-emptive scheduler doesn't
				// always work here. Older versions require it.
				runtime.Gosched()
				break
			}
			data := map[string]interface{}{}
			json.Unmarshal(message[len(message)-1], &data)
			if data == nil {
				writeLog("Received invalid message")
				break
			}
			workerFunction := registeredWorkerFunctions[data["method"].(string)]
			if workerFunction == nil {
				writeLog("Unregistered worker function:", data["method"].(string))
				break
			}
			if runningWorkers == 0 {
				socket.SetSockOptInt(zmq.RCVTIMEO, activeTimeout)
			}
			runningWorkers += 1
			go callworker(responseChannel, message, data["parameters"], workerFunction)
		}
	}
}

func RunWorkerServer(routerAddress string, routerBind bool, count int) {
	if count <= 0 {
		count = runtime.NumCPU()
	}
	quit := make(chan int)
	wait := make(chan int)
	RunWorker(routerAddress, count, quit, wait)
	<-wait
}
<commit_msg>refactored worker to struct<commit_after>// +build new

package gototo

import (
	"encoding/json"
	zmq "github.com/JeremyOT/gozmq"
	"log"
	"runtime"
)

// Run a ZMQ router->dealer at the specified addresses.
func RunRouter(routerAddress, dealerAddress string, routerBind, dealerBind bool) error {
	context, _ := zmq.NewContext()
	defer context.Close()
	router, _ := context.NewSocket(zmq.ROUTER)
	defer router.Close()
	if routerBind {
		router.Bind(routerAddress)
	} else {
		router.Connect(routerAddress)
	}
	dealer, _ := context.NewSocket(zmq.DEALER)
	defer dealer.Close()
	if dealerBind {
		dealer.Bind(dealerAddress)
	} else {
		dealer.Connect(dealerAddress)
	}
	return zmq.Device(zmq.QUEUE, router, dealer)
}

type WorkerFunction func(interface{}) interface{}

type Worker struct {
	logger *log.Logger
	registeredWorkerFunctions map[string]WorkerFunction
	activeTimeout int
	passiveTimeout int
	quit chan int
	wait chan int
	address string
	maxWorkers int
	runningWorkers int
}

// Create a new worker bound to address that will run at most count functions at a time.
// If count == 0, it will default to runtime.NumCPU().
func New(address string, count int) *Worker {
	if count == 0 {
		count = runtime.NumCPU()
	}
	return &Worker{registeredWorkerFunctions: make(map[string]WorkerFunction),
		activeTimeout: 1,
		passiveTimeout: 100,
		quit: make(chan int),
		wait: make(chan int),
		maxWorkers: count,
		address: address,
	}
}

func (w *Worker) Quit() {
	w.quit <- 1
}

func (w *Worker) Wait() {
	<-w.wait
}

func (w *Worker) SetLogger(l *log.Logger) {
	w.logger = l
}

func (w *Worker) writeLog(message ...interface{}) {
	if w.logger != nil {
		w.logger.Println(message)
	} else {
		log.Println(message)
	}
}

func (w *Worker) RegisterWorkerFunction(name string, workerFunction WorkerFunction) {
	w.registeredWorkerFunctions[name] = workerFunction
}

// Run a worker function and send the response to responseChannel
func runFunction(responseChannel chan [][]byte, message [][]byte, parameters interface{}, workerFunction WorkerFunction) {
	response := workerFunction(parameters)
	responseData, _ := json.Marshal(response)
	message[len(message)-1] = responseData
	responseChannel <- message
}

func (w *Worker) Run() {
	defer func() { close(w.wait) }()
	context, _ := zmq.NewContext()
	defer context.Close()
	socket, _ := context.NewSocket(zmq.ROUTER)
	defer socket.Close()
	socket.SetSockOptInt(zmq.RCVTIMEO, w.passiveTimeout)
	socket.Bind(w.address)
	w.runningWorkers = 0
	responseChannel := make(chan [][]byte)
	sendResponse := func(response [][]byte) {
		w.runningWorkers -= 1
		socket.SendMultipart(response, 0)
		if w.runningWorkers == 0 {
			socket.SetSockOptInt(zmq.RCVTIMEO, w.passiveTimeout)
		}
	}
	for {
		if w.runningWorkers == w.maxWorkers {
			select {
			case response := <-responseChannel:
				sendResponse(response)
			case <-w.quit:
				return
			}
		}
		select {
		case <-w.quit:
			return
		case response := <-responseChannel:
			sendResponse(response)
			break
		default:
			message, err := socket.RecvMultipart(0)
			if err != nil {
				break
			}
			data := map[string]interface{}{}
			json.Unmarshal(message[len(message)-1], &data)
			if data == nil {
				w.writeLog("Received invalid message")
				break
			}
			workerFunction := w.registeredWorkerFunctions[data["method"].(string)]
			if workerFunction == nil {
				w.writeLog("Unregistered worker function:", data["method"].(string))
				break
			}
			if w.runningWorkers == 0 {
				socket.SetSockOptInt(zmq.RCVTIMEO, w.activeTimeout)
			}
			w.runningWorkers += 1
			go runFunction(responseChannel, message, data["parameters"], workerFunction)
		}
	}
}

<|endoftext|>


<commit_before>package testserver

import (
	"net/http/httptest"

	"github.com/gorilla/mux"
	"github.com/pivotal-cf-experimental/warrant/internal/server/clients"
	"github.com/pivotal-cf-experimental/warrant/internal/server/common"
	"github.com/pivotal-cf-experimental/warrant/internal/server/domain"
	"github.com/pivotal-cf-experimental/warrant/internal/server/groups"
	"github.com/pivotal-cf-experimental/warrant/internal/server/tokens"
	"github.com/pivotal-cf-experimental/warrant/internal/server/users"
)

var defaultScopes = []string{
	"scim.read",
	"cloudcontroller.admin",
	"password.write",
	"scim.write",
	"openid",
	"cloud_controller.write",
	"cloud_controller.read",
	"doppler.firehose",
	"notification_preferences.write",
	"notification_preferences.read",
}

// UAA is a fake implementation of the UAA HTTP service.
type UAA struct {
	server  *httptest.Server
	users   *domain.Users
	clients *domain.Clients
	groups  *domain.Groups
	tokens  *domain.Tokens

	publicKey  string
	privateKey string
}

// NewUAA returns a new UAA initialized with the given Config.
func NewUAA() *UAA {
	privateKey := common.TestPrivateKey
	publicKey := common.TestPublicKey

	tokensCollection := domain.NewTokens(publicKey, privateKey, defaultScopes)
	usersCollection := domain.NewUsers()
	clientsCollection := domain.NewClients()
	groupsCollection := domain.NewGroups()

	router := mux.NewRouter()
	uaa := &UAA{
		server:    httptest.NewUnstartedServer(router),
		tokens:    tokensCollection,
		users:     usersCollection,
		clients:   clientsCollection,
		groups:    groupsCollection,
		publicKey: publicKey,
	}

	tokenRouter := tokens.NewRouter(
		tokensCollection,
		usersCollection,
		clientsCollection,
		publicKey,
		privateKey,
		uaa)

	router.Handle("/Users{a:.*}", users.NewRouter(usersCollection, tokensCollection))
	router.Handle("/Groups{a:.*}", groups.NewRouter(groupsCollection, tokensCollection))
	router.Handle("/oauth/clients{a:.*}", clients.NewRouter(clientsCollection, tokensCollection))
	router.Handle("/oauth{a:.*}", tokenRouter)
	router.Handle("/token_key{a:.*}", tokenRouter)

	return uaa
}

func (s *UAA) PublicKey() string {
	return s.publicKey
}

func (s *UAA) PrivateKey() string {
	return s.privateKey
}

// Start will cause the HTTP server to bind to a port
// and start serving requests.
func (s *UAA) Start() {
	s.server.Start()
}

// Close will cause the HTTP server to stop serving
// requests and close its connection.
func (s *UAA) Close() {
	s.server.Close()
}

// Reset will clear all internal resource state within
// the server. This means that all users, clients, and
// groups will be deleted.
func (s *UAA) Reset() {
	s.users.Clear()
	s.clients.Clear()
	s.groups.Clear()
}

// URL returns the url that the server is hosted on.
func (s *UAA) URL() string {
	return s.server.URL
}

// SetDefaultScopes allows the default scopes applied to a
// user to be configured.
func (s *UAA) SetDefaultScopes(scopes []string) {
	s.tokens.DefaultScopes = scopes
} // TODO: move this configuration onto the Config

// ResetDefaultScopes resets the default scopes back to their
// original values.
func (s *UAA) ResetDefaultScopes() {
	s.tokens.DefaultScopes = defaultScopes
}

// UserTokenFor returns a user token with the given id,
// scopes, and audiences.
func (s *UAA) UserTokenFor(userID string, scopes, audiences []string) string {
	// TODO: remove from API so that tokens are fetched like
	// they would be with a real UAA server.

	return s.tokens.Encrypt(domain.Token{
		UserID:    userID,
		Scopes:    scopes,
		Audiences: audiences,
	})
}
<commit_msg>Include private signing key when booting testserver UAA<commit_after>package testserver

import (
	"net/http/httptest"

	"github.com/gorilla/mux"
	"github.com/pivotal-cf-experimental/warrant/internal/server/clients"
	"github.com/pivotal-cf-experimental/warrant/internal/server/common"
	"github.com/pivotal-cf-experimental/warrant/internal/server/domain"
	"github.com/pivotal-cf-experimental/warrant/internal/server/groups"
	"github.com/pivotal-cf-experimental/warrant/internal/server/tokens"
	"github.com/pivotal-cf-experimental/warrant/internal/server/users"
)

var defaultScopes = []string{
	"scim.read",
	"cloudcontroller.admin",
	"password.write",
	"scim.write",
	"openid",
	"cloud_controller.write",
	"cloud_controller.read",
	"doppler.firehose",
	"notification_preferences.write",
	"notification_preferences.read",
}

// UAA is a fake implementation of the UAA HTTP service.
type UAA struct {
	server  *httptest.Server
	users   *domain.Users
	clients *domain.Clients
	groups  *domain.Groups
	tokens  *domain.Tokens

	publicKey  string
	privateKey string
}

// NewUAA returns a new UAA initialized with the given Config.
func NewUAA() *UAA {
	privateKey := common.TestPrivateKey
	publicKey := common.TestPublicKey

	tokensCollection := domain.NewTokens(publicKey, privateKey, defaultScopes)
	usersCollection := domain.NewUsers()
	clientsCollection := domain.NewClients()
	groupsCollection := domain.NewGroups()

	router := mux.NewRouter()
	uaa := &UAA{
		server:     httptest.NewUnstartedServer(router),
		tokens:     tokensCollection,
		users:      usersCollection,
		clients:    clientsCollection,
		groups:     groupsCollection,
		privateKey: privateKey,
		publicKey:  publicKey,
	}

	tokenRouter := tokens.NewRouter(
		tokensCollection,
		usersCollection,
		clientsCollection,
		publicKey,
		privateKey,
		uaa)

	router.Handle("/Users{a:.*}", users.NewRouter(usersCollection, tokensCollection))
	router.Handle("/Groups{a:.*}", groups.NewRouter(groupsCollection, tokensCollection))
	router.Handle("/oauth/clients{a:.*}", clients.NewRouter(clientsCollection, tokensCollection))
	router.Handle("/oauth{a:.*}", tokenRouter)
	router.Handle("/token_key{a:.*}", tokenRouter)

	return uaa
}

func (s *UAA) PublicKey() string {
	return s.publicKey
}

func (s *UAA) PrivateKey() string {
	return s.privateKey
}

// Start will cause the HTTP server to bind to a port
// and start serving requests.
func (s *UAA) Start() {
	s.server.Start()
}

// Close will cause the HTTP server to stop serving
// requests and close its connection.
func (s *UAA) Close() {
	s.server.Close()
}

// Reset will clear all internal resource state within
// the server. This means that all users, clients, and
// groups will be deleted.
func (s *UAA) Reset() {
	s.users.Clear()
	s.clients.Clear()
	s.groups.Clear()
}

// URL returns the url that the server is hosted on.
func (s *UAA) URL() string {
	return s.server.URL
}

// SetDefaultScopes allows the default scopes applied to a
// user to be configured.
func (s *UAA) SetDefaultScopes(scopes []string) {
	s.tokens.DefaultScopes = scopes
} // TODO: move this configuration onto the Config

// ResetDefaultScopes resets the default scopes back to their
// original values.
func (s *UAA) ResetDefaultScopes() {
	s.tokens.DefaultScopes = defaultScopes
}

// UserTokenFor returns a user token with the given id,
// scopes, and audiences.
func (s *UAA) UserTokenFor(userID string, scopes, audiences []string) string {
	// TODO: remove from API so that tokens are fetched like
	// they would be with a real UAA server.

	return s.tokens.Encrypt(domain.Token{
		UserID:    userID,
		Scopes:    scopes,
		Audiences: audiences,
	})
}
<|endoftext|>