<commit_before>package main_test

import (
	"fmt"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/auction/communication/http/auction_http_client"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep/cmd/rep/testrunner"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager/lagertest"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var runner *testrunner.Runner

var _ = Describe("The Rep", func() {
	var (
		fakeExecutor    *ghttp.Server
		bbs             *Bbs.BBS
		pollingInterval time.Duration
	)

	BeforeEach(func() {
		fakeExecutor = ghttp.NewServer()
		// these tests only look for the start of a sequence of requests
		fakeExecutor.AllowUnhandledRequests = true
		fakeExecutor.RouteToHandler("GET", "/ping", ghttp.RespondWith(http.StatusOK, nil))

		bbs = Bbs.NewBBS(etcdRunner.Adapter(), timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		pollingInterval = 50 * time.Millisecond

		runner = testrunner.New(
			representativePath,
			cellID,
			"the-stack",
			"the-lrp-host",
			fakeExecutor.URL(),
			fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"info",
			auctionServerPort,
			time.Second,
			pollingInterval,
		)

		runner.Start()
	})

	AfterEach(func(done Done) {
		runner.KillWithFire()
		fakeExecutor.Close()
		close(done)
	})

	Describe("when a rep starts up", func() {
		BeforeEach(func() {
			_, err := bbs.ReportActualLRPAsStarting("some-process-guid1", "some-instance-guid1", cellID, "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())

			lrp2rep1, err := bbs.ReportActualLRPAsStarting("some-process-guid2", "some-instance-guid2", cellID, "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
			err = bbs.ReportActualLRPAsRunning(lrp2rep1, cellID)
			Ω(err).ShouldNot(HaveOccurred())

			_, err = bbs.ReportActualLRPAsStarting("some-process-guid3", "some-instance-guid3", "different-cell-id", "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())

			lrp2rep2, err := bbs.ReportActualLRPAsStarting("some-process-guid4", "some-instance-guid4", "different-cell-id", "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
			err = bbs.ReportActualLRPAsRunning(lrp2rep2, "different-cell-id")
			Ω(err).ShouldNot(HaveOccurred())

			runner.Stop()
			actualLrpsForRep1 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID(cellID)
			}
			Consistently(actualLrpsForRep1).Should(HaveLen(2))
			actualLrpsForRep2 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID("different-cell-id")
			}
			Consistently(actualLrpsForRep2).Should(HaveLen(2))
		})

		JustBeforeEach(func() {
			runner.Start()
		})

		It("should delete its corresponding actual LRPs", func() {
			actualLrpsForRep1 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID(cellID)
			}
			Eventually(actualLrpsForRep1).Should(BeEmpty())

			actualLrpsForRep2 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID("different-cell-id")
			}
			Consistently(actualLrpsForRep2).Should(HaveLen(2))
		})
	})

	Describe("when an interrupt signal is sent to the representative", func() {
		BeforeEach(func() {
			runner.Stop()
		})

		It("should die", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(0))
		})
	})

	Describe("maintaining presence", func() {
		var cellPresence models.CellPresence

		BeforeEach(func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())
			cellPresence = cells[0]
		})

		It("should maintain presence", func() {
			Ω(cellPresence.Stack).Should(Equal("the-stack"))
			Ω(cellPresence.CellID).Should(Equal(cellID))
		})

		Context("when the presence fails to be maintained", func() {
			It("should not exit, but keep trying to maintain presence at the same ID", func() {
				etcdRunner.Stop()
				etcdRunner.Start()

				Eventually(bbs.Cells, 5).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(cells[0]).Should(Equal(cellPresence))

				Ω(runner.Session).ShouldNot(gexec.Exit())
			})
		})
	})

	Describe("when a task is written to the BBS", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers/the-task-guid"),
				ghttp.RespondWith(http.StatusCreated, `{"executor_guid":"executor-guid","guid":"guid-123"}`)),
			)

			bbs.DesireTask(models.Task{
				TaskGuid: "the-task-guid",
				Stack:    "the-stack",
				Action: &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/file"},
				},
			})
		})

		It("makes a request to the executor", func() {
			Eventually(fakeExecutor.ReceivedRequests).Should(HaveLen(2))
		})
	})

	Describe("acting as an auction representative", func() {
		BeforeEach(func() {
			fakeExecutor.AllowUnhandledRequests = true
			fakeExecutor.UnhandledRequestStatusCode = http.StatusOK
			fakeExecutor.RouteToHandler("GET", "/resources/total", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
				MemoryMB:   1024,
				DiskMB:     2048,
				Containers: 4,
			}))
			fakeExecutor.RouteToHandler("GET", "/resources/remaining", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
				MemoryMB:   512,
				DiskMB:     1024,
				Containers: 2,
			}))
			fakeExecutor.RouteToHandler("GET", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, []executor.Container{}))
		})

		It("makes a request to the executor", func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())

			client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

			state, err := client.State()
			Ω(err).ShouldNot(HaveOccurred())
			Ω(state.TotalResources).Should(Equal(auctiontypes.Resources{
				MemoryMB:   1024,
				DiskMB:     2048,
				Containers: 4,
			}))
			Ω(state.AvailableResources).Should(Equal(auctiontypes.Resources{
				MemoryMB:   512,
				DiskMB:     1024,
				Containers: 2,
			}))
			Ω(state.Stack).Should(Equal("the-stack"))
		})
	})

	Describe("polling the BBS for tasks to reap", func() {
		var task models.Task

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			task = models.Task{
				TaskGuid: "a-new-task-guid",
				Domain:   "the-domain",
				Action: &models.RunAction{
					Path: "the-path",
					Args: []string{},
				},
				Stack: "the-stack",
			}

			err := bbs.DesireTask(task)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.ClaimTask(task.TaskGuid, cellID)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually marks tasks with no corresponding container as failed", func() {
			Eventually(bbs.CompletedTasks, 5*pollingInterval).Should(HaveLen(1))

			completedTasks, err := bbs.CompletedTasks()
			Ω(err).ShouldNot(HaveOccurred())

			Ω(completedTasks[0].TaskGuid).Should(Equal(task.TaskGuid))
			Ω(completedTasks[0].Failed).Should(BeTrue())
		})
	})

	Describe("polling the BBS for actual LRPs to reap", func() {
		var actualLRP models.ActualLRP

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			var err error
			actualLRP, err = bbs.ReportActualLRPAsStarting("process-guid", "a-new-instance-guid", cellID, "the-domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually reaps actual LRPs with no corresponding container", func() {
			Eventually(bbs.ActualLRPs, 5*pollingInterval).Should(BeEmpty())
		})
	})

	Describe("when a StopLRPInstance request comes in", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("DELETE", "/containers/some-instance-guid"),
					ghttp.RespondWith(http.StatusOK, nil),
				),
			)

			err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Domain:       "the-domain",
				Index:        3,
			}, cellID)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.RequestStopLRPInstance(models.StopLRPInstance{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Index:        3,
			})
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("should delete the container and resolve the StopLRPInstance", func() {
			findDeleteRequest := func() string {
				for _, req := range fakeExecutor.ReceivedRequests() {
					if req.Method == "DELETE" {
						return req.URL.Path
					}
				}
				return ""
			}

			Eventually(findDeleteRequest).Should(Equal("/containers/some-instance-guid"))
			Eventually(bbs.ActualLRPs).Should(BeEmpty())
			Eventually(bbs.StopLRPInstances).Should(BeEmpty())
		})
	})
})
<commit_msg>Try to appease the race detector<commit_after>package main_test

import (
	"fmt"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/auction/communication/http/auction_http_client"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep/cmd/rep/testrunner"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager/lagertest"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var runner *testrunner.Runner

var _ = Describe("The Rep", func() {
	var (
		fakeExecutor    *ghttp.Server
		bbs             *Bbs.BBS
		pollingInterval time.Duration
	)

	BeforeEach(func() {
		fakeExecutor = ghttp.NewServer()
		// these tests only look for the start of a sequence of requests
		fakeExecutor.AllowUnhandledRequests = true
		fakeExecutor.RouteToHandler("GET", "/ping", ghttp.RespondWith(http.StatusOK, nil))

		bbs = Bbs.NewBBS(etcdRunner.Adapter(), timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		pollingInterval = 50 * time.Millisecond

		runner = testrunner.New(
			representativePath,
			cellID,
			"the-stack",
			"the-lrp-host",
			fakeExecutor.URL(),
			fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"info",
			auctionServerPort,
			time.Second,
			pollingInterval,
		)

		runner.Start()
	})

	AfterEach(func(done Done) {
		runner.KillWithFire()
		fakeExecutor.Close()
		close(done)
	})

	Describe("when a rep starts up", func() {
		BeforeEach(func() {
			_, err := bbs.ReportActualLRPAsStarting("some-process-guid1", "some-instance-guid1", cellID, "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())

			lrp2rep1, err := bbs.ReportActualLRPAsStarting("some-process-guid2", "some-instance-guid2", cellID, "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
			err = bbs.ReportActualLRPAsRunning(lrp2rep1, cellID)
			Ω(err).ShouldNot(HaveOccurred())

			_, err = bbs.ReportActualLRPAsStarting("some-process-guid3", "some-instance-guid3", "different-cell-id", "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())

			lrp2rep2, err := bbs.ReportActualLRPAsStarting("some-process-guid4", "some-instance-guid4", "different-cell-id", "domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
			err = bbs.ReportActualLRPAsRunning(lrp2rep2, "different-cell-id")
			Ω(err).ShouldNot(HaveOccurred())

			runner.Stop()
			actualLrpsForRep1 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID(cellID)
			}
			Consistently(actualLrpsForRep1).Should(HaveLen(2))
			actualLrpsForRep2 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID("different-cell-id")
			}
			Consistently(actualLrpsForRep2).Should(HaveLen(2))
		})

		JustBeforeEach(func() {
			runner.Start()
		})

		It("should delete its corresponding actual LRPs", func() {
			actualLrpsForRep1 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID(cellID)
			}
			Eventually(actualLrpsForRep1).Should(BeEmpty())

			actualLrpsForRep2 := func() ([]models.ActualLRP, error) {
				return bbs.ActualLRPsByCellID("different-cell-id")
			}
			Consistently(actualLrpsForRep2).Should(HaveLen(2))
		})
	})

	Describe("when an interrupt signal is sent to the representative", func() {
		BeforeEach(func() {
			runner.Stop()
		})

		It("should die", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(0))
		})
	})

	Describe("maintaining presence", func() {
		var cellPresence models.CellPresence

		BeforeEach(func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())
			cellPresence = cells[0]
		})

		It("should maintain presence", func() {
			Ω(cellPresence.Stack).Should(Equal("the-stack"))
			Ω(cellPresence.CellID).Should(Equal(cellID))
		})

		Context("when the presence fails to be maintained", func() {
			It("should not exit, but keep trying to maintain presence at the same ID", func() {
				etcdRunner.Stop()
				etcdRunner.Start()

				Eventually(bbs.Cells, 5).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(cells[0]).Should(Equal(cellPresence))

				Ω(runner.Session).ShouldNot(gexec.Exit())
			})
		})
	})

	Describe("when a task is written to the BBS", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers/the-task-guid"),
				ghttp.RespondWith(http.StatusCreated, `{"executor_guid":"executor-guid","guid":"guid-123"}`)),
			)

			bbs.DesireTask(models.Task{
				TaskGuid: "the-task-guid",
				Stack:    "the-stack",
				Action: &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/file"},
				},
			})
		})

		It("makes a request to the executor", func() {
			Eventually(fakeExecutor.ReceivedRequests).Should(HaveLen(2))
		})
	})

	Describe("acting as an auction representative", func() {
		BeforeEach(func() {
			fakeExecutor.RouteToHandler("GET", "/resources/total", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
				MemoryMB:   1024,
				DiskMB:     2048,
				Containers: 4,
			}))
			fakeExecutor.RouteToHandler("GET", "/resources/remaining", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
				MemoryMB:   512,
				DiskMB:     1024,
				Containers: 2,
			}))
			fakeExecutor.RouteToHandler("GET", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, []executor.Container{}))
		})

		It("makes a request to the executor", func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())

			client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

			state, err := client.State()
			Ω(err).ShouldNot(HaveOccurred())
			Ω(state.TotalResources).Should(Equal(auctiontypes.Resources{
				MemoryMB:   1024,
				DiskMB:     2048,
				Containers: 4,
			}))
			Ω(state.AvailableResources).Should(Equal(auctiontypes.Resources{
				MemoryMB:   512,
				DiskMB:     1024,
				Containers: 2,
			}))
			Ω(state.Stack).Should(Equal("the-stack"))
		})
	})

	Describe("polling the BBS for tasks to reap", func() {
		var task models.Task

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			task = models.Task{
				TaskGuid: "a-new-task-guid",
				Domain:   "the-domain",
				Action: &models.RunAction{
					Path: "the-path",
					Args: []string{},
				},
				Stack: "the-stack",
			}

			err := bbs.DesireTask(task)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.ClaimTask(task.TaskGuid, cellID)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually marks tasks with no corresponding container as failed", func() {
			Eventually(bbs.CompletedTasks, 5*pollingInterval).Should(HaveLen(1))

			completedTasks, err := bbs.CompletedTasks()
			Ω(err).ShouldNot(HaveOccurred())

			Ω(completedTasks[0].TaskGuid).Should(Equal(task.TaskGuid))
			Ω(completedTasks[0].Failed).Should(BeTrue())
		})
	})

	Describe("polling the BBS for actual LRPs to reap", func() {
		var actualLRP models.ActualLRP

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			var err error
			actualLRP, err = bbs.ReportActualLRPAsStarting("process-guid", "a-new-instance-guid", cellID, "the-domain", 0)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually reaps actual LRPs with no corresponding container", func() {
			Eventually(bbs.ActualLRPs, 5*pollingInterval).Should(BeEmpty())
		})
	})

	Describe("when a StopLRPInstance request comes in", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("DELETE", "/containers/some-instance-guid"),
					ghttp.RespondWith(http.StatusOK, nil),
				),
			)

			err := bbs.ReportActualLRPAsRunning(models.ActualLRP{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Domain:       "the-domain",
				Index:        3,
			}, cellID)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.RequestStopLRPInstance(models.StopLRPInstance{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Index:        3,
			})
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("should delete the container and resolve the StopLRPInstance", func() {
			findDeleteRequest := func() string {
				for _, req := range fakeExecutor.ReceivedRequests() {
					if req.Method == "DELETE" {
						return req.URL.Path
					}
				}
				return ""
			}

			Eventually(findDeleteRequest).Should(Equal("/containers/some-instance-guid"))
			Eventually(bbs.ActualLRPs).Should(BeEmpty())
			Eventually(bbs.StopLRPInstances).Should(BeEmpty())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"launchpad.net/gnuflag"
)

type collectorCmd struct {
	fs     *gnuflag.FlagSet
	config string
	dry    bool
}

func (collectorCmd) Run(context *cmd.Context, client *cmd.Client) error {
	return nil
}

func (collectorCmd) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "collector",
		Usage:   "collector",
		Desc:    "Starts the tsuru collector.",
		MinArgs: 0,
	}
}

func (c *collectorCmd) Flags() *gnuflag.FlagSet {
	if c.fs == nil {
		c.fs = gnuflag.NewFlagSet("api", gnuflag.ExitOnError)
		c.fs.BoolVar(&c.dry, "dry", false, "dry-run: does not run the collector (for testing purpose)")
		c.fs.BoolVar(&c.dry, "d", false, "dry-run: does not run the collector (for testing purpose)")
		c.fs.StringVar(&c.config, "config", "/etc/tsuru/tsuru.conf", "tsr collector config file.")
		c.fs.StringVar(&c.config, "c", "/etc/tsuru/tsuru.conf", "tsr collector config file.")
	}
	return c.fs
}
<commit_msg>tsr: running collector.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/collector"
	"launchpad.net/gnuflag"
)

type collectorCmd struct {
	fs     *gnuflag.FlagSet
	config string
	dry    bool
}

func (c *collectorCmd) Run(context *cmd.Context, client *cmd.Client) error {
	flags := map[string]interface{}{}
	flags["dry"] = c.dry
	flags["config"] = c.config
	collector.Run(flags)
	return nil
}

func (collectorCmd) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "collector",
		Usage:   "collector",
		Desc:    "Starts the tsuru collector.",
		MinArgs: 0,
	}
}

func (c *collectorCmd) Flags() *gnuflag.FlagSet {
	if c.fs == nil {
		c.fs = gnuflag.NewFlagSet("api", gnuflag.ExitOnError)
		c.fs.BoolVar(&c.dry, "dry", false, "dry-run: does not run the collector (for testing purpose)")
		c.fs.BoolVar(&c.dry, "d", false, "dry-run: does not run the collector (for testing purpose)")
		c.fs.StringVar(&c.config, "config", "/etc/tsuru/tsuru.conf", "tsr collector config file.")
		c.fs.StringVar(&c.config, "c", "/etc/tsuru/tsuru.conf", "tsr collector config file.")
	}
	return c.fs
}
<|endoftext|>


<commit_before>// Copyright 2013 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"syscall"
)

func loadVTOC(name string) (*os.File, []file, error) {
	var l int64
	f, err := os.Open(name)
	if err != nil {
		return nil, nil, err
	}
	r := io.LimitReader(f, 8)
	_, err = fmt.Fscanf(r, "%x", &l)
	if err != nil {
		return nil, nil, errors.New(fmt.Sprintf("%s: can't scan vtoc length: %v", name, err))
	}

	r = io.LimitReader(f, l)
	dec := json.NewDecoder(r)

	var vtoc []file
	if err := dec.Decode(&vtoc); err != nil {
		return nil, nil, errors.New(fmt.Sprintf("%s: can't decode: %v", name, err))
	}
	return f, vtoc, nil
}

func buildVTOC(dirs []string) ([]*file, error) {
	var vtoc []*file
	for _, v := range dirs {
		debug("Process %v", v)
		err := filepath.Walk(v, func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			debug("visit %v", name)
			var s syscall.Stat_t
			if err := syscall.Lstat(name, &s); err != nil {
				return errors.New(fmt.Sprintf("%s: %v", name, err))
			}
			f := &file{
				Name:    name,
				Mode:    fi.Mode(),
				ModTime: fi.ModTime(),
				IsDir:   fi.IsDir(),
				Uid:     int(s.Uid),
				Gid:     int(s.Gid),
			}
			switch f.Mode.String()[0] {
			case '-':
				f.Size = fi.Size()
			case 'L':
				f.Link, err = os.Readlink(name)
			case 'D':
				f.Dev = s.Rdev
			}
			vtoc = append(vtoc, f)
			return err
		})
		if err != nil {
			return nil, err
		}
	}

	return vtoc, nil
}

func encodeVTOC(vtoc []*file) (int64, error) {
	return -1, errors.New("not yet")
}

func writeVTOC(f io.Writer, vtoc []*file) (int, error) {
	var outvtoc = make([]*file, len(vtoc))
	// Do a little cleanup
	for i := range vtoc {
		// The path names may be absolute but they will also have been cleaned.
		// If the first char is /, it won't be.
		// N.B. This assumes Unix-style file names. But I don't care about Windows.
		v := *vtoc[i]
		outvtoc[i] = &v
		if outvtoc[i].Name[0] == '/' {
			outvtoc[i].Name = outvtoc[i].Name[1:]
		}
	}

	var v bytes.Buffer
	enc := json.NewEncoder(&v)
	if err := enc.Encode(outvtoc); err != nil {
		return -1, errors.New(fmt.Sprintf("Encoding files: %v", err))
	}

	if _, err := fmt.Fprintf(f, "%07x\n", v.Len()); err != nil {
		return -1, err
	}
	debug("vtoc size is %d", v.Len())
	return f.Write(v.Bytes())
}

func NewVTOCEncoder(opts ...VTOCOpt) error {
	var v vtoc

	for _, o := range opts {
		if err := o(&v); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>fix old errors.New(fmt.Sprintf) usage<commit_after>// Copyright 2013 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"syscall"
)

func loadVTOC(name string) (*os.File, []file, error) {
	var l int64
	f, err := os.Open(name)
	if err != nil {
		return nil, nil, err
	}
	r := io.LimitReader(f, 8)
	_, err = fmt.Fscanf(r, "%x", &l)
	if err != nil {
		return nil, nil, fmt.Errorf("%s: can't scan vtoc length: %v", name, err)
	}

	r = io.LimitReader(f, l)
	dec := json.NewDecoder(r)

	var vtoc []file
	if err := dec.Decode(&vtoc); err != nil {
		return nil, nil, fmt.Errorf("%s: can't decode: %v", name, err)
	}
	return f, vtoc, nil
}

func buildVTOC(dirs []string) ([]*file, error) {
	var vtoc []*file
	for _, v := range dirs {
		debug("Process %v", v)
		err := filepath.Walk(v, func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			debug("visit %v", name)
			var s syscall.Stat_t
			if err := syscall.Lstat(name, &s); err != nil {
				return fmt.Errorf("%s: %v", name, err)
			}
			f := &file{
				Name:    name,
				Mode:    fi.Mode(),
				ModTime: fi.ModTime(),
				IsDir:   fi.IsDir(),
				Uid:     int(s.Uid),
				Gid:     int(s.Gid),
			}
			switch f.Mode.String()[0] {
			case '-':
				f.Size = fi.Size()
			case 'L':
				f.Link, err = os.Readlink(name)
			case 'D':
				f.Dev = s.Rdev
			}
			vtoc = append(vtoc, f)
			return err
		})
		if err != nil {
			return nil, err
		}
	}

	return vtoc, nil
}

func encodeVTOC(vtoc []*file) (int64, error) {
	return -1, errors.New("not yet")
}

func writeVTOC(f io.Writer, vtoc []*file) (int, error) {
	var outvtoc = make([]*file, len(vtoc))
	// Do a little cleanup
	for i := range vtoc {
		// The path names may be absolute but they will also have been cleaned.
		// If the first char is /, it won't be.
		// N.B. This assumes Unix-style file names. But I don't care about Windows.
		v := *vtoc[i]
		outvtoc[i] = &v
		if outvtoc[i].Name[0] == '/' {
			outvtoc[i].Name = outvtoc[i].Name[1:]
		}
	}

	var v bytes.Buffer
	enc := json.NewEncoder(&v)
	if err := enc.Encode(outvtoc); err != nil {
		return -1, fmt.Errorf("Encoding files: %v", err)
	}

	if _, err := fmt.Fprintf(f, "%07x\n", v.Len()); err != nil {
		return -1, err
	}
	debug("vtoc size is %d", v.Len())
	return f.Write(v.Bytes())
}

func NewVTOCEncoder(opts ...VTOCOpt) error {
	var v vtoc

	for _, o := range opts {
		if err := o(&v); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package jsre

const pp_js = `
function pp(object) {
    var str = "";

    if(object instanceof Array) {
        str += "[ ";
        for(var i = 0, l = object.length; i < l; i++) {
            str += pp(object[i]);

            if(i < l-1) {
                str += ", ";
            }
        }
        str += " ]";
    } else if(typeof(object) === "object") {
        str += "{ ";
        var last = Object.keys(object).pop()
        for(var k in object) {
            str += k + ": " + pp(object[k]);

            if(k !== last) {
                str += ", ";
            }
        }
        str += " }";
    } else if(typeof(object) === "string") {
        str += "\033[32m'" + object + "'";
    } else if(typeof(object) === "undefined") {
        str += "\033[1m\033[30m" + object;
    } else if(typeof(object) === "number") {
        str += "\033[31m" + object;
    } else if(typeof(object) === "function") {
	str += "\033[35m[Function]";
    } else {
        str += object;
    }

    str += "\033[0m";

    return str;
}

function prettyPrint(/* */) {
    var args = arguments;
    var ret = "";
    for(var i = 0, l = args.length; i < l; i++) {
	    ret += pp(args[i]) + "\n";
    }
    return ret;
}

var print = prettyPrint;
`
<commit_msg>Improved pretty printer for console<commit_after>package jsre

const pp_js = `
function pp(object, indent) {
    var str = "";
    /*
    var o = object;
    try {
	object = JSON.stringify(object)
	object = JSON.parse(object);
   } catch(e) {
	object = o;
   }
   */

    if(object instanceof Array) {
        str += "[";
        for(var i = 0, l = object.length; i < l; i++) {
            str += pp(object[i], indent);

            if(i < l-1) {
                str += ", ";
            }
        }
        str += " ]";
    } else if(object instanceof BigNumber) {
	    return pp(object.toString(), indent);
    } else if(typeof(object) === "object") {
        str += "{\n";
	indent += "  ";
        var last = Object.keys(object).pop()
        for(var k in object) {
            str += indent + k + ": " + pp(object[k], indent);

            if(k !== last) {
                str += ",";
            }
	    str += "\n";
        }
        str += indent.substr(2, indent.length) + "}";
    } else if(typeof(object) === "string") {
        str += "\033[32m'" + object + "'";
    } else if(typeof(object) === "undefined") {
        str += "\033[1m\033[30m" + object;
    } else if(typeof(object) === "number") {
        str += "\033[31m" + object;
    } else if(typeof(object) === "function") {
	str += "\033[35m[Function]";
    } else {
        str += object;
    }

    str += "\033[0m";

    return str;
}

function prettyPrint(/* */) {
    var args = arguments;
    var ret = "";
    for(var i = 0, l = args.length; i < l; i++) {
	    ret += pp(args[i], "") + "\n";
    }
    return ret;
}

var print = prettyPrint;
`
<|endoftext|>


<commit_before>package client

import (
	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

// UnitsService communicates with the source unit-related endpoints in
// the Sourcegraph API.
type UnitsService interface {
	// List units.
	List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error)
}

// UnitSpec specifies a source unit.
type UnitSpec struct {
	Repo     string
	CommitID string
	UnitType string
	Unit     string
}

func (s *UnitSpec) RouteVars() map[string]string {
	m := map[string]string{"RepoURI": s.Repo, "UnitType": s.UnitType, "Unit": s.Unit}
	if s.CommitID != "" {
		m["Rev"] = s.CommitID
	}
	return m
}

// unitsService implements UnitsService.
type unitsService struct {
	client *Client
}

var _ UnitsService = &unitsService{}

// UnitListOptions specifies options for UnitsService.List.
type UnitListOptions struct {
	// Filters
	RepositoryURI string `url:",omitempty"`
	CommitID      string `url:",omitempty"`
	UnitType      string `url:",omitempty"`
	Unit          string `url:",omitempty"`

	// Paging
	ListOptions
}

func (s *unitsService) List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error) {
	url, err := s.client.url(api_router.Units, nil, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var units []*unit.RepoSourceUnit
	resp, err := s.client.Do(req, &units)
	if err != nil {
		return nil, resp, err
	}

	return units, resp, nil
}

type MockUnitsService struct {
	List_ func(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error)
}

var _ UnitsService = MockUnitsService{}

func (s MockUnitsService) List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error) {
	if s.List_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.List_(opt)
}
<commit_msg>add unit get to api/client<commit_after>package client

import (
	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

// UnitsService communicates with the source unit-related endpoints in
// the Sourcegraph API.
type UnitsService interface {
	// Get fetches a unit.
	Get(spec *UnitSpec) (*unit.RepoSourceUnit, Response, error)

	// List units.
	List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error)
}

// UnitSpec specifies a source unit.
type UnitSpec struct {
	Repo     string
	CommitID string
	UnitType string
	Unit     string
}

func SpecFromUnit(u *unit.RepoSourceUnit) *UnitSpec {
	return &UnitSpec{
		Repo:     string(u.Repo),
		CommitID: u.CommitID,
		UnitType: u.UnitType,
		Unit:     u.Unit,
	}
}

func (s *UnitSpec) RouteVars() map[string]string {
	m := map[string]string{"RepoURI": s.Repo, "UnitType": s.UnitType, "Unit": s.Unit}
	if s.CommitID != "" {
		m["Rev"] = s.CommitID
	}
	return m
}

// unitsService implements UnitsService.
type unitsService struct {
	client *Client
}

var _ UnitsService = &unitsService{}

// UnitListOptions specifies options for UnitsService.List.
type UnitListOptions struct {
	// Filters
	RepositoryURI string `url:",omitempty"`
	CommitID      string `url:",omitempty"`
	UnitType      string `url:",omitempty"`
	Unit          string `url:",omitempty"`

	// Paging
	ListOptions
}

func (s *unitsService) Get(spec *UnitSpec) (*unit.RepoSourceUnit, Response, error) {
	url, err := s.client.url(api_router.Unit, spec.RouteVars(), nil)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var u unit.RepoSourceUnit
	resp, err := s.client.Do(req, &u)
	if err != nil {
		return nil, resp, err
	}

	return &u, resp, nil
}

func (s *unitsService) List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error) {
	url, err := s.client.url(api_router.Units, nil, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var units []*unit.RepoSourceUnit
	resp, err := s.client.Do(req, &units)
	if err != nil {
		return nil, resp, err
	}

	return units, resp, nil
}

type MockUnitsService struct {
	List_ func(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error)
	Get_  func(spec *UnitSpec) (*unit.RepoSourceUnit, Response, error)
}

var _ UnitsService = MockUnitsService{}

func (s MockUnitsService) List(opt *UnitListOptions) ([]*unit.RepoSourceUnit, Response, error) {
	if s.List_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.List_(opt)
}

func (s MockUnitsService) Get(spec *UnitSpec) (*unit.RepoSourceUnit, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(spec)
}
<|endoftext|>


<commit_before>package chyle

import (
	"os/exec"
	"testing"

	"github.com/stretchr/testify/assert"

	"srcd.works/go-git.v4"
	"srcd.works/go-git.v4/plumbing/object"
)

func TestResolveRef(t *testing.T) {
	type g struct {
		ref string
		f   func(*object.Commit, error)
	}

	tests := []g{
		g{
			"HEAD",
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("HEAD").ID().String(), "Must resolve HEAD reference")
			},
		},
		g{
			"test1",
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("test1").ID().String(), "Must resolve branch reference")
			},
		},
		g{
			getCommitFromRef("test1").ID().String(),
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("test1").ID().String(), "Must resolve commit id")
			},
		},
	}

	for _, test := range tests {
		test.f(resolveRef(test.ref, repo))
	}
}

func TestResolveRefWithErrors(t *testing.T) {
	type g struct {
		ref  string
		repo *git.Repository
		f    func(*object.Commit, error)
	}

	tests := []g{
		g{
			"whatever",
			repo,
			func(o *object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
				assert.EqualError(t, err, `Can't find reference "whatever"`, "Must return an error")
			},
		},
	}

	for _, test := range tests {
		test.f(resolveRef(test.ref, test.repo))
	}
}

func TestFetchCommits(t *testing.T) {
	type g struct {
		toRef   string
		fromRef string
		f       func(*[]object.Commit, error)
	}

	tests := []g{
		g{
			getCommitFromRef("HEAD").ID().String(),
			getCommitFromRef("test").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
			},
		},
		g{
			getCommitFromRef("HEAD~1").ID().String(),
			getCommitFromRef("HEAD~3").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
			},
		},
		g{
			getCommitFromRef("HEAD~3").ID().String(),
			getCommitFromRef("test~2^2").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.NoError(t, err, "Must return no errors")
				assert.Len(t, *cs, 5, "Must contains 3 commits")

				commitTests := []string{
					"Merge branch 'test2' into test1\n",
					"feat(file6) : new file 6\n\ncreate a new file 6\n",
					"feat(file5) : new file 5\n\ncreate a new file 5\n",
					"feat(file4) : new file 4\n\ncreate a new file 4\n",
					"feat(file3) : new file 3\n\ncreate a new file 3\n",
				}

				for i, c := range *cs {
					assert.Equal(t, commitTests[i], c.Message, "Must match message")
				}
			},
		},
		g{
			getCommitFromRef("HEAD~4").ID().String(),
			getCommitFromRef("test~2^2^2").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.NoError(t, err, "Must return no errors")
				assert.Len(t, *cs, 5, "Must contains 3 commits")

				commitTests := []string{
					"feat(file6) : new file 6\n\ncreate a new file 6\n",
					"feat(file5) : new file 5\n\ncreate a new file 5\n",
					"feat(file4) : new file 4\n\ncreate a new file 4\n",
					"feat(file3) : new file 3\n\ncreate a new file 3\n",
					"feat(file2) : new file 2\n\ncreate a new file 2\n",
				}

				for i, c := range *cs {
					assert.Equal(t, commitTests[i], c.Message, "Must match message")
				}
			},
		},
	}

	for _, test := range tests {
		test.f(fetchCommits("test", test.toRef, test.fromRef))
	}
}

func TestShallowCloneProducesNoErrors(t *testing.T) {
	path := "shallow-repository-test"
	cmd := exec.Command("rm", "-rf", path)
	_, err := cmd.Output()

	assert.NoError(t, err, "Must delete repository")

	cmd = exec.Command("git", "clone", "--depth", "2", "git@github.com:octocat/Spoon-Knife.git", path)
	_, err = cmd.Output()

	assert.NoError(t, err, "Must shallow clone repository")

	cmd = exec.Command("git", "rev-parse", "HEAD~1")
	cmd.Dir = path

	fromRef, err := cmd.Output()
	fromRef = fromRef[:len(fromRef)-1]

	assert.NoError(t, err, "Must extract HEAD~1")

	cmd = exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = path

	toRef, err := cmd.Output()
	toRef = toRef[:len(toRef)-1]

	assert.NoError(t, err, "Must extract HEAD")

	commits, err := fetchCommits("shallow-repository-test", string(fromRef), string(toRef))

	assert.NoError(t, err, "Must return no errors")
	assert.Len(t, *commits, 1, "Must fetch commits in shallow clone")
}
<commit_msg>fix(chyle/git_test) : use https<commit_after>package chyle

import (
	"os/exec"
	"testing"

	"github.com/stretchr/testify/assert"

	"srcd.works/go-git.v4"
	"srcd.works/go-git.v4/plumbing/object"
)

func TestResolveRef(t *testing.T) {
	type g struct {
		ref string
		f   func(*object.Commit, error)
	}

	tests := []g{
		g{
			"HEAD",
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("HEAD").ID().String(), "Must resolve HEAD reference")
			},
		},
		g{
			"test1",
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("test1").ID().String(), "Must resolve branch reference")
			},
		},
		g{
			getCommitFromRef("test1").ID().String(),
			func(o *object.Commit, err error) {
				assert.NoError(t, err, "Must contains no error")
				assert.True(t, o.ID().String() == getCommitFromRef("test1").ID().String(), "Must resolve commit id")
			},
		},
	}

	for _, test := range tests {
		test.f(resolveRef(test.ref, repo))
	}
}

func TestResolveRefWithErrors(t *testing.T) {
	type g struct {
		ref  string
		repo *git.Repository
		f    func(*object.Commit, error)
	}

	tests := []g{
		g{
			"whatever",
			repo,
			func(o *object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
				assert.EqualError(t, err, `Can't find reference "whatever"`, "Must return an error")
			},
		},
	}

	for _, test := range tests {
		test.f(resolveRef(test.ref, test.repo))
	}
}

func TestFetchCommits(t *testing.T) {
	type g struct {
		toRef   string
		fromRef string
		f       func(*[]object.Commit, error)
	}

	tests := []g{
		g{
			getCommitFromRef("HEAD").ID().String(),
			getCommitFromRef("test").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
			},
		},
		g{
			getCommitFromRef("HEAD~1").ID().String(),
			getCommitFromRef("HEAD~3").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.Error(t, err, "Must return an error")
			},
		},
		g{
			getCommitFromRef("HEAD~3").ID().String(),
			getCommitFromRef("test~2^2").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.NoError(t, err, "Must return no errors")
				assert.Len(t, *cs, 5, "Must contains 3 commits")

				commitTests := []string{
					"Merge branch 'test2' into test1\n",
					"feat(file6) : new file 6\n\ncreate a new file 6\n",
					"feat(file5) : new file 5\n\ncreate a new file 5\n",
					"feat(file4) : new file 4\n\ncreate a new file 4\n",
					"feat(file3) : new file 3\n\ncreate a new file 3\n",
				}

				for i, c := range *cs {
					assert.Equal(t, commitTests[i], c.Message, "Must match message")
				}
			},
		},
		g{
			getCommitFromRef("HEAD~4").ID().String(),
			getCommitFromRef("test~2^2^2").ID().String(),
			func(cs *[]object.Commit, err error) {
				assert.NoError(t, err, "Must return no errors")
				assert.Len(t, *cs, 5, "Must contains 3 commits")

				commitTests := []string{
					"feat(file6) : new file 6\n\ncreate a new file 6\n",
					"feat(file5) : new file 5\n\ncreate a new file 5\n",
					"feat(file4) : new file 4\n\ncreate a new file 4\n",
					"feat(file3) : new file 3\n\ncreate a new file 3\n",
					"feat(file2) : new file 2\n\ncreate a new file 2\n",
				}

				for i, c := range *cs {
					assert.Equal(t, commitTests[i], c.Message, "Must match message")
				}
			},
		},
	}

	for _, test := range tests {
		test.f(fetchCommits("test", test.toRef, test.fromRef))
	}
}

func TestShallowCloneProducesNoErrors(t *testing.T) {
	path := "shallow-repository-test"
	cmd := exec.Command("rm", "-rf", path)
	_, err := cmd.Output()

	assert.NoError(t, err, "Must delete repository")

	cmd = exec.Command("git", "clone", "--depth", "2", "https://github.com/octocat/Spoon-Knife.git", path)
	_, err = cmd.Output()

	assert.NoError(t, err, "Must shallow clone repository")

	cmd = exec.Command("git", "rev-parse", "HEAD~1")
	cmd.Dir = path

	fromRef, err := cmd.Output()
	fromRef = fromRef[:len(fromRef)-1]

	assert.NoError(t, err, "Must extract HEAD~1")

	cmd = exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = path

	toRef, err := cmd.Output()
	toRef = toRef[:len(toRef)-1]

	assert.NoError(t, err, "Must extract HEAD")

	commits, err := fetchCommits("shallow-repository-test", string(fromRef), string(toRef))

	assert.NoError(t, err, "Must return no errors")
	assert.Len(t, *commits, 1, "Must fetch commits in shallow clone")
}
<|endoftext|>


<commit_before>package golucy

// Copyright 2013 Philip Southam
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
#include "Clownfish/Obj.h"
#define DECREF       cfish_Obj_decref
#define ObjToString CFISH_Obj_To_String

#include "Clownfish/Hash.h"
#define CfishHashKeys CFISH_Hash_Keys //returns cfish_Varray*

#include "Lucy/Search/IndexSearcher.h"
#define LucyIndexSearcher lucy_IndexSearcher
#define LucyIxSearcherNew lucy_IxSearcher_new
#define LucyIxSearcherHits LUCY_IxSearcher_Hits
#define LucyIxSearcherGetSchema LUCY_IxSearcher_Get_Schema
#define LucyIxSearchFetchDocVec LUCY_IxSearcher_Fetch_Doc_Vec

#include "Lucy/Analysis/EasyAnalyzer.h"
#define LucyEasyAnalyzerNew lucy_EasyAnalyzer_new

#include "Lucy/Plan/Schema.h"
#define LucySchema lucy_Schema
#define LucySchemaAllFields LUCY_Schema_All_Fields

#include "Lucy/Search/QueryParser.h"
#define LucyQueryParser lucy_QueryParser
#define LucyQParserNew lucy_QParser_new
#define LucyQParserParse LUCY_QParser_Parse

#include "Lucy/Document/HitDoc.h"
#define LucyHitDoc lucy_HitDoc
#define LucyHitDocExtract LUCY_HitDoc_Extract
#define LucyHitDocGetDocId  LUCY_HitDoc_Get_Doc_ID
#define LucyHitdocDump LUCY_HitDoc_Dump //returns cfish_Hash*
#define LucyHitDocGetScore LUCY_HitDoc_Get_Score

#include "Lucy/Search/Hits.h"
#define LucyHitsTotal LUCY_Hits_Total_Hits
#define LucyHitsNext LUCY_Hits_Next

#include "Lucy/Search/Query.h"
#define LucyQuery lucy_Query
#define LucyQueryMakeCompiler LUCY_Query_Make_Compiler

#include "Lucy/Search/Compiler.h"
#define LucyCompilerHighlightSpans LUCY_Compiler_Highlight_Spans

#include "Clownfish/VArray.h"
#define VaGetSize CFISH_VA_Get_Size
#define VaFetch CFISH_VA_Fetch

#include "Lucy/Search/Span.h"
#define LucySpanGetOffset LUCY_Span_Get_Offset
#define LucySpanGetLength LUCY_Span_Get_Length
*/
import "C"

import "strings"

type Query struct {
	QueryStr   string
	lucySchema *C.LucySchema // we're now carrying this around in 2 places :-/
	lucyQuery  *C.LucyQuery
}

type SearchResult struct {
	Id           string
	Text         string
	Score        float32
	MatchedTerms []string
}

type IndexReader struct {
	Index        *Index
	lucySearcher *C.LucyIndexSearcher
}

func (index *Index) NewIndexReader() *IndexReader {
	ixLocation := cb_newf(index.Path)
	defer C.DECREF(ixLocation)
	return &IndexReader{Index: index, lucySearcher: C.LucyIxSearcherNew(ixLocation)}
}

func (ixReader *IndexReader) ParseQuery(queryStr string) *Query {
	lucySchema := C.LucyIxSearcherGetSchema(ixReader.lucySearcher)
	language := cb_newf("en") // should be configurable
	defer C.DECREF(language)
	analyzer := C.LucyEasyAnalyzerNew(language)
	defer C.DECREF(analyzer)
	qp := C.LucyQParserNew(
		lucySchema,
		analyzer,                          //should this be configurable?
		cb_newf("AND"),                    // should be configurable
		C.LucySchemaAllFields(lucySchema), // should be configurable
	)
	defer C.DECREF(qp)
	return &Query{
		QueryStr:   queryStr,
		lucySchema: lucySchema,
		lucyQuery:  C.LucyQParserParse(qp, cb_new_from_utf8(queryStr)),
	}
}

func (ixReader *IndexReader) Search(query *Query, offset, limit uint, idField string, contentField string, includeMatchedTerms bool) (uint, []*SearchResult) {
	// Should probably have some sort
	// of `Results` object/iterator so that we don't have to specify
	// offset/limit and where I can attach matched terms to the result.
	lIdField, lContentField := cb_newf(idField), cb_newf(contentField) // total hack, need to return more than one field
	defer C.DECREF(lIdField)
	defer C.DECREF(lContentField)
	hits := C.LucyIxSearcherHits(ixReader.lucySearcher, query.lucyQuery, C.uint32_t(offset), C.uint32_t(limit), nil)
	defer C.DECREF(hits)
	totalNumHits := uint(C.LucyHitsTotal(hits))
	num2Return := minUInt(limit, totalNumHits)
	results := make([]*SearchResult, num2Return)
	var hit *C.LucyHitDoc
	compiler := C.LucyQueryMakeCompiler(query.lucyQuery, ixReader.lucySearcher, 1.0, false)
	defer C.DECREF(compiler)

	matchedTerms := func(docId C.int32_t, result *SearchResult) {
		docVec := C.LucyIxSearchFetchDocVec(ixReader.lucySearcher, docId)
		defer C.DECREF(docVec)
		spans := C.LucyCompilerHighlightSpans(compiler, ixReader.lucySearcher, docVec, lContentField)
		defer C.DECREF(spans)
		spanCnt := C.VaGetSize(spans)
		if spanCnt == 0 {
			// should never get here, but just in case...
			return
		}
		result.MatchedTerms = make([]string, spanCnt)
		var i C.uint32_t
		for i = 0; i < spanCnt; i++ {
			span := C.VaFetch(spans, i)
			offset := C.LucySpanGetOffset(span)
			length := C.LucySpanGetLength(span)
			result.MatchedTerms[i] = string([]rune(result.Text)[offset : offset+length])
		}
		// make terms unique?
		result.MatchedTerms = set(result.MatchedTerms)
	}
	var i uint
	for i = 0; i < num2Return; i++ {
		hit = C.LucyHitsNext(hits)
		if hit == nil {
			break
		}
		docId := C.LucyHitDocGetDocId(hit)
		contentValue := cb_ptr2char(C.LucyHitDocExtract(hit, lContentField, nil)) // do i need to free this
		idValue := cb_ptr2char(C.LucyHitDocExtract(hit, lIdField, nil))           // do i need to free this
		results[i] = &SearchResult{
			Id:    C.GoString(idValue),
			Text:  C.GoString(contentValue),
			Score: float32(C.LucyHitDocGetScore(hit)),
		}
		if includeMatchedTerms {
			matchedTerms(docId, results[i])
		}
		C.DECREF(hit)
	}
	return totalNumHits, results
}

func set(vals []string) []string {
	s := make(map[string]bool)
	for _, val := range vals {
		s[strings.ToLower(val)] = true
	}
	retval := make([]string, len(s))
	i := 0
	for k, _ := range s {
		retval[i] = k
		i++
	}
	return retval
}

func (ixReader *IndexReader) Close() {
	C.DECREF(ixReader.lucySearcher)
}

func (query *Query) Close() {
	//	C.DECREF(query.lucySchema)
	C.DECREF(query.lucyQuery)
}
<commit_msg>removing unused struct field<commit_after>package golucy

import "runtime"

// Copyright 2013 Philip Southam
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
#include "Clownfish/Obj.h"
#define DECREF       cfish_Obj_decref
#define ObjToString CFISH_Obj_To_String

#include "Clownfish/Hash.h"
#define CfishHashKeys CFISH_Hash_Keys //returns cfish_Varray*

#include "Lucy/Search/IndexSearcher.h"
#define LucyIndexSearcher lucy_IndexSearcher
#define LucyIxSearcherNew lucy_IxSearcher_new
#define LucyIxSearcherHits LUCY_IxSearcher_Hits
#define LucyIxSearcherGetSchema LUCY_IxSearcher_Get_Schema
#define LucyIxSearchFetchDocVec LUCY_IxSearcher_Fetch_Doc_Vec

#include "Lucy/Analysis/EasyAnalyzer.h"
#define LucyEasyAnalyzerNew lucy_EasyAnalyzer_new

#include "Lucy/Plan/Schema.h"
#define LucySchema lucy_Schema
#define LucySchemaAllFields LUCY_Schema_All_Fields

#include "Lucy/Search/QueryParser.h"
#define LucyQueryParser lucy_QueryParser
#define LucyQParserNew lucy_QParser_new
#define LucyQParserParse LUCY_QParser_Parse

#include "Lucy/Document/HitDoc.h"
#define LucyHitDoc lucy_HitDoc
#define LucyHitDocExtract LUCY_HitDoc_Extract
#define LucyHitDocGetDocId  LUCY_HitDoc_Get_Doc_ID
#define LucyHitdocDump LUCY_HitDoc_Dump //returns cfish_Hash*
#define LucyHitDocGetScore LUCY_HitDoc_Get_Score

#include "Lucy/Search/Hits.h"
#define LucyHitsTotal LUCY_Hits_Total_Hits
#define LucyHitsNext LUCY_Hits_Next

#include "Lucy/Search/Query.h"
#define LucyQuery lucy_Query
#define LucyQueryMakeCompiler LUCY_Query_Make_Compiler

#include "Lucy/Search/Compiler.h"
#define LucyCompilerHighlightSpans LUCY_Compiler_Highlight_Spans

#include "Clownfish/VArray.h"
#define VaGetSize CFISH_VA_Get_Size
#define VaFetch CFISH_VA_Fetch

#include "Lucy/Search/Span.h"
#define LucySpanGetOffset LUCY_Span_Get_Offset
#define LucySpanGetLength LUCY_Span_Get_Length
*/
import "C"

import "strings"

type Query struct {
	QueryStr  string
	lucyQuery *C.LucyQuery
}

type SearchResult struct {
	Id           string
	Text         string
	Score        float32
	MatchedTerms []string
}

type IndexReader struct {
	Index        *Index
	lucySearcher *C.LucyIndexSearcher
}

func (index *Index) NewIndexReader() *IndexReader {
	ixLocation := cb_newf(index.Path)
	defer C.DECREF(ixLocation)
	return &IndexReader{Index: index, lucySearcher: C.LucyIxSearcherNew(ixLocation)}
}

func (ixReader *IndexReader) ParseQuery(queryStr string) *Query {
	lucySchema := C.LucyIxSearcherGetSchema(ixReader.lucySearcher)
	language := cb_newf("en") // should be configurable
	defer C.DECREF(language)
	analyzer := C.LucyEasyAnalyzerNew(language)
	defer C.DECREF(analyzer)
	qp := C.LucyQParserNew(
		lucySchema,
		analyzer,                          //should this be configurable?
		cb_newf("AND"),                    // should be configurable
		C.LucySchemaAllFields(lucySchema), // should be configurable
	)
	defer C.DECREF(qp)
	return &Query{
		QueryStr:  queryStr,
		lucyQuery: C.LucyQParserParse(qp, cb_new_from_utf8(queryStr)),
	}
}

func (ixReader *IndexReader) Search(query *Query, offset, limit uint, idField string, contentField string, includeMatchedTerms bool) (uint, []*SearchResult) {
	// Should probably have some sort
	// of `Results` object/iterator so that we don't have to specify
	// offset/limit and where I can attach matched terms to the result.
	lIdField, lContentField := cb_newf(idField), cb_newf(contentField) // total hack, need to return more than one field
	defer C.DECREF(lIdField)
	defer C.DECREF(lContentField)
	hits := C.LucyIxSearcherHits(ixReader.lucySearcher, query.lucyQuery, C.uint32_t(offset), C.uint32_t(limit), nil)
	defer C.DECREF(hits)
	totalNumHits := uint(C.LucyHitsTotal(hits))
	num2Return := minUInt(limit, totalNumHits)
	results := make([]*SearchResult, num2Return)
	var hit *C.LucyHitDoc
	compiler := C.LucyQueryMakeCompiler(query.lucyQuery, ixReader.lucySearcher, 1.0, false)
	defer C.DECREF(compiler)

	matchedTerms := func(docId C.int32_t, result *SearchResult) {
		docVec := C.LucyIxSearchFetchDocVec(ixReader.lucySearcher, docId)
		defer C.DECREF(docVec)
		spans := C.LucyCompilerHighlightSpans(compiler, ixReader.lucySearcher, docVec, lContentField)
		defer C.DECREF(spans)
		spanCnt := C.VaGetSize(spans)
		if spanCnt == 0 {
			// should never get here, but just in case...
			return
		}
		result.MatchedTerms = make([]string, spanCnt)
		var i C.uint32_t
		for i = 0; i < spanCnt; i++ {
			span := C.VaFetch(spans, i)
			offset := C.LucySpanGetOffset(span)
			length := C.LucySpanGetLength(span)
			result.MatchedTerms[i] = string([]rune(result.Text)[offset : offset+length])
		}
		// make terms unique?
		result.MatchedTerms = set(result.MatchedTerms)
	}
	var i uint
	for i = 0; i < num2Return; i++ {
		hit = C.LucyHitsNext(hits)
		if hit == nil {
			break
		}
		docId := C.LucyHitDocGetDocId(hit)
		contentValue := cb_ptr2char(C.LucyHitDocExtract(hit, lContentField, nil)) // do i need to free this
		idValue := cb_ptr2char(C.LucyHitDocExtract(hit, lIdField, nil))           // do i need to free this
		results[i] = &SearchResult{
			Id:    C.GoString(idValue),
			Text:  C.GoString(contentValue),
			Score: float32(C.LucyHitDocGetScore(hit)),
		}
		if includeMatchedTerms {
			matchedTerms(docId, results[i])
		}
		C.DECREF(hit)
	}
	return totalNumHits, results
}

func set(vals []string) []string {
	s := make(map[string]bool)
	for _, val := range vals {
		s[strings.ToLower(val)] = true
	}
	retval := make([]string, len(s))
	i := 0
	for k, _ := range s {
		retval[i] = k
		i++
	}
	return retval
}

func (ixReader *IndexReader) Close() {
	C.DECREF(ixReader.lucySearcher)
}

func (query *Query) Close() {
	//	C.DECREF(query.lucySchema)
	C.DECREF(query.lucyQuery)
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"strings"

	"helm.sh/helm/pkg/helmpath"
	"helm.sh/helm/pkg/helmpath/xdg"

	"helm.sh/helm/pkg/cli"

	"helm.sh/helm/pkg/plugin"

	"github.com/spf13/cobra"

	"helm.sh/helm/cmd/helm/require"
)

var (
	envHelp = `
Env prints out all the environment information in use by Helm.
`
)

func newEnvCmd(out io.Writer) *cobra.Command {
	o := &envOptions{}

	cmd := &cobra.Command{
		Use:   "env",
		Short: "environment information in use the Helm client",
		Long:  envHelp,
		Args:  require.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			return o.run(out)
		},
	}

	return cmd
}

type envOptions struct {
}

// run .
func (o *envOptions) run(out io.Writer) error {
	plugin.SetupPluginEnv(&cli.EnvSettings{}, "", "")
	o.setXdgPaths()
	o.setGoPaths()
	for _, prefix := range []string{
		"HELM_",
		"XDG_",
		"GO",
	} {
		for _, e := range os.Environ() {
			if strings.HasPrefix(e, prefix) {
				fmt.Println(e)
			}
		}
	}
	return nil
}

func (o *envOptions) setXdgPaths() {
	for key, val := range map[string]string{
		xdg.CacheHomeEnvVar:  helmpath.CachePath(),
		xdg.ConfigHomeEnvVar: helmpath.ConfigPath(),
		xdg.DataHomeEnvVar:   helmpath.DataPath(),
	} {
		if eVal := os.Getenv(key); len(eVal) <= 0 {
			os.Setenv(key, val)
		}
	}
}

func (o *envOptions) setGoPaths() {
	for key, val := range map[string]string{
		"GOPATH": "~/go",
		"GOBIN":  "~/go/bin",
	} {
		if eVal := os.Getenv(key); len(eVal) <= 0 {
			os.Setenv(key, val)
		}
	}
}
<commit_msg>Updated to get Helm env Paths and XDG env paths only<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"strings"

	"helm.sh/helm/pkg/helmpath"
	"helm.sh/helm/pkg/helmpath/xdg"

	"github.com/spf13/cobra"

	"helm.sh/helm/cmd/helm/require"
)

var (
	envHelp = `
Env prints out all the environment information in use by Helm.
`
)

func newEnvCmd(out io.Writer) *cobra.Command {
	o := &envOptions{}

	cmd := &cobra.Command{
		Use:   "env",
		Short: "Helm client environment information",
		Long:  envHelp,
		Args:  require.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			return o.run(out)
		},
	}

	return cmd
}

type envOptions struct {
}

func (o *envOptions) run(out io.Writer) error {
	o.setHelmPaths()
	o.setXdgPaths()
	for _, prefix := range []string{
		"HELM_",
		"XDG_",
	} {
		for _, e := range os.Environ() {
			if strings.HasPrefix(e, prefix) {
				fmt.Println(e)
			}
		}
	}
	return nil
}

func (o *envOptions) setHelmPaths() {
	for key, val := range map[string]string{
		"HELM_HOME":                  helmpath.DataPath(),
		"HELM_PATH_STARTER":          helmpath.DataPath("starters"),
		"HELM_DEBUG":                 fmt.Sprint(settings.Debug),
		"HELM_REGISTRY_CONFIG":       settings.RegistryConfig,
		"HELM_PATH_REPOSITORY_FILE":  settings.RepositoryConfig,
		"HELM_PATH_REPOSITORY_CACHE": settings.RepositoryCache,
		"HELM_PLUGIN":                settings.PluginsDirectory,
	} {
		if eVal := os.Getenv(key); len(eVal) <= 0 {
			os.Setenv(key, val)
		}
	}
}

func (o *envOptions) setXdgPaths() {
	for key, val := range map[string]string{
		xdg.CacheHomeEnvVar:  helmpath.CachePath(),
		xdg.ConfigHomeEnvVar: helmpath.ConfigPath(),
		xdg.DataHomeEnvVar:   helmpath.DataPath(),
	} {
		if eVal := os.Getenv(key); len(eVal) <= 0 {
			os.Setenv(key, val)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/akalin/gopar/par1"
	"github.com/akalin/gopar/par2"
)

type par1LogEncoderDelegate struct{}

func (par1LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogEncoderDelegate) OnVolumeFileWrite(i, n int, path string, dataByteCount, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing volume file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote volume file %q (%d data bytes, %d bytes)\n", i, n, path, dataByteCount, byteCount)
	}
}

type par1LogDecoderDelegate struct{}

func (par1LogDecoderDelegate) OnHeaderLoad(headerInfo string) {
	fmt.Printf("Loaded header: %s\n", headerInfo)
}

func (par1LogDecoderDelegate) OnFileEntryLoad(i, n int, filename, entryInfo string) {
	fmt.Printf("[%d/%d] Loaded entry for %q: %s\n", i, n, filename, entryInfo)
}

func (par1LogDecoderDelegate) OnCommentLoad(comment []byte) {
	fmt.Printf("Comment: %q\n", comment)
}

func (par1LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, corrupt bool, err error) {
	if err != nil {
		if corrupt {
			fmt.Printf("[%d/%d] Loading data file %q failed; marking as corrupt and skipping: %+v\n", i, n, path, err)
		} else {
			fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
		}
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogDecoderDelegate) OnVolumeFileLoad(i uint64, path string, storedSetHash, computedSetHash [16]byte, dataByteCount int, err error) {
	if os.IsNotExist(err) {
		// Do nothing.
	} else if err != nil {
		fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err)
	} else {
		fmt.Printf("[%d] Loaded volume file %q (%d data bytes)\n", i, path, dataByteCount)
		if storedSetHash != computedSetHash {
			fmt.Printf("[%d] Warning: stored set hash in %q %x doesn't match computed set hash %x\n", i, path, storedSetHash, computedSetHash)
		}
	}
}

type par2LogEncoderDelegate struct{}

func (par2LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par2LogEncoderDelegate) OnIndexFileWrite(path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("Writing index file %q failed: %+v\n", path, err)
	} else {
		fmt.Printf("Wrote index file %q (%d bytes)\n", path, byteCount)
	}
}

func (par2LogEncoderDelegate) OnRecoveryFileWrite(start, count, total int, path string, dataByteCount, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d+%d/%d] Writing recovery file %q failed: %+v\n", start, count, total, path, err)
	} else {
		fmt.Printf("[%d+%d/%d] Wrote recovery file %q (%d data bytes, %d bytes)\n", start, count, total, path, dataByteCount, byteCount)
	}
}

type par2LogDecoderDelegate struct{}

func (par2LogDecoderDelegate) OnCreatorPacketLoad(clientID string) {
	fmt.Printf("Loaded creator packet with client ID %q\n", clientID)
}

func (par2LogDecoderDelegate) OnMainPacketLoad(sliceByteCount, recoverySetCount, nonRecoverySetCount int) {
	fmt.Printf("Loaded main packet: slice byte count=%d, recovery set size=%d, non-recovery set size=%d\n", sliceByteCount, recoverySetCount, nonRecoverySetCount)
}

func (par2LogDecoderDelegate) OnFileDescriptionPacketLoad(fileID [16]byte, filename string, byteCount int) {
	fmt.Printf("Loaded file description packet for %q (ID=%x, %d bytes)\n", filename, fileID, byteCount)
}

func (par2LogDecoderDelegate) OnIFSCPacketLoad(fileID [16]byte) {
	fmt.Printf("Loaded checksums for file with ID %x\n", fileID)
}

func (par2LogDecoderDelegate) OnRecoveryPacketLoad(exponent uint16, byteCount int) {
	fmt.Printf("Loaded recovery packet: exponent=%d, byte count=%d\n", exponent, byteCount)
}

func (par2LogDecoderDelegate) OnUnknownPacketLoad(packetType [16]byte, byteCount int) {
	fmt.Printf("Loaded unknown packet of type %q and byte count %d\n", packetType, byteCount)
}

func (par2LogDecoderDelegate) OnOtherPacketSkip(setID [16]byte, packetType [16]byte, byteCount int) {
	fmt.Printf("Skipped packet with set ID %x of type %q and byte count %d\n", setID, packetType, byteCount)
}

func (par2LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par2LogDecoderDelegate) OnParityFileLoad(i int, path string, err error) {
	if err != nil {
		fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err)
	} else {
		fmt.Printf("[%d] Loaded volume file %q\n", i, path)
	}
}

func (par2LogDecoderDelegate) OnDetectCorruptDataChunk(fileID [16]byte, filename string, startByteOffset, endByteOffset int) {
	fmt.Printf("Corrupt data chunk: %q (ID %x), bytes %d to %d\n", filename, fileID, startByteOffset, endByteOffset-1)
}

func (par2LogDecoderDelegate) OnDetectDataFileHashMismatch(fileID [16]byte, filename string) {
	fmt.Printf("Hash mismatch for %q (ID %x)\n", filename, fileID)
}

func (par2LogDecoderDelegate) OnDetectDataFileWrongByteCount(fileID [16]byte, filename string) {
	fmt.Printf("Wrong byte count for %q (ID %x)\n", filename, fileID)
}

func (par2LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func printUsageAndExit(name string, flagSet *flag.FlagSet) {
	name = filepath.Base(name)
	fmt.Printf(`
Usage:
  %s c(reate) [options] <PAR file> [files]
  %s v(erify) [options] <PAR file>
  %s r(epair) [options] <PAR file>

Options:
`, name, name, name)
	flagSet.PrintDefaults()
	fmt.Printf("\n")
	os.Exit(2)
}

type encoder interface {
	LoadFileData() error
	ComputeParityData() error
	Write(string) error
}

type decoder interface {
	LoadFileData() error
	LoadParityData() error
	Verify() (bool, error)
	Repair() ([]string, error)
}

func newEncoder(parFile string, filePaths []string, numParityShards int) (encoder, error) {
	// TODO: Detect file type more robustly.
	ext := path.Ext(parFile)
	// TODO: Make this configurable.
	const sliceByteCount = 2000
	if ext == ".par2" {
		return par2.NewEncoder(par2LogEncoderDelegate{}, filePaths, sliceByteCount, numParityShards)
	}
	return par1.NewEncoder(par1LogEncoderDelegate{}, filePaths, numParityShards)
}

func newDecoder(parFile string) (decoder, error) {
	// TODO: Detect file type more robustly.
	ext := path.Ext(parFile)
	if ext == ".par2" {
		return par2.NewDecoder(par2LogDecoderDelegate{}, parFile)
	}
	return par1.NewDecoder(par1LogDecoderDelegate{}, parFile)
}

func main() {
	name := os.Args[0]
	flagSet := flag.NewFlagSet(name, flag.ExitOnError)
	flagSet.SetOutput(os.Stdout)
	usage := flagSet.Bool("h", false, "print usage info")
	numParityShards := flagSet.Int("c", 3, "number of recovery blocks to create (or files, for PAR1)")
	flagSet.Parse(os.Args[1:])

	if flagSet.NArg() < 2 || *usage {
		printUsageAndExit(name, flagSet)
	}

	cmd := flagSet.Arg(0)
	parFile := flagSet.Arg(1)

	switch strings.ToLower(cmd) {
	case "c":
		fallthrough
	case "create":
		if flagSet.NArg() == 2 {
			printUsageAndExit(name, flagSet)
		}

		encoder, err := newEncoder(parFile, flagSet.Args()[2:], *numParityShards)
		if err != nil {
			panic(err)
		}

		err = encoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = encoder.ComputeParityData()
		if err != nil {
			panic(err)
		}

		err = encoder.Write(parFile)
		if err != nil {
			fmt.Printf("Write parity error: %s", err)
			os.Exit(-1)
		}

	case "v":
		fallthrough
	case "verify":
		decoder, err := newDecoder(parFile)
		if err != nil {
			panic(err)
		}

		err = decoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = decoder.LoadParityData()
		if err != nil {
			panic(err)
		}

		ok, err := decoder.Verify()
		if err != nil {
			panic(err)
		}

		fmt.Printf("Verify result: %t\n", ok)
		if !ok {
			os.Exit(-1)
		}

	case "r":
		fallthrough
	case "repair":
		decoder, err := newDecoder(parFile)
		if err != nil {
			panic(err)
		}

		err = decoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = decoder.LoadParityData()
		if err != nil {
			panic(err)
		}

		repairedFiles, err := decoder.Repair()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Repair error: %s\n", err)
			os.Exit(-1)
		}

		fmt.Printf("Repaired files: %v\n", repairedFiles)
		if err != nil {
			os.Exit(-1)
		}

	default:
		printUsageAndExit(name, flagSet)
	}
}
<commit_msg>Add switch for CPU profile<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime/pprof"
	"strings"

	"github.com/akalin/gopar/par1"
	"github.com/akalin/gopar/par2"
)

type par1LogEncoderDelegate struct{}

func (par1LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogEncoderDelegate) OnVolumeFileWrite(i, n int, path string, dataByteCount, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing volume file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote volume file %q (%d data bytes, %d bytes)\n", i, n, path, dataByteCount, byteCount)
	}
}

type par1LogDecoderDelegate struct{}

func (par1LogDecoderDelegate) OnHeaderLoad(headerInfo string) {
	fmt.Printf("Loaded header: %s\n", headerInfo)
}

func (par1LogDecoderDelegate) OnFileEntryLoad(i, n int, filename, entryInfo string) {
	fmt.Printf("[%d/%d] Loaded entry for %q: %s\n", i, n, filename, entryInfo)
}

func (par1LogDecoderDelegate) OnCommentLoad(comment []byte) {
	fmt.Printf("Comment: %q\n", comment)
}

func (par1LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, corrupt bool, err error) {
	if err != nil {
		if corrupt {
			fmt.Printf("[%d/%d] Loading data file %q failed; marking as corrupt and skipping: %+v\n", i, n, path, err)
		} else {
			fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
		}
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par1LogDecoderDelegate) OnVolumeFileLoad(i uint64, path string, storedSetHash, computedSetHash [16]byte, dataByteCount int, err error) {
	if os.IsNotExist(err) {
		// Do nothing.
	} else if err != nil {
		fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err)
	} else {
		fmt.Printf("[%d] Loaded volume file %q (%d data bytes)\n", i, path, dataByteCount)
		if storedSetHash != computedSetHash {
			fmt.Printf("[%d] Warning: stored set hash in %q %x doesn't match computed set hash %x\n", i, path, storedSetHash, computedSetHash)
		}
	}
}

type par2LogEncoderDelegate struct{}

func (par2LogEncoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par2LogEncoderDelegate) OnIndexFileWrite(path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("Writing index file %q failed: %+v\n", path, err)
	} else {
		fmt.Printf("Wrote index file %q (%d bytes)\n", path, byteCount)
	}
}

func (par2LogEncoderDelegate) OnRecoveryFileWrite(start, count, total int, path string, dataByteCount, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d+%d/%d] Writing recovery file %q failed: %+v\n", start, count, total, path, err)
	} else {
		fmt.Printf("[%d+%d/%d] Wrote recovery file %q (%d data bytes, %d bytes)\n", start, count, total, path, dataByteCount, byteCount)
	}
}

type par2LogDecoderDelegate struct{}

func (par2LogDecoderDelegate) OnCreatorPacketLoad(clientID string) {
	fmt.Printf("Loaded creator packet with client ID %q\n", clientID)
}

func (par2LogDecoderDelegate) OnMainPacketLoad(sliceByteCount, recoverySetCount, nonRecoverySetCount int) {
	fmt.Printf("Loaded main packet: slice byte count=%d, recovery set size=%d, non-recovery set size=%d\n", sliceByteCount, recoverySetCount, nonRecoverySetCount)
}

func (par2LogDecoderDelegate) OnFileDescriptionPacketLoad(fileID [16]byte, filename string, byteCount int) {
	fmt.Printf("Loaded file description packet for %q (ID=%x, %d bytes)\n", filename, fileID, byteCount)
}

func (par2LogDecoderDelegate) OnIFSCPacketLoad(fileID [16]byte) {
	fmt.Printf("Loaded checksums for file with ID %x\n", fileID)
}

func (par2LogDecoderDelegate) OnRecoveryPacketLoad(exponent uint16, byteCount int) {
	fmt.Printf("Loaded recovery packet: exponent=%d, byte count=%d\n", exponent, byteCount)
}

func (par2LogDecoderDelegate) OnUnknownPacketLoad(packetType [16]byte, byteCount int) {
	fmt.Printf("Loaded unknown packet of type %q and byte count %d\n", packetType, byteCount)
}

func (par2LogDecoderDelegate) OnOtherPacketSkip(setID [16]byte, packetType [16]byte, byteCount int) {
	fmt.Printf("Skipped packet with set ID %x of type %q and byte count %d\n", setID, packetType, byteCount)
}

func (par2LogDecoderDelegate) OnDataFileLoad(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Loading data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Loaded data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func (par2LogDecoderDelegate) OnParityFileLoad(i int, path string, err error) {
	if err != nil {
		fmt.Printf("[%d] Loading volume file %q failed: %+v\n", i, path, err)
	} else {
		fmt.Printf("[%d] Loaded volume file %q\n", i, path)
	}
}

func (par2LogDecoderDelegate) OnDetectCorruptDataChunk(fileID [16]byte, filename string, startByteOffset, endByteOffset int) {
	fmt.Printf("Corrupt data chunk: %q (ID %x), bytes %d to %d\n", filename, fileID, startByteOffset, endByteOffset-1)
}

func (par2LogDecoderDelegate) OnDetectDataFileHashMismatch(fileID [16]byte, filename string) {
	fmt.Printf("Hash mismatch for %q (ID %x)\n", filename, fileID)
}

func (par2LogDecoderDelegate) OnDetectDataFileWrongByteCount(fileID [16]byte, filename string) {
	fmt.Printf("Wrong byte count for %q (ID %x)\n", filename, fileID)
}

func (par2LogDecoderDelegate) OnDataFileWrite(i, n int, path string, byteCount int, err error) {
	if err != nil {
		fmt.Printf("[%d/%d] Writing data file %q failed: %+v\n", i, n, path, err)
	} else {
		fmt.Printf("[%d/%d] Wrote data file %q (%d bytes)\n", i, n, path, byteCount)
	}
}

func printUsageAndExit(name string, flagSet *flag.FlagSet) {
	name = filepath.Base(name)
	fmt.Printf(`
Usage:
  %s c(reate) [options] <PAR file> [files]
  %s v(erify) [options] <PAR file>
  %s r(epair) [options] <PAR file>

Options:
`, name, name, name)
	flagSet.PrintDefaults()
	fmt.Printf("\n")
	os.Exit(2)
}

type encoder interface {
	LoadFileData() error
	ComputeParityData() error
	Write(string) error
}

type decoder interface {
	LoadFileData() error
	LoadParityData() error
	Verify() (bool, error)
	Repair() ([]string, error)
}

func newEncoder(parFile string, filePaths []string, numParityShards int) (encoder, error) {
	// TODO: Detect file type more robustly.
	ext := path.Ext(parFile)
	// TODO: Make this configurable.
	const sliceByteCount = 2000
	if ext == ".par2" {
		return par2.NewEncoder(par2LogEncoderDelegate{}, filePaths, sliceByteCount, numParityShards)
	}
	return par1.NewEncoder(par1LogEncoderDelegate{}, filePaths, numParityShards)
}

func newDecoder(parFile string) (decoder, error) {
	// TODO: Detect file type more robustly.
	ext := path.Ext(parFile)
	if ext == ".par2" {
		return par2.NewDecoder(par2LogDecoderDelegate{}, parFile)
	}
	return par1.NewDecoder(par1LogDecoderDelegate{}, parFile)
}

func main() {
	name := os.Args[0]
	flagSet := flag.NewFlagSet(name, flag.ExitOnError)
	flagSet.SetOutput(os.Stdout)
	usage := flagSet.Bool("h", false, "print usage info")
	cpuprofile := flagSet.String("cpuprofile", "", "if non-empty, where to write the CPU profile")
	numParityShards := flagSet.Int("c", 3, "number of recovery blocks to create (or files, for PAR1)")
	flagSet.Parse(os.Args[1:])

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		defer f.Close()

		err = pprof.StartCPUProfile(f)
		if err != nil {
			panic(err)
		}
		defer pprof.StopCPUProfile()
	}

	if flagSet.NArg() < 2 || *usage {
		printUsageAndExit(name, flagSet)
	}

	cmd := flagSet.Arg(0)
	parFile := flagSet.Arg(1)

	switch strings.ToLower(cmd) {
	case "c":
		fallthrough
	case "create":
		if flagSet.NArg() == 2 {
			printUsageAndExit(name, flagSet)
		}

		encoder, err := newEncoder(parFile, flagSet.Args()[2:], *numParityShards)
		if err != nil {
			panic(err)
		}

		err = encoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = encoder.ComputeParityData()
		if err != nil {
			panic(err)
		}

		err = encoder.Write(parFile)
		if err != nil {
			fmt.Printf("Write parity error: %s", err)
			os.Exit(-1)
		}

	case "v":
		fallthrough
	case "verify":
		decoder, err := newDecoder(parFile)
		if err != nil {
			panic(err)
		}

		err = decoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = decoder.LoadParityData()
		if err != nil {
			panic(err)
		}

		ok, err := decoder.Verify()
		if err != nil {
			panic(err)
		}

		fmt.Printf("Verify result: %t\n", ok)
		if !ok {
			os.Exit(-1)
		}

	case "r":
		fallthrough
	case "repair":
		decoder, err := newDecoder(parFile)
		if err != nil {
			panic(err)
		}

		err = decoder.LoadFileData()
		if err != nil {
			panic(err)
		}

		err = decoder.LoadParityData()
		if err != nil {
			panic(err)
		}

		repairedFiles, err := decoder.Repair()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Repair error: %s\n", err)
			os.Exit(-1)
		}

		fmt.Printf("Repaired files: %v\n", repairedFiles)
		if err != nil {
			os.Exit(-1)
		}

	default:
		printUsageAndExit(name, flagSet)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"

	"github.com/maguro/pbf/cmd/pbf/cli"
	_ "github.com/maguro/pbf/cmd/pbf/extract"
	_ "github.com/maguro/pbf/cmd/pbf/info"
)

func main() {
	if err := cli.RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Remove extraneous import<commit_after>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"

	"github.com/maguro/pbf/cmd/pbf/cli"
	_ "github.com/maguro/pbf/cmd/pbf/info"
)

func main() {
	if err := cli.RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"github.com/polydawn/go-errcat"
	"github.com/polydawn/refmt"
	"github.com/polydawn/refmt/json"
	"gopkg.in/alecthomas/kingpin.v2"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	tar "go.polydawn.net/rio/transmat/tar"
)

/*
	Output serialization formats
*/
const (
	FmtJson = "json"
	FmtDumb = "dumb"
)

type baseCLI struct {
	Deadline       string        // Deadline time (RFC3339 or @UNIX)
	Format         string        // Output api format, eg. json
	ProgressEnable bool          // Emit progress notification yes/no
	ProgressRate   time.Duration // How frequently to emit progress notification
	Test           string        // Testmode
	Timeout        time.Duration // Timeout duration (exclusive with deadline) eg. "60s"
	PackCLI        struct {
		Filters             api.FilesetFilters // TODO: file filters for pack/unpack
		Path                string             // Pack target path
		TargetWarehouseAddr string             // Warehouse address to push to
	}
	UnpackCLI struct {
		Filters              api.FilesetFilters
		Path                 string   // Unpack target path
		SourcesWarehouseAddr []string // Warehouse address to push to
		WareID               string   // Ware id string "<kind>:<hash>"
	}
	MirrorCLI struct {
		SourcesWarehouseAddr []string // Warehouse addresses to fetch from
		TargetWarehouseAddr  string   // Warehouse address to push to
		WareID               string   // Ware id string "<kind>:<hash>"
	}
}

func configurePack(cli *baseCLI, appPack *kingpin.CmdClause) {
	// Non-filter flags
	appPack.Arg("path", "Target path").
		Required().
		StringVar(&cli.PackCLI.Path)
	appPack.Flag("target", "Warehouse in which to place the ware").
		StringVar(&cli.PackCLI.TargetWarehouseAddr)

	// Filter flags
	appPack.Flag("uid", "Set UID filter [keep, <int>]").
		StringVar(&cli.PackCLI.Filters.Uid)
	appPack.Flag("gid", "Set GID filter [keep, <int>]").
		StringVar(&cli.PackCLI.Filters.Gid)
	appPack.Flag("mtime", "Set mtime filter [keep, <@UNIX>, <RFC3339>]. Will be set to a date if not specified.").
		StringVar(&cli.PackCLI.Filters.Mtime)
	// Sticky flag not used for pack
}

func configureUnpack(cli *baseCLI, appUnpack *kingpin.CmdClause) {
	// Non-filter flags
	appUnpack.Arg("path", "Target path").
		Required().
		StringVar(&cli.UnpackCLI.Path)
	appUnpack.Arg("ware", "Ware ID").
		Required().
		StringVar(&cli.UnpackCLI.WareID)
	appUnpack.Flag("source", "Warehouses from which to fetch the ware").
		StringsVar(&cli.UnpackCLI.SourcesWarehouseAddr)

	// Filter flags
	appUnpack.Flag("uid", "Set UID filter [keep, mine, <int>]").
		Default("mine").
		StringVar(&cli.UnpackCLI.Filters.Uid)
	appUnpack.Flag("gid", "Set GID filter [keep, mine, <int>]").
		Default("mine").
		StringVar(&cli.UnpackCLI.Filters.Gid)
	appUnpack.Flag("mtime", "Set mtime filter [keep, <@UNIX>, <RFC3339>]").
		Default("keep").
		StringVar(&cli.UnpackCLI.Filters.Mtime)
	appUnpack.Flag("sticky", "Keep setuid, setgid, and sticky bits").
		BoolVar(&cli.UnpackCLI.Filters.Sticky)
}

func configureMirror(cli *baseCLI, appMirror *kingpin.CmdClause) {
	appMirror.Arg("ware", "Ware ID").
		Required().
		StringVar(&cli.MirrorCLI.WareID)
	appMirror.Arg("target", "Warehouse in which to place the ware").
		Required().
		StringVar(&cli.MirrorCLI.TargetWarehouseAddr)
	appMirror.Flag("source", "Warehouses from which to fetch the ware").
		StringsVar(&cli.MirrorCLI.SourcesWarehouseAddr)
}

/*
	Blocks until a sigint is received, then calls cancel.
*/
func CancelOnInterrupt(cancel context.CancelFunc) {
	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, os.Interrupt)
	<-signalChan
	cancel()
	close(signalChan)
}

func main() {
	ctx := context.Background()
	exitCode := Main(ctx, os.Args, os.Stdin, os.Stdout, os.Stderr)
	os.Exit(int(exitCode))
}

func Main(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) rio.ExitCode {
	exitCode := rio.ExitSuccess
	ctx, cancel := context.WithCancel(ctx)
	go CancelOnInterrupt(cancel)

	cli := baseCLI{}

	app := kingpin.New("rio", "Repeatable I/O")
	app.HelpFlag.Short('h')

	app.UsageWriter(stderr)
	app.ErrorWriter(stderr)

	app.Flag("deadline", "Deadline (RFC3339)").
		StringVar(&cli.Deadline)
	app.Flag("timeout", "Timeout for command").
		DurationVar(&cli.Timeout)
	app.Flag("format", "Output api format").
		Default(FmtDumb).
		EnumVar(&cli.Format, FmtJson, FmtDumb)
	app.Flag("progress-rate", "How frequently to emit progress notification").
		DurationVar(&cli.ProgressRate)
	app.Flag("progress", "Emit progress notification").
		BoolVar(&cli.ProgressEnable)
	app.Flag("test", "Testmodes").
		StringVar(&cli.Test)

	appPack := app.Command("pack", "pack a fileset into a ware")
	configurePack(&cli, appPack)

	appUnpack := app.Command("unpack", "fetch a ware")
	configureUnpack(&cli, appUnpack)

	appMirror := app.Command("mirror", "mirror a ware to another warehouse")
	configureUnpack(&cli, appMirror)

	var termErr error
	app.Terminate(func(status int) {
		termErr = fmt.Errorf("parsing error: %d\n", status)
	})
	cmd, err := app.Parse(args[1:])
	if err != nil {
		fmt.Fprintln(stderr, err)
		return rio.ExitUsage
	}
	if termErr != nil {
		fmt.Fprintln(stderr, termErr)
		return rio.ExitUsage
	}
	var wareID api.WareID
	// FIXME: We'll need to support more than tar eventually
	switch cmd {
	case appPack.FullCommand():
		wareID, err = executePack(ctx, cli, rio.PackFunc(tar.Pack))
		SerializeResult(cli.Format, wareID, err, stdout, stderr)
	case appUnpack.FullCommand():
		wareID, err = executeUnpack(ctx, cli, rio.UnpackFunc(tar.Unpack))
		SerializeResult(cli.Format, wareID, err, stdout, stderr)
	case appMirror.FullCommand():
		fmt.Fprintln(stderr, "not implemented")
		return rio.ExitNotImplemented
	}

	return exitCode
}

func SerializeResult(format string, wareID api.WareID, resultErr error, stdout io.Writer, stderr io.Writer) {
	var serialErr *rio.Error
	if resultErr != nil {
		serialErr = &rio.Error{}
		if e2, ok := resultErr.(errcat.Error); ok {
			serialErr.Category_ = errcat.Category(resultErr).(rio.ErrorCategory)
			serialErr.Message_ = e2.Message()
			serialErr.Details_ = e2.Details()
		} else {
			serialErr.Category_ = rio.ErrRPCBreakdown // :/
			serialErr.Message_ = resultErr.Error()
		}
	}
	ev := rio.Event{Result: &rio.Event_Result{
		WareID: wareID,
		Error:  serialErr,
	}}
	switch format {
	case FmtJson:
		marshaller := refmt.NewMarshallerAtlased(json.EncodeOptions{}, stdout, rio.Atlas)
		err := marshaller.Marshal(&ev)
		if err != nil {
			panic(err)
		}
	case FmtDumb:
		if resultErr != nil {
			fmt.Fprintln(stderr, resultErr)
		} else {
			fmt.Fprintln(stdout, wareID)
		}
	default:
		panic(fmt.Errorf("rio: invalid format %s", format))
	}
}

func convertWarehouseSlice(slice []string) []api.WarehouseAddr {
	result := make([]api.WarehouseAddr, len(slice))
	for idx, item := range slice {
		result[idx] = api.WarehouseAddr(item)
	}
	return result
}

func executeUnpack(ctx context.Context, cli baseCLI, unpackFn rio.UnpackFunc) (api.WareID, error) {
	monitor := rio.Monitor{}
	wareID, err := api.ParseWareID(cli.UnpackCLI.WareID)
	if err != nil {
		return api.WareID{}, err
	}
	path := cli.UnpackCLI.Path
	warehouses := convertWarehouseSlice(cli.UnpackCLI.SourcesWarehouseAddr)
	return unpackFn(ctx, wareID, path, cli.UnpackCLI.Filters, warehouses, monitor)
}

func executePack(ctx context.Context, cli baseCLI, packFn rio.PackFunc) (api.WareID, error) {
	monitor := rio.Monitor{}
	warehouse := api.WarehouseAddr(cli.PackCLI.TargetWarehouseAddr)

	return packFn(ctx, cli.PackCLI.Path, cli.PackCLI.Filters, warehouse, monitor)
}
<commit_msg>api/rio: switch to using the API's extracted error setter method.<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"github.com/polydawn/refmt"
	"github.com/polydawn/refmt/json"
	"gopkg.in/alecthomas/kingpin.v2"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	tar "go.polydawn.net/rio/transmat/tar"
)

/*
	Output serialization formats
*/
const (
	FmtJson = "json"
	FmtDumb = "dumb"
)

type baseCLI struct {
	Deadline       string        // Deadline time (RFC3339 or @UNIX)
	Format         string        // Output api format, eg. json
	ProgressEnable bool          // Emit progress notification yes/no
	ProgressRate   time.Duration // How frequently to emit progress notification
	Test           string        // Testmode
	Timeout        time.Duration // Timeout duration (exclusive with deadline) eg. "60s"
	PackCLI        struct {
		Filters             api.FilesetFilters // TODO: file filters for pack/unpack
		Path                string             // Pack target path
		TargetWarehouseAddr string             // Warehouse address to push to
	}
	UnpackCLI struct {
		Filters              api.FilesetFilters
		Path                 string   // Unpack target path
		SourcesWarehouseAddr []string // Warehouse address to push to
		WareID               string   // Ware id string "<kind>:<hash>"
	}
	MirrorCLI struct {
		SourcesWarehouseAddr []string // Warehouse addresses to fetch from
		TargetWarehouseAddr  string   // Warehouse address to push to
		WareID               string   // Ware id string "<kind>:<hash>"
	}
}

func configurePack(cli *baseCLI, appPack *kingpin.CmdClause) {
	// Non-filter flags
	appPack.Arg("path", "Target path").
		Required().
		StringVar(&cli.PackCLI.Path)
	appPack.Flag("target", "Warehouse in which to place the ware").
		StringVar(&cli.PackCLI.TargetWarehouseAddr)

	// Filter flags
	appPack.Flag("uid", "Set UID filter [keep, <int>]").
		StringVar(&cli.PackCLI.Filters.Uid)
	appPack.Flag("gid", "Set GID filter [keep, <int>]").
		StringVar(&cli.PackCLI.Filters.Gid)
	appPack.Flag("mtime", "Set mtime filter [keep, <@UNIX>, <RFC3339>]. Will be set to a date if not specified.").
		StringVar(&cli.PackCLI.Filters.Mtime)
	// Sticky flag not used for pack
}

func configureUnpack(cli *baseCLI, appUnpack *kingpin.CmdClause) {
	// Non-filter flags
	appUnpack.Arg("path", "Target path").
		Required().
		StringVar(&cli.UnpackCLI.Path)
	appUnpack.Arg("ware", "Ware ID").
		Required().
		StringVar(&cli.UnpackCLI.WareID)
	appUnpack.Flag("source", "Warehouses from which to fetch the ware").
		StringsVar(&cli.UnpackCLI.SourcesWarehouseAddr)

	// Filter flags
	appUnpack.Flag("uid", "Set UID filter [keep, mine, <int>]").
		Default("mine").
		StringVar(&cli.UnpackCLI.Filters.Uid)
	appUnpack.Flag("gid", "Set GID filter [keep, mine, <int>]").
		Default("mine").
		StringVar(&cli.UnpackCLI.Filters.Gid)
	appUnpack.Flag("mtime", "Set mtime filter [keep, <@UNIX>, <RFC3339>]").
		Default("keep").
		StringVar(&cli.UnpackCLI.Filters.Mtime)
	appUnpack.Flag("sticky", "Keep setuid, setgid, and sticky bits").
		BoolVar(&cli.UnpackCLI.Filters.Sticky)
}

func configureMirror(cli *baseCLI, appMirror *kingpin.CmdClause) {
	appMirror.Arg("ware", "Ware ID").
		Required().
		StringVar(&cli.MirrorCLI.WareID)
	appMirror.Arg("target", "Warehouse in which to place the ware").
		Required().
		StringVar(&cli.MirrorCLI.TargetWarehouseAddr)
	appMirror.Flag("source", "Warehouses from which to fetch the ware").
		StringsVar(&cli.MirrorCLI.SourcesWarehouseAddr)
}

/*
	Blocks until a sigint is received, then calls cancel.
*/
func CancelOnInterrupt(cancel context.CancelFunc) {
	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, os.Interrupt)
	<-signalChan
	cancel()
	close(signalChan)
}

func main() {
	ctx := context.Background()
	exitCode := Main(ctx, os.Args, os.Stdin, os.Stdout, os.Stderr)
	os.Exit(int(exitCode))
}

func Main(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) rio.ExitCode {
	exitCode := rio.ExitSuccess
	ctx, cancel := context.WithCancel(ctx)
	go CancelOnInterrupt(cancel)

	cli := baseCLI{}

	app := kingpin.New("rio", "Repeatable I/O")
	app.HelpFlag.Short('h')

	app.UsageWriter(stderr)
	app.ErrorWriter(stderr)

	app.Flag("deadline", "Deadline (RFC3339)").
		StringVar(&cli.Deadline)
	app.Flag("timeout", "Timeout for command").
		DurationVar(&cli.Timeout)
	app.Flag("format", "Output api format").
		Default(FmtDumb).
		EnumVar(&cli.Format, FmtJson, FmtDumb)
	app.Flag("progress-rate", "How frequently to emit progress notification").
		DurationVar(&cli.ProgressRate)
	app.Flag("progress", "Emit progress notification").
		BoolVar(&cli.ProgressEnable)
	app.Flag("test", "Testmodes").
		StringVar(&cli.Test)

	appPack := app.Command("pack", "pack a fileset into a ware")
	configurePack(&cli, appPack)

	appUnpack := app.Command("unpack", "fetch a ware")
	configureUnpack(&cli, appUnpack)

	appMirror := app.Command("mirror", "mirror a ware to another warehouse")
	configureUnpack(&cli, appMirror)

	var termErr error
	app.Terminate(func(status int) {
		termErr = fmt.Errorf("parsing error: %d\n", status)
	})
	cmd, err := app.Parse(args[1:])
	if err != nil {
		fmt.Fprintln(stderr, err)
		return rio.ExitUsage
	}
	if termErr != nil {
		fmt.Fprintln(stderr, termErr)
		return rio.ExitUsage
	}
	var wareID api.WareID
	// FIXME: We'll need to support more than tar eventually
	switch cmd {
	case appPack.FullCommand():
		wareID, err = executePack(ctx, cli, rio.PackFunc(tar.Pack))
		SerializeResult(cli.Format, wareID, err, stdout, stderr)
	case appUnpack.FullCommand():
		wareID, err = executeUnpack(ctx, cli, rio.UnpackFunc(tar.Unpack))
		SerializeResult(cli.Format, wareID, err, stdout, stderr)
	case appMirror.FullCommand():
		fmt.Fprintln(stderr, "not implemented")
		return rio.ExitNotImplemented
	}

	return exitCode
}

func SerializeResult(format string, wareID api.WareID, resultErr error, stdout io.Writer, stderr io.Writer) {
	result := &rio.Event_Result{
		WareID: wareID,
	}
	result.SetError(resultErr)
	ev := rio.Event{Result: result}
	switch format {
	case FmtJson:
		marshaller := refmt.NewMarshallerAtlased(json.EncodeOptions{}, stdout, rio.Atlas)
		err := marshaller.Marshal(&ev)
		if err != nil {
			panic(err)
		}
	case FmtDumb:
		if resultErr != nil {
			fmt.Fprintln(stderr, resultErr)
		} else {
			fmt.Fprintln(stdout, wareID)
		}
	default:
		panic(fmt.Errorf("rio: invalid format %s", format))
	}
}

func convertWarehouseSlice(slice []string) []api.WarehouseAddr {
	result := make([]api.WarehouseAddr, len(slice))
	for idx, item := range slice {
		result[idx] = api.WarehouseAddr(item)
	}
	return result
}

func executeUnpack(ctx context.Context, cli baseCLI, unpackFn rio.UnpackFunc) (api.WareID, error) {
	monitor := rio.Monitor{}
	wareID, err := api.ParseWareID(cli.UnpackCLI.WareID)
	if err != nil {
		return api.WareID{}, err
	}
	path := cli.UnpackCLI.Path
	warehouses := convertWarehouseSlice(cli.UnpackCLI.SourcesWarehouseAddr)
	return unpackFn(ctx, wareID, path, cli.UnpackCLI.Filters, warehouses, monitor)
}

func executePack(ctx context.Context, cli baseCLI, packFn rio.PackFunc) (api.WareID, error) {
	monitor := rio.Monitor{}
	warehouse := api.WarehouseAddr(cli.PackCLI.TargetWarehouseAddr)

	return packFn(ctx, cli.PackCLI.Path, cli.PackCLI.Filters, warehouse, monitor)
}
<|endoftext|>