<commit_before>package main

import (
	"conui"
	"core"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
)

func NewSyncCommand() cli.Command {
	return cli.Command{
		Name:  "sync",
		Usage: "Synchronize files to devices",
		Action: func(c *cli.Context) {
			err := checkEnvVariables(c)
			if err != nil {
				panic(fatal{fmt.Sprintf("Could not set environment variables: %s", err)})
			}
			if !c.GlobalBool("no-file-log") {
				lp := cleanPath(c.GlobalString("log"))
				var err error
				GDS_LOG_FD, err = os.Create(lp)
				if err != nil {
					panic(fatal{fmt.Sprintf("Could not create log file: %s", err)})
				}
				log.Out = GDS_LOG_FD
			}
			lvl, err := logrus.ParseLevel(c.GlobalString("log-level"))
			if err != nil {
				panic(fatalShowHelp{fmt.Sprintf("Error parsing log level: %s", err)})
			}
			log.Level = lvl
			syncStart(c)
		},
	}
}

// loadInitialState prepares the applicaton for usage
func loadInitialState(c *cli.Context) *core.Context {
	cPath, err := getConfigFile(c.GlobalString("config"))
	if err != nil {
		panic(fatal{err})
	}
	log.WithFields(logrus.Fields{
		"path": cPath,
	}).Info("Using configuration file")

	c2, err := core.ContextFromPath(cPath)
	if err != nil {
		panic(fatal{fmt.Sprintf("Error loading config: %s", err.Error())})
	}

	c2.Files, err = core.NewFileList(c2)
	if err != nil {
		panic(fatal{fmt.Sprintf("Error retrieving FileList %s", err.Error())})
	}

	c2.Catalog, err = core.NewCatalog(c2)
	if err != nil {
		// Not wrapped in fatal here because NewCatalog returns custom error types
		panic(err)
	}

	return c2
}

func dumpContextToFile(c *cli.Context, c2 *core.Context) {
	cf, err := getContextFile(c.GlobalString("context"))
	if err != nil {
		panic(fatal{fmt.Sprintf("Could not create context JSON output file: %s", err.Error())})
	}
	j, err := json.Marshal(c2)
	if err == nil {
		err = ioutil.WriteFile(cf, j, 0644)
	}
	if err != nil {
		panic(fatal{fmt.Sprintf("Could not marshal JSON to file: %s", err.Error())})
	}
}

// BuildConsole creates the UI widgets First is the main progress guage for the overall progress Widgets are then created for
// each of the devices, but are hidden initially.
func BuildConsole(c *core.Context) {
	visible := c.OutputStreamNum
	var panel []conui.Widget
	for x, y := range c.Devices {
		panel = append(panel, conui.NewDevicePanel(y.Name, y.SizeTotal))
		if visible > 0 {
			log.Debugln("Making device", x, "visible")
			panel[x].SetVisible(true)
			if x == 0 {
				panel[x].SetSelected(true)
			}
			visible--
		}
	}
	panel[len(c.Devices)-1] = conui.NewProgressGauge(c.Catalog.TotalSize())
	panel[len(c.Devices)-1].SetVisible(true)
	conui.Body.AddRows(panel[len(c.Devices)-1])
	for x, _ := range c.Devices {
		conui.Body.AddRows(panel[x])
	}
	conui.Layout()
}

func eventListener(c *core.Context) {
	defer cleanupAtExit()
	for {
		select {
		case e := <-conui.Events:
			if e.Type == conui.EventKey && e.Ch == 'j' {
				conui.Body.SelectNext()
			}
			if e.Type == conui.EventKey && e.Ch == 'k' {
				conui.Body.SelectPrevious()
			}
			if e.Type == conui.EventKey && e.Ch == 'd' {
				p := conui.Body.Selected()
				p.SetVisible(false)
			}
			if e.Type == conui.EventKey && e.Ch == 's' {
				p := conui.Body.Selected()
				p.SetVisible(true)
			}
			if e.Type == conui.EventKey && e.Key == conui.KeyEnter {
				p := conui.Body.Selected().Prompt()
				if p != nil {
					p.Action()
				}
			}
			if e.Type == conui.EventKey && e.Ch == 'q' {
				conui.Close()
				os.Exit(0)
			}
			if e.Type == conui.EventResize {
				// conui.Body.Width = conui.TermWidth()
				conui.Layout()
				go func() { conui.Redraw <- true }()
			}
		case <-conui.Redraw:
			conui.Render()
		}
	}
}

// deviceMountHandler checks to see if the device is mounted and writable. Meant to be run as a goroutine.
func deviceMountHandler(c *core.Context, deviceIndex int) {
	// Listen on the channel for a mount request
	ns := time.Now()
	log.Debugf("Waiting for receive on SyncDeviceMount[%d]", deviceIndex)
	<-c.SyncDeviceMount[deviceIndex]
	log.Debugf("Receive from SyncDeviceMount[%d] after wait of %s", deviceIndex, time.Since(ns))

	// Get the panel widget so we can write messages for the user to it
	wg := conui.Body.Select(deviceIndex)
	d, err := c.Devices.DeviceByName(wg.Border.Label)
	if err != nil {
		log.Error(err)
	}

	deviceIsReady := false
	checkDevice := func(p *conui.PromptAction) error {
		// The actual checking
		err := ensureDeviceIsReady(*d)
		if err != nil {
			log.Errorf("checkDevice error: %s", err)
			switch err.(type) {

			case deviceTestPermissionDeniedError:
				p.Message = "Device is mounted but not writable... " +
					"Please fix write permissions then press Enter to continue."
			case deviceNotFoundByUUIDError:
				p.Message = "Please mount device and press Enter to continue..."
			}
			return err
		}
		deviceIsReady = true
		return err
	}

	// The prompt that will be displayed in the device panel
	var pAction func()
	prompt := &conui.PromptAction{}
	// Allow the user to press enter on the device panel to force a device check
	pAction = func() {
		// With the device selected in the panel, the user has pressed the enter key.
		log.Printf("Action for panel %q!", wg.Border.Label)
		checkDevice(prompt)
	}
	prompt.Action = pAction

	// Finally, set the prompt for the device in the panel
	wg.SetPrompt(prompt)

	// Check device automatically periodically
	for {
		if deviceIsReady {
			break
		}
		// Rate limit
		err = checkDevice(prompt)
		if err != nil {
			time.Sleep(time.Second * 15)
			continue
		}
		break
	}
	// The prompt is not needed anymore
	wg.SetPrompt(nil)
	c.SyncDeviceMount[deviceIndex] <- true
}

func update(c *core.Context) {
	for x := 0; x < len(c.Devices); x++ {
		go deviceMountHandler(c, x)
		c.SyncDeviceMount[x] = make(chan bool)
		c.SyncProgress[x] = make(chan core.SyncProgress, 100)
		c.SyncFileProgress[x] = make(chan core.SyncFileProgress, 100)
		go func(index int) {
			lTime := time.Now()
			for {
				select {
				case p := <-c.SyncProgress[index]:
					prg := conui.Body.ProgressGauge()
					prg.SizeWritn = p.ProgressSizeWritn
					dw := conui.Body.DevicePanelByIndex(index)
					dw.SizeWritn = p.DeviceSizeWritn
				case fp := <-c.SyncFileProgress[index]:
					if time.Since(lTime) > (time.Second / 5) {
						log.WithFields(logrus.Fields{
							"fp.FileName":     fp.FileName,
							"fp.SizeTotal":    fp.SizeTotal,
							"fp.FileWritn":    fp.SizeWritn,
							"fp.SizeWritnInc": fp.SizeWritnInc,
							"bps":             fp.BytesPerSecond,
						}).Debugln("Sync file progress")
						dw := conui.Body.DevicePanelByIndex(index)
						dw.SizeWritn += fp.SizeWritnInc
						lTime = time.Now()
					}
				}
			}
		}(x)
	}
	go func() {
		for {
			if !termbox.IsInit {
				break
			}
			conui.Redraw <- true
			time.Sleep(time.Second / 5)
		}
	}()
}

func syncStart(c *cli.Context) {
	defer cleanupAtExit()
	log.WithFields(logrus.Fields{
		"version": 0.2,
		"date":    time.Now().Format(time.RFC3339),
	}).Infoln("Ghetto Device Storage")
	c2 := loadInitialState(c)

	conui.Init()
	BuildConsole(c2)
	go eventListener(c2)
	update(c2)

	// Sync the things
	// errs := core.Sync(c2, c.GlobalBool("no-dev-context"))
	// if len(errs) > 0 {
	// for _, e := range errs {
	// log.Errorf("Sync error: %s", e.Error())
	// }
	// }

	// // Fin
	// dumpContextToFile(c, c2)
	// log.Info("ALL DONE -- Sync complete!")

	// Keep the UI going so the user can see the greatness of their efforts
	for {
		// Breathing room
		time.Sleep(time.Second)
	}
}
<commit_msg>Re-enable file syncing<commit_after>package main

import (
	"conui"
	"core"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
)

func NewSyncCommand() cli.Command {
	return cli.Command{
		Name:  "sync",
		Usage: "Synchronize files to devices",
		Action: func(c *cli.Context) {
			err := checkEnvVariables(c)
			if err != nil {
				panic(fatal{fmt.Sprintf("Could not set environment variables: %s", err)})
			}
			if !c.GlobalBool("no-file-log") {
				lp := cleanPath(c.GlobalString("log"))
				var err error
				GDS_LOG_FD, err = os.Create(lp)
				if err != nil {
					panic(fatal{fmt.Sprintf("Could not create log file: %s", err)})
				}
				log.Out = GDS_LOG_FD
			}
			lvl, err := logrus.ParseLevel(c.GlobalString("log-level"))
			if err != nil {
				panic(fatalShowHelp{fmt.Sprintf("Error parsing log level: %s", err)})
			}
			log.Level = lvl
			syncStart(c)
		},
	}
}

// loadInitialState prepares the applicaton for usage
func loadInitialState(c *cli.Context) *core.Context {
	cPath, err := getConfigFile(c.GlobalString("config"))
	if err != nil {
		panic(fatal{err})
	}
	log.WithFields(logrus.Fields{
		"path": cPath,
	}).Info("Using configuration file")

	c2, err := core.ContextFromPath(cPath)
	if err != nil {
		panic(fatal{fmt.Sprintf("Error loading config: %s", err.Error())})
	}

	c2.Files, err = core.NewFileList(c2)
	if err != nil {
		panic(fatal{fmt.Sprintf("Error retrieving FileList %s", err.Error())})
	}

	c2.Catalog, err = core.NewCatalog(c2)
	if err != nil {
		// Not wrapped in fatal here because NewCatalog returns custom error types
		panic(err)
	}

	return c2
}

func dumpContextToFile(c *cli.Context, c2 *core.Context) {
	cf, err := getContextFile(c.GlobalString("context"))
	if err != nil {
		panic(fatal{fmt.Sprintf("Could not create context JSON output file: %s", err.Error())})
	}
	j, err := json.Marshal(c2)
	if err == nil {
		err = ioutil.WriteFile(cf, j, 0644)
	}
	if err != nil {
		panic(fatal{fmt.Sprintf("Could not marshal JSON to file: %s", err.Error())})
	}
}

// BuildConsole creates the UI widgets First is the main progress guage for the overall progress Widgets are then created for
// each of the devices, but are hidden initially.
func BuildConsole(c *core.Context) {
	visible := c.OutputStreamNum
	var panel []conui.Widget
	for x, y := range c.Devices {
		panel = append(panel, conui.NewDevicePanel(y.Name, y.SizeTotal))
		if visible > 0 {
			log.Debugln("Making device", x, "visible")
			panel[x].SetVisible(true)
			if x == 0 {
				panel[x].SetSelected(true)
			}
			visible--
		}
	}
	panel[len(c.Devices)-1] = conui.NewProgressGauge(c.Catalog.TotalSize())
	panel[len(c.Devices)-1].SetVisible(true)
	conui.Body.AddRows(panel[len(c.Devices)-1])
	for x, _ := range c.Devices {
		conui.Body.AddRows(panel[x])
	}
	conui.Layout()
}

func eventListener(c *core.Context) {
	defer cleanupAtExit()
	for {
		select {
		case e := <-conui.Events:
			if e.Type == conui.EventKey && e.Ch == 'j' {
				conui.Body.SelectNext()
			}
			if e.Type == conui.EventKey && e.Ch == 'k' {
				conui.Body.SelectPrevious()
			}
			if e.Type == conui.EventKey && e.Ch == 'd' {
				p := conui.Body.Selected()
				p.SetVisible(false)
			}
			if e.Type == conui.EventKey && e.Ch == 's' {
				p := conui.Body.Selected()
				p.SetVisible(true)
			}
			if e.Type == conui.EventKey && e.Key == conui.KeyEnter {
				p := conui.Body.Selected().Prompt()
				if p != nil {
					p.Action()
				}
			}
			if e.Type == conui.EventKey && e.Ch == 'q' {
				conui.Close()
				os.Exit(0)
			}
			if e.Type == conui.EventResize {
				// conui.Body.Width = conui.TermWidth()
				conui.Layout()
				go func() { conui.Redraw <- true }()
			}
		case <-conui.Redraw:
			conui.Render()
		}
	}
}

// deviceMountHandler checks to see if the device is mounted and writable. Meant to be run as a goroutine.
func deviceMountHandler(c *core.Context, deviceIndex int) {
	// Listen on the channel for a mount request
	ns := time.Now()
	log.Debugf("Waiting for receive on SyncDeviceMount[%d]", deviceIndex)
	<-c.SyncDeviceMount[deviceIndex]
	log.Debugf("Receive from SyncDeviceMount[%d] after wait of %s", deviceIndex, time.Since(ns))

	// Get the panel widget so we can write messages for the user to it
	wg := conui.Body.Select(deviceIndex)
	d, err := c.Devices.DeviceByName(wg.Border.Label)
	if err != nil {
		log.Error(err)
	}

	deviceIsReady := false
	checkDevice := func(p *conui.PromptAction) error {
		// The actual checking
		err := ensureDeviceIsReady(*d)
		if err != nil {
			log.Errorf("checkDevice error: %s", err)
			switch err.(type) {

			case deviceTestPermissionDeniedError:
				p.Message = "Device is mounted but not writable... " +
					"Please fix write permissions then press Enter to continue."
			case deviceNotFoundByUUIDError:
				p.Message = "Please mount device and press Enter to continue..."
			}
			return err
		}
		deviceIsReady = true
		return err
	}

	// The prompt that will be displayed in the device panel
	var pAction func()
	prompt := &conui.PromptAction{}
	// Allow the user to press enter on the device panel to force a device check
	pAction = func() {
		// With the device selected in the panel, the user has pressed the enter key.
		log.Printf("Action for panel %q!", wg.Border.Label)
		checkDevice(prompt)
	}
	prompt.Action = pAction

	// Finally, set the prompt for the device in the panel
	wg.SetPrompt(prompt)

	// Check device automatically periodically
	for {
		if deviceIsReady {
			break
		}
		// Rate limit
		err = checkDevice(prompt)
		if err != nil {
			time.Sleep(time.Second * 15)
			continue
		}
		break
	}
	// The prompt is not needed anymore
	wg.SetPrompt(nil)
	c.SyncDeviceMount[deviceIndex] <- true
}

func update(c *core.Context) {
	for x := 0; x < len(c.Devices); x++ {
		go deviceMountHandler(c, x)
		c.SyncDeviceMount[x] = make(chan bool)
		c.SyncProgress[x] = make(chan core.SyncProgress, 100)
		c.SyncFileProgress[x] = make(chan core.SyncFileProgress, 100)
		go func(index int) {
			lTime := time.Now()
			for {
				select {
				case p := <-c.SyncProgress[index]:
					prg := conui.Body.ProgressGauge()
					prg.SizeWritn = p.ProgressSizeWritn
					dw := conui.Body.DevicePanelByIndex(index)
					dw.SizeWritn = p.DeviceSizeWritn
				case fp := <-c.SyncFileProgress[index]:
					if time.Since(lTime) > (time.Second / 5) {
						log.WithFields(logrus.Fields{
							"fp.FileName":     fp.FileName,
							"fp.SizeTotal":    fp.SizeTotal,
							"fp.FileWritn":    fp.SizeWritn,
							"fp.SizeWritnInc": fp.SizeWritnInc,
							"bps":             fp.BytesPerSecond,
						}).Debugln("Sync file progress")
						dw := conui.Body.DevicePanelByIndex(index)
						dw.SizeWritn += fp.SizeWritnInc
						lTime = time.Now()
					}
				}
			}
		}(x)
	}
	go func() {
		for {
			if !termbox.IsInit {
				break
			}
			conui.Redraw <- true
			time.Sleep(time.Second / 5)
		}
	}()
}

func syncStart(c *cli.Context) {
	defer cleanupAtExit()
	log.WithFields(logrus.Fields{
		"version": 0.2,
		"date":    time.Now().Format(time.RFC3339),
	}).Infoln("Ghetto Device Storage")
	c2 := loadInitialState(c)

	conui.Init()
	BuildConsole(c2)
	go eventListener(c2)
	update(c2)

	// Sync the things
	errs := core.Sync(c2, c.GlobalBool("no-dev-context"))
	if len(errs) > 0 {
		for _, e := range errs {
			log.Errorf("Sync error: %s", e.Error())
		}
	}

	// Fin
	dumpContextToFile(c, c2)
	log.Info("ALL DONE -- Sync complete!")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"strings"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pkg/cobramainutil"
	"github.com/pachyderm/pachyderm/src/pkg/mainutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
	"github.com/spf13/cobra"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS": "0.0.0.0:651",
	}
)

type appEnv struct {
	Address string `env:"PPS_ADDRESS"`
}

func main() {
	mainutil.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)

	clientConn, err := grpc.Dial(appEnv.Address)
	if err != nil {
		return err
	}
	apiClient := pps.NewApiClient(clientConn)

	var protoFlag bool

	versionCmd := cobramainutil.Command{
		Use:  "version",
		Long: "Print the version.",
		Run: func(cmd *cobra.Command, args []string) error {
			getVersionResponse, err := ppsutil.GetVersion(apiClient)
			if err != nil {
				return err
			}
			fmt.Printf("Client: %s\nServer: %s\n", common.VersionString(), pps.VersionString(getVersionResponse.Version))
			return nil
		},
	}.ToCobraCommand()

	inspectCmd := cobramainutil.Command{
		Use:        "inspect github.com/user/repository [path/to/specDir]",
		Long:       "Inspect a pipeline specification.",
		MinNumArgs: 1,
		MaxNumArgs: 2,
		Run: func(cmd *cobra.Command, args []string) error {
			path := args[0]
			if !strings.HasPrefix(path, "github.com/") {
				return fmt.Errorf("%s is not supported", path)
			}
			split := strings.Split(path, "/")
			if len(split) != 3 {
				return fmt.Errorf("%s is not supported", path)
			}
			branch := ""
			accessToken := ""
			contextDir := ""
			if len(args) > 1 {
				contextDir = args[1]
			}
			getPipelineResponse, err := ppsutil.GetPipelineGithub(
				apiClient,
				contextDir,
				split[1],
				split[2],
				branch,
				accessToken,
			)
			if err != nil {
				return err
			}
			if protoFlag {
				fmt.Printf("%v\n", getPipelineResponse.Pipeline)
			} else {
				data, err := json.MarshalIndent(getPipelineResponse.Pipeline, "", "\t ")
				if err != nil {
					return err
				}
				fmt.Println(string(data))
			}
			return nil
		},
	}.ToCobraCommand()
	inspectCmd.Flags().BoolVar(&protoFlag, "proto", false, "Print in proto format instead of JSON.")

	startCmd := cobramainutil.Command{
		Use:        "start github.com/user/repository [path/to/specDir]",
		Long:       "Start a pipeline specification run.",
		MinNumArgs: 1,
		MaxNumArgs: 2,
		Run: func(cmd *cobra.Command, args []string) error {
			path := args[0]
			if !strings.HasPrefix(path, "github.com/") {
				return fmt.Errorf("%s is not supported", path)
			}
			split := strings.Split(path, "/")
			if len(split) != 3 {
				return fmt.Errorf("%s is not supported", path)
			}
			branch := ""
			accessToken := ""
			contextDir := ""
			if len(args) > 1 {
				contextDir = args[1]
			}
			startPipelineRunResponse, err := ppsutil.StartPipelineRunGithub(
				apiClient,
				contextDir,
				split[1],
				split[2],
				branch,
				accessToken,
			)
			if err != nil {
				return err
			}
			fmt.Printf("started pipeline run with id %s\n", startPipelineRunResponse.PipelineRunId)
			return nil
		},
	}.ToCobraCommand()

	statusCmd := cobramainutil.Command{
		Use:     "status pipelineRunID",
		Long:    "Get the status of a pipeline run.",
		NumArgs: 1,
		Run: func(cmd *cobra.Command, args []string) error {
			getPipelineRunStatusResponse, err := ppsutil.GetPipelineRunStatus(
				apiClient,
				args[0],
			)
			if err != nil {
				return err
			}
			name, ok := pps.PipelineRunStatusType_name[int32(getPipelineRunStatusResponse.PipelineRunStatus.PipelineRunStatusType)]
			if !ok {
				return fmt.Errorf("unknown run status")
			}
			fmt.Printf("%s: %s\n", args[0], strings.Replace(name, "PIPELINE_RUN_STATUS_TYPE_", "", -1))
			return nil
		},
	}.ToCobraCommand()

	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}
	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(inspectCmd)
	rootCmd.AddCommand(startCmd)
	rootCmd.AddCommand(statusCmd)
	return rootCmd.Execute()
}
<commit_msg>clean up pps cli<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"strings"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pkg/cobramainutil"
	"github.com/pachyderm/pachyderm/src/pkg/mainutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
	"github.com/spf13/cobra"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS": "0.0.0.0:651",
	}
)

type appEnv struct {
	Address string `env:"PPS_ADDRESS"`
}

func main() {
	mainutil.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)

	clientConn, err := grpc.Dial(appEnv.Address)
	if err != nil {
		return err
	}
	apiClient := pps.NewApiClient(clientConn)

	var protoFlag bool

	versionCmd := cobramainutil.Command{
		Use:  "version",
		Long: "Print the version.",
		Run: func(cmd *cobra.Command, args []string) error {
			getVersionResponse, err := ppsutil.GetVersion(apiClient)
			if err != nil {
				return err
			}
			fmt.Printf("Client: %s\nServer: %s\n", common.VersionString(), pps.VersionString(getVersionResponse.Version))
			return nil
		},
	}.ToCobraCommand()

	inspectCmd := cobramainutil.Command{
		Use:        "inspect github.com/user/repository [path/to/specDir]",
		Long:       "Inspect a pipeline specification.",
		MinNumArgs: 1,
		MaxNumArgs: 2,
		Run: func(cmd *cobra.Command, args []string) error {
			pipelineArgs, err := getPipelineArgs(args)
			if err != nil {
				return err
			}
			getPipelineResponse, err := ppsutil.GetPipelineGithub(
				apiClient,
				pipelineArgs.contextDir,
				pipelineArgs.user,
				pipelineArgs.repository,
				pipelineArgs.branch,
				pipelineArgs.accessToken,
			)
			if err != nil {
				return err
			}
			if protoFlag {
				fmt.Printf("%v\n", getPipelineResponse.Pipeline)
			} else {
				data, err := json.MarshalIndent(getPipelineResponse.Pipeline, "", "\t ")
				if err != nil {
					return err
				}
				fmt.Println(string(data))
			}
			return nil
		},
	}.ToCobraCommand()
	inspectCmd.Flags().BoolVar(&protoFlag, "proto", false, "Print in proto format instead of JSON.")

	startCmd := cobramainutil.Command{
		Use:        "start github.com/user/repository [path/to/specDir]",
		Long:       "Start a pipeline specification run.",
		MinNumArgs: 1,
		MaxNumArgs: 2,
		Run: func(cmd *cobra.Command, args []string) error {
			pipelineArgs, err := getPipelineArgs(args)
			if err != nil {
				return err
			}
			startPipelineRunResponse, err := ppsutil.StartPipelineRunGithub(
				apiClient,
				pipelineArgs.contextDir,
				pipelineArgs.user,
				pipelineArgs.repository,
				pipelineArgs.branch,
				pipelineArgs.accessToken,
			)
			if err != nil {
				return err
			}
			fmt.Printf("started pipeline run with id %s\n", startPipelineRunResponse.PipelineRunId)
			return nil
		},
	}.ToCobraCommand()

	statusCmd := cobramainutil.Command{
		Use:     "status pipelineRunID",
		Long:    "Get the status of a pipeline run.",
		NumArgs: 1,
		Run: func(cmd *cobra.Command, args []string) error {
			getPipelineRunStatusResponse, err := ppsutil.GetPipelineRunStatus(
				apiClient,
				args[0],
			)
			if err != nil {
				return err
			}
			name, ok := pps.PipelineRunStatusType_name[int32(getPipelineRunStatusResponse.PipelineRunStatus.PipelineRunStatusType)]
			if !ok {
				return fmt.Errorf("unknown run status")
			}
			fmt.Printf("%s: %s\n", args[0], strings.Replace(name, "PIPELINE_RUN_STATUS_TYPE_", "", -1))
			return nil
		},
	}.ToCobraCommand()

	rootCmd := &cobra.Command{
		Use: "pps",
		Long: `Access the PPS API.

Note that this CLI is experimental and does not even check for common errors.
The environment variable PPS_ADDRESS controls what server the CLI connects to, the default is 0.0.0.0:651.`,
	}
	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(inspectCmd)
	rootCmd.AddCommand(startCmd)
	rootCmd.AddCommand(statusCmd)
	return rootCmd.Execute()
}

type pipelineArgs struct {
	contextDir  string
	user        string
	repository  string
	branch      string
	accessToken string
}

func getPipelineArgs(args []string) (*pipelineArgs, error) {
	path := args[0]
	if !strings.HasPrefix(path, "github.com/") {
		return nil, fmt.Errorf("%s is not supported", path)
	}
	split := strings.Split(path, "/")
	if len(split) != 3 {
		return nil, fmt.Errorf("%s is not supported", path)
	}
	contextDir := ""
	if len(args) > 1 {
		contextDir = args[1]
	}
	return &pipelineArgs{
		contextDir:  contextDir,
		user:        split[1],
		repository:  split[2],
		branch:      "",
		accessToken: "",
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "6698ca2900e2"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		username = os.Getenv("USER")
		if username == "" {
			username = "nobody"
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "golang.org", "x")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<commit_msg>cmd/api: bump go.tools golden CL hash<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "ae08a5291439"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		username = os.Getenv("USER")
		if username == "" {
			username = "nobody"
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "golang.org", "x")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via the dist tool.
// To invoke manually: go tool dist test -run api --no-rebuild
package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

func goCmd() string {
	var exeSuffix string
	if runtime.GOOS == "windows" {
		exeSuffix = ".exe"
	}
	path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix)
	if _, err := os.Stat(path); err == nil {
		return path
	}
	return "go"
}

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}

	out, err := exec.Command(goCmd(), "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7", "go1.8", "go1.9", "go1.10"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}
<commit_msg>cmd/api: don’t rely on hardcoded go versions<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via the dist tool.
// To invoke manually: go tool dist test -run api --no-rebuild
package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

func goCmd() string {
	var exeSuffix string
	if runtime.GOOS == "windows" {
		exeSuffix = ".exe"
	}
	path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix)
	if _, err := os.Stat(path); err == nil {
		return path
	}
	return "go"
}

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}

	apiDir := filepath.Join(goroot, "api")
	out, err := exec.Command(goCmd(), "tool", "api",
		"-c", findAPIDirFiles(apiDir),
		"-next", filepath.Join(apiDir, "next.txt"),
		"-except", filepath.Join(apiDir, "except.txt")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// findAPIDirFiles returns a comma-separated list of Go API files
// (go1.txt, go1.1.txt, etc.) located in apiDir.
func findAPIDirFiles(apiDir string) string {
	dir, err := os.Open(apiDir)
	if err != nil {
		log.Fatal(err)
	}
	defer dir.Close()
	fs, err := dir.Readdirnames(-1)
	if err != nil {
		log.Fatal(err)
	}
	var apiFiles []string
	for _, fn := range fs {
		if strings.HasPrefix(fn, "go1") {
			apiFiles = append(apiFiles, filepath.Join(apiDir, fn))
		}
	}
	return strings.Join(apiFiles, ",")
}
<|endoftext|>


<commit_before>package handlers

import (
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ArchiveData struct {
	Years   map[int]Year
	Posts   *[]models.Entry
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []models.Entry

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	years := make(map[int]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		years[year] = make(Year)
		log.Infof(c, "Adding %d.", year)

		for _, month := range months {
			if year < newest.Year() || (year == newest.Year() && month <= newest.Month()) {
				years[year][month] = make([]Day, daysIn(month, year))
				log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[year][month]))
			}
		}
	}

	q := models.ArchivePageQuery()
	t := q.Run(c)
	for {
		var p models.Entry
		_, err := t.Next(&p)
		if err == datastore.Done {
			break // No further entities match the query.
		}

		if err != nil {
			log.Errorf(c, "Error fetching next Entry: %v", err)
			break
		}

		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()
		log.Infof(c, "Trying post id %d", p.Id)

		if years[year] == nil {
			log.Errorf(c, "%d isn't a valid year.", year)
		} else {
			if years[year][month] == nil {
				log.Errorf(c, "%d/%d isn't a valid month.", year, month)
			} else {
				if years[year][month][day] == nil {
					log.Infof(c, "Making %d/%d/%d", year, month, day)
					years[year][month][day] = make(Day, 0)
				}
				log.Infof(c, "Appending %d/%d/%d: %+v", year, month, day, years[year][month][day])
				years[year][month][day] = append(years[year][month][day], p)
			}
		}
	}
	log.Infof(c, "Added posts.")

	data := &ArchiveData{Years: years, IsAdmin: user.IsAdmin(c), Posts: entries}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<commit_msg>only use ints<commit_after>package handlers

import (
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ArchiveData struct {
	Years   map[int]Year
	Posts   *[]models.Entry
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []int64

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	years := make(map[int]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		years[year] = make(Year)
		log.Infof(c, "Adding %d.", year)

		for _, month := range months {
			if year < newest.Year() || (year == newest.Year() && month <= newest.Month()) {
				years[year][month] = make([]Day, daysIn(month, year))
				log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[year][month]))
			}
		}
	}

	q := models.ArchivePageQuery()
	t := q.Run(c)
	for {
		var p models.Entry
		_, err := t.Next(&p)
		if err == datastore.Done {
			break // No further entities match the query.
		}

		if err != nil {
			log.Errorf(c, "Error fetching next Entry: %v", err)
			break
		}

		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()
		log.Infof(c, "Trying post id %d", p.Id)

		if years[year] == nil {
			log.Errorf(c, "%d isn't a valid year.", year)
		} else {
			if years[year][month] == nil {
				log.Errorf(c, "%d/%d isn't a valid month.", year, month)
			} else {
				if years[year][month][day] == nil {
					log.Infof(c, "Making %d/%d/%d", year, month, day)
					years[year][month][day] = make(Day, 0)
				}
				log.Infof(c, "Appending %d/%d/%d: %+v", year, month, day, years[year][month][day])
				years[year][month][day] = append(years[year][month][day], p.Id)
			}
		}
	}
	log.Infof(c, "Added posts.")

	data := &ArchiveData{Years: years, IsAdmin: user.IsAdmin(c), Posts: entries}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<|endoftext|>


<commit_before>package ocrworker

import (
	"encoding/json"
	"fmt"
	"time"
	"io/ioutil"

	"github.com/couchbaselabs/logg"
	"github.com/streadway/amqp"
	"github.com/mitchellh/goamz/aws"
  "github.com/mitchellh/goamz/s3"
)

type OcrRpcWorker struct {
	rabbitConfig RabbitConfig
	conn         *amqp.Connection
	channel      *amqp.Channel
	tag          string
	Done         chan error
}

const tag = "foo" // TODO: should be unique for each worker instance (eg, uuid)

func NewOcrRpcWorker(rc RabbitConfig) (*OcrRpcWorker, error) {
	ocrRpcWorker := &OcrRpcWorker{
		rabbitConfig: rc,
		conn:         nil,
		channel:      nil,
		tag:          tag,
		Done:         make(chan error),
	}
	return ocrRpcWorker, nil
}

func (w OcrRpcWorker) Run() error {

	var err error

	logg.LogTo("OCR_WORKER", "Run() called...")

	logg.LogTo("OCR_WORKER", "dialing %q", w.rabbitConfig.AmqpURI)
	w.conn, err = amqp.Dial(w.rabbitConfig.AmqpURI)
	if err != nil {
		logg.LogTo("OCR_WORKER", "error connecting to rabbitmq %v", err)
		return err
	}

	go func() {
		fmt.Printf("closing: %s", <-w.conn.NotifyClose(make(chan *amqp.Error)))
	}()

	logg.LogTo("OCR_WORKER", "got Connection, getting Channel")
	w.channel, err = w.conn.Channel()
	if err != nil {
		return err
	}

	if err = w.channel.ExchangeDeclare(
		w.rabbitConfig.Exchange,     // name of the exchange
		w.rabbitConfig.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return err
	}

	// just use the routing key as the queue name, since there's no reason
	// to have a different name
	queueName := w.rabbitConfig.RoutingKey

	queue, err := w.channel.QueueDeclare(
		queueName, // name of the queue
		true,      // durable
		false,     // delete when usused
		false,     // exclusive
		false,     // noWait
		nil,       // arguments
	)
	if err != nil {
		return err
	}

	logg.LogTo("OCR_WORKER", "binding to: %v", w.rabbitConfig.RoutingKey)

	if err = w.channel.QueueBind(
		queue.Name,                // name of the queue
		w.rabbitConfig.RoutingKey, // bindingKey
		w.rabbitConfig.Exchange,   // sourceExchange
		false, // noWait
		nil,   // arguments
	); err != nil {
		return err
	}

	logg.LogTo("OCR_WORKER", "Queue bound to Exchange, starting Consume (consumer tag %q)", tag)
	deliveries, err := w.channel.Consume(
		queue.Name, // name
		tag,        // consumerTag,
		true,       // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)
	if err != nil {
		return err
	}

	go w.handle(deliveries, w.Done)

	return nil
}

func (w *OcrRpcWorker) Shutdown() error {
	// will close() the deliveries channel
	if err := w.channel.Cancel(w.tag, true); err != nil {
		return fmt.Errorf("Worker cancel failed: %s", err)
	}

	if err := w.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}

	defer logg.LogTo("OCR_WORKER", "Shutdown OK")

	// wait for handle() to exit
	return <-w.Done
}

func (w *OcrRpcWorker) handle(deliveries <-chan amqp.Delivery, done chan error) {
	for d := range deliveries {
		logg.LogTo(
			"OCR_WORKER",
			"got %d byte delivery: [%v]. Routing key: %v  Reply to: %v",
			len(d.Body),
			d.DeliveryTag,
			d.RoutingKey,
			d.ReplyTo,
		)

		ocrResult, err := w.resultForDelivery(d)
		if err != nil {
			msg := "Error generating ocr result.  Error: %v"
			logg.LogError(fmt.Errorf(msg, err))
		}

		err = w.uploadToS3(ocrResult.Text)
		if err != nil {
			logg.LogError(fmt.Errorf("Error uploading file to s3. Error: %v", err))
		}

		logg.LogTo("OCR_WORKER", "Sending rpc response: %+v", ocrResult)
		err = w.sendRpcResponse(ocrResult, d.ReplyTo, d.CorrelationId)
		if err != nil {
			msg := "Error returning ocr result: %v.  Error: %v"
			logg.LogError(fmt.Errorf(msg, ocrResult, err))
			// if we can't send our response, let's just abort
			done <- err
			break
		}

	}
	logg.LogTo("OCR_WORKER", "handle: deliveries channel closed")
	done <- fmt.Errorf("handle: deliveries channel closed")
}

func (w *OcrRpcWorker) resultForDelivery(d amqp.Delivery) (OcrResult, error) {

	ocrRequest := OcrRequest{}
	ocrResult := OcrResult{Text: "Error"}
	err := json.Unmarshal(d.Body, &ocrRequest)
	if err != nil {
		msg := "Error unmarshaling json: %v.  Error: %v"
		errMsg := fmt.Sprintf(msg, string(d.Body), err)
		logg.LogError(fmt.Errorf(errMsg))
		ocrResult.Text = errMsg
		return ocrResult, err
	}

	ocrEngine := NewOcrEngine(ocrRequest.EngineType)

	ocrResult, err = ocrEngine.ProcessRequest(ocrRequest)

	if err != nil {
		msg := "Error processing image url: %v.  Error: %v"
		errMsg := fmt.Sprintf(msg, ocrRequest.ImgUrl, err)
		logg.LogError(fmt.Errorf(errMsg))
		ocrResult.Text = errMsg
		return ocrResult, err
	}

	return ocrResult, nil

}

func (w *OcrRpcWorker) uploadToS3(content string) error {

	auth, err := aws.EnvAuth()
  if err != nil {
		logg.LogTo("OCR_WORKER", "%v", err)
		return err
  }
  client := s3.New(auth, aws.USWest)

	err = ioutil.WriteFile("/test.txt", []byte(content), 0644)
	if err != nil {
		logg.LogTo("OCR_WORKER", "%v", err)
		return err
	}

	bucket := client.Bucket("avid-documents")
	err = bucket.Put("/test.txt", []byte(content), "text/plain", s3.PublicRead)
	return err

}

func (w *OcrRpcWorker) sendRpcResponse(r OcrResult, replyTo string, correlationId string) error {

	if w.rabbitConfig.Reliable {
		// Do not use w.rabbitConfig.Reliable=true due to major issues
		// that will completely  wedge the rpc worker.  Setting the
		// buffered channels length higher would delay the problem,
		// but then it would still happen later.
		if err := w.channel.Confirm(false); err != nil {
			return err
		}

		ack, nack := w.channel.NotifyConfirm(make(chan uint64, 100), make(chan uint64, 100))

		defer confirmDeliveryWorker(ack, nack)
	}

	logg.LogTo("OCR_WORKER", "sendRpcResponse to: %v", replyTo)
	if err := w.channel.Publish(
		w.rabbitConfig.Exchange, // publish to an exchange
		replyTo,                 // routing to 0 or more queues
		false,                   // mandatory
		false,                   // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/plain",
			ContentEncoding: "",
			Body:            []byte(r.Text),
			DeliveryMode:    amqp.Transient, // 1=non-persistent, 2=persistent
			Priority:        0,              // 0-9
			CorrelationId:   correlationId,
			// a bunch of application/implementation-specific fields
		},
	); err != nil {
		return err
	}
	logg.LogTo("OCR_WORKER", "sendRpcResponse succeeded")
	return nil

}

func confirmDeliveryWorker(ack, nack chan uint64) {
	logg.LogTo("OCR_WORKER", "awaiting delivery confirmation ...")
	select {
	case tag := <-ack:
		logg.LogTo("OCR_WORKER", "confirmed delivery, tag: %v", tag)
	case tag := <-nack:
		logg.LogTo("OCR_WORKER", "failed to confirm delivery: %v", tag)
	case <-time.After(RPC_RESPONSE_TIMEOUT):
		// this is bad, the worker will probably be dsyfunctional
		// at this point, so panic
		logg.LogPanic("timeout trying to confirm delivery")
	}
}
<commit_msg>fix bucket error<commit_after>package ocrworker

import (
	"encoding/json"
	"fmt"
	"time"
	"io/ioutil"

	"github.com/couchbaselabs/logg"
	"github.com/streadway/amqp"
	"github.com/mitchellh/goamz/aws"
  "github.com/mitchellh/goamz/s3"
)

type OcrRpcWorker struct {
	rabbitConfig RabbitConfig
	conn         *amqp.Connection
	channel      *amqp.Channel
	tag          string
	Done         chan error
}

const tag = "foo" // TODO: should be unique for each worker instance (eg, uuid)

func NewOcrRpcWorker(rc RabbitConfig) (*OcrRpcWorker, error) {
	ocrRpcWorker := &OcrRpcWorker{
		rabbitConfig: rc,
		conn:         nil,
		channel:      nil,
		tag:          tag,
		Done:         make(chan error),
	}
	return ocrRpcWorker, nil
}

func (w OcrRpcWorker) Run() error {

	var err error

	logg.LogTo("OCR_WORKER", "Run() called...")

	logg.LogTo("OCR_WORKER", "dialing %q", w.rabbitConfig.AmqpURI)
	w.conn, err = amqp.Dial(w.rabbitConfig.AmqpURI)
	if err != nil {
		logg.LogTo("OCR_WORKER", "error connecting to rabbitmq %v", err)
		return err
	}

	go func() {
		fmt.Printf("closing: %s", <-w.conn.NotifyClose(make(chan *amqp.Error)))
	}()

	logg.LogTo("OCR_WORKER", "got Connection, getting Channel")
	w.channel, err = w.conn.Channel()
	if err != nil {
		return err
	}

	if err = w.channel.ExchangeDeclare(
		w.rabbitConfig.Exchange,     // name of the exchange
		w.rabbitConfig.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return err
	}

	// just use the routing key as the queue name, since there's no reason
	// to have a different name
	queueName := w.rabbitConfig.RoutingKey

	queue, err := w.channel.QueueDeclare(
		queueName, // name of the queue
		true,      // durable
		false,     // delete when usused
		false,     // exclusive
		false,     // noWait
		nil,       // arguments
	)
	if err != nil {
		return err
	}

	logg.LogTo("OCR_WORKER", "binding to: %v", w.rabbitConfig.RoutingKey)

	if err = w.channel.QueueBind(
		queue.Name,                // name of the queue
		w.rabbitConfig.RoutingKey, // bindingKey
		w.rabbitConfig.Exchange,   // sourceExchange
		false, // noWait
		nil,   // arguments
	); err != nil {
		return err
	}

	logg.LogTo("OCR_WORKER", "Queue bound to Exchange, starting Consume (consumer tag %q)", tag)
	deliveries, err := w.channel.Consume(
		queue.Name, // name
		tag,        // consumerTag,
		true,       // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)
	if err != nil {
		return err
	}

	go w.handle(deliveries, w.Done)

	return nil
}

func (w *OcrRpcWorker) Shutdown() error {
	// will close() the deliveries channel
	if err := w.channel.Cancel(w.tag, true); err != nil {
		return fmt.Errorf("Worker cancel failed: %s", err)
	}

	if err := w.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}

	defer logg.LogTo("OCR_WORKER", "Shutdown OK")

	// wait for handle() to exit
	return <-w.Done
}

func (w *OcrRpcWorker) handle(deliveries <-chan amqp.Delivery, done chan error) {
	for d := range deliveries {
		logg.LogTo(
			"OCR_WORKER",
			"got %d byte delivery: [%v]. Routing key: %v  Reply to: %v",
			len(d.Body),
			d.DeliveryTag,
			d.RoutingKey,
			d.ReplyTo,
		)

		ocrResult, err := w.resultForDelivery(d)
		if err != nil {
			msg := "Error generating ocr result.  Error: %v"
			logg.LogError(fmt.Errorf(msg, err))
		}

		err = w.uploadToS3(ocrResult.Text)
		if err != nil {
			logg.LogError(fmt.Errorf("Error uploading file to s3. Error: %v", err))
		}

		logg.LogTo("OCR_WORKER", "Sending rpc response: %+v", ocrResult)
		err = w.sendRpcResponse(ocrResult, d.ReplyTo, d.CorrelationId)
		if err != nil {
			msg := "Error returning ocr result: %v.  Error: %v"
			logg.LogError(fmt.Errorf(msg, ocrResult, err))
			// if we can't send our response, let's just abort
			done <- err
			break
		}

	}
	logg.LogTo("OCR_WORKER", "handle: deliveries channel closed")
	done <- fmt.Errorf("handle: deliveries channel closed")
}

func (w *OcrRpcWorker) resultForDelivery(d amqp.Delivery) (OcrResult, error) {

	ocrRequest := OcrRequest{}
	ocrResult := OcrResult{Text: "Error"}
	err := json.Unmarshal(d.Body, &ocrRequest)
	if err != nil {
		msg := "Error unmarshaling json: %v.  Error: %v"
		errMsg := fmt.Sprintf(msg, string(d.Body), err)
		logg.LogError(fmt.Errorf(errMsg))
		ocrResult.Text = errMsg
		return ocrResult, err
	}

	ocrEngine := NewOcrEngine(ocrRequest.EngineType)

	ocrResult, err = ocrEngine.ProcessRequest(ocrRequest)

	if err != nil {
		msg := "Error processing image url: %v.  Error: %v"
		errMsg := fmt.Sprintf(msg, ocrRequest.ImgUrl, err)
		logg.LogError(fmt.Errorf(errMsg))
		ocrResult.Text = errMsg
		return ocrResult, err
	}

	return ocrResult, nil

}

func (w *OcrRpcWorker) uploadToS3(content string) error {

	auth, err := aws.EnvAuth()
  if err != nil {
		logg.LogTo("OCR_WORKER", "%v", err)
		return err
  }
  client := s3.New(auth, aws.USEast)

	err = ioutil.WriteFile("/test.txt", []byte(content), 0644)
	if err != nil {
		logg.LogTo("OCR_WORKER", "%v", err)
		return err
	}

	bucket := client.Bucket("avid-documents")
	err = bucket.Put("/test.txt", []byte(content), "text/plain", s3.PublicRead)
	return err

}

func (w *OcrRpcWorker) sendRpcResponse(r OcrResult, replyTo string, correlationId string) error {

	if w.rabbitConfig.Reliable {
		// Do not use w.rabbitConfig.Reliable=true due to major issues
		// that will completely  wedge the rpc worker.  Setting the
		// buffered channels length higher would delay the problem,
		// but then it would still happen later.
		if err := w.channel.Confirm(false); err != nil {
			return err
		}

		ack, nack := w.channel.NotifyConfirm(make(chan uint64, 100), make(chan uint64, 100))

		defer confirmDeliveryWorker(ack, nack)
	}

	logg.LogTo("OCR_WORKER", "sendRpcResponse to: %v", replyTo)
	if err := w.channel.Publish(
		w.rabbitConfig.Exchange, // publish to an exchange
		replyTo,                 // routing to 0 or more queues
		false,                   // mandatory
		false,                   // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/plain",
			ContentEncoding: "",
			Body:            []byte(r.Text),
			DeliveryMode:    amqp.Transient, // 1=non-persistent, 2=persistent
			Priority:        0,              // 0-9
			CorrelationId:   correlationId,
			// a bunch of application/implementation-specific fields
		},
	); err != nil {
		return err
	}
	logg.LogTo("OCR_WORKER", "sendRpcResponse succeeded")
	return nil

}

func confirmDeliveryWorker(ack, nack chan uint64) {
	logg.LogTo("OCR_WORKER", "awaiting delivery confirmation ...")
	select {
	case tag := <-ack:
		logg.LogTo("OCR_WORKER", "confirmed delivery, tag: %v", tag)
	case tag := <-nack:
		logg.LogTo("OCR_WORKER", "failed to confirm delivery: %v", tag)
	case <-time.After(RPC_RESPONSE_TIMEOUT):
		// this is bad, the worker will probably be dsyfunctional
		// at this point, so panic
		logg.LogPanic("timeout trying to confirm delivery")
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!nacl,!plan9

package syslog

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"testing"
	"time"
)

func runPktSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string
	ct := 0
	for {
		var n int
		var err error

		c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
		n, _, err = c.ReadFrom(buf[:])
		rcvd += string(buf[:n])
		if err != nil {
			if oe, ok := err.(*net.OpError); ok {
				if ct < 3 && oe.Temporary() {
					ct++
					continue
				}
			}
			break
		}
	}
	c.Close()
	done <- rcvd
}

var crashy = false

func runStreamSyslog(l net.Listener, done chan<- string, wg *sync.WaitGroup) {
	for {
		var c net.Conn
		var err error
		if c, err = l.Accept(); err != nil {
			return
		}
		wg.Add(1)
		go func(c net.Conn) {
			defer wg.Done()
			c.SetReadDeadline(time.Now().Add(5 * time.Second))
			b := bufio.NewReader(c)
			for ct := 1; !crashy || ct&7 != 0; ct++ {
				s, err := b.ReadString('\n')
				if err != nil {
					break
				}
				done <- s
			}
			c.Close()
		}(c)
	}
}

func startServer(n, la string, done chan<- string) (addr string, sock io.Closer, wg *sync.WaitGroup) {
	if n == "udp" || n == "tcp" {
		la = "127.0.0.1:0"
	} else {
		// unix and unixgram: choose an address if none given
		if la == "" {
			// use ioutil.TempFile to get a name that is unique
			f, err := ioutil.TempFile("", "syslogtest")
			if err != nil {
				log.Fatal("TempFile: ", err)
			}
			f.Close()
			la = f.Name()
		}
		os.Remove(la)
	}

	wg = new(sync.WaitGroup)
	if n == "udp" || n == "unixgram" {
		l, e := net.ListenPacket(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.LocalAddr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runPktSyslog(l, done)
		}()
	} else {
		l, e := net.Listen(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.Addr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runStreamSyslog(l, done, wg)
		}()
	}
	return
}

func TestWithSimulated(t *testing.T) {
	msg := "Test 123"
	transport := []string{"unix", "unixgram", "udp", "tcp"}

	for _, tr := range transport {
		done := make(chan string)
		addr, sock, srvWG := startServer(tr, "", done)
		defer srvWG.Wait()
		defer sock.Close()
		if tr == "unix" || tr == "unixgram" {
			defer os.Remove(addr)
		}
		s, err := Dial(tr, addr, LOG_INFO|LOG_USER, "syslog_test")
		if err != nil {
			t.Fatalf("Dial() failed: %v", err)
		}
		err = s.Info(msg)
		if err != nil {
			t.Fatalf("log failed: %v", err)
		}
		check(t, msg, <-done)
		s.Close()
	}
}

func TestFlap(t *testing.T) {
	net := "unix"
	done := make(chan string)
	addr, sock, srvWG := startServer(net, "", done)
	defer srvWG.Wait()
	defer os.Remove(addr)
	defer sock.Close()

	s, err := Dial(net, addr, LOG_INFO|LOG_USER, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %v", err)
	}
	msg := "Moo 2"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	// restart the server
	_, sock2, srvWG2 := startServer(net, addr, done)
	defer srvWG2.Wait()
	defer sock2.Close()

	// and try retransmitting
	msg = "Moo 3"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	s.Close()
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Skip("skipping syslog test during -short")
	}

	s, err := New(LOG_INFO|LOG_USER, "the_tag")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func check(t *testing.T, in, out string) {
	tmpl := fmt.Sprintf("<%d>%%s %%s syslog_test[%%d]: %s\n", LOG_USER+LOG_INFO, in)
	if hostname, err := os.Hostname(); err != nil {
		t.Error("Error retrieving hostname")
	} else {
		var parsedHostname, timestamp string
		var pid int
		if n, err := fmt.Sscanf(out, tmpl, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
			t.Errorf("Got %q, does not match template %q (%d %s)", out, tmpl, n, err)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			addr, sock, srvWG := startServer("udp", "", done)
			defer srvWG.Wait()
			defer sock.Close()
			l, err := Dial("udp", addr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer l.Close()
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %v", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
				t.Errorf("s.Info() = '%q', didn't match '%q' (%d %s)", rcvd, test.exp, n, err)
			}
		}
	}
}

func TestConcurrentWrite(t *testing.T) {
	addr, sock, srvWG := startServer("udp", "", make(chan string, 1))
	defer srvWG.Wait()
	defer sock.Close()
	w, err := Dial("udp", addr, LOG_USER|LOG_ERR, "how's it going?")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %v", err)
	}
	var wg sync.WaitGroup
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			err := w.Info("test")
			if err != nil {
				t.Errorf("Info() failed: %v", err)
				return
			}
		}()
	}
	wg.Wait()
}

func TestConcurrentReconnect(t *testing.T) {
	crashy = true
	defer func() { crashy = false }()

	const N = 10
	const M = 100
	net := "unix"
	done := make(chan string, N*M)
	addr, sock, srvWG := startServer(net, "", done)
	defer os.Remove(addr)

	// count all the messages arriving
	count := make(chan int)
	go func() {
		ct := 0
		for range done {
			ct++
			// we are looking for 500 out of 1000 events
			// here because lots of log messages are lost
			// in buffers (kernel and/or bufio)
			if ct > N*M/2 {
				break
			}
		}
		count <- ct
	}()

	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			w, err := Dial(net, addr, LOG_USER|LOG_ERR, "tag")
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer w.Close()
			for i := 0; i < M; i++ {
				err := w.Info("test")
				if err != nil {
					t.Errorf("Info() failed: %v", err)
					return
				}
			}
		}()
	}
	wg.Wait()
	sock.Close()
	srvWG.Wait()
	close(done)

	select {
	case <-count:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout in concurrent reconnect")
	}
}
<commit_msg>log/syslog: avoid unix sockets on darwin/arm<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!nacl,!plan9

package syslog

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"runtime"
	"sync"
	"testing"
	"time"
)

func runPktSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string
	ct := 0
	for {
		var n int
		var err error

		c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
		n, _, err = c.ReadFrom(buf[:])
		rcvd += string(buf[:n])
		if err != nil {
			if oe, ok := err.(*net.OpError); ok {
				if ct < 3 && oe.Temporary() {
					ct++
					continue
				}
			}
			break
		}
	}
	c.Close()
	done <- rcvd
}

var crashy = false

func runStreamSyslog(l net.Listener, done chan<- string, wg *sync.WaitGroup) {
	for {
		var c net.Conn
		var err error
		if c, err = l.Accept(); err != nil {
			return
		}
		wg.Add(1)
		go func(c net.Conn) {
			defer wg.Done()
			c.SetReadDeadline(time.Now().Add(5 * time.Second))
			b := bufio.NewReader(c)
			for ct := 1; !crashy || ct&7 != 0; ct++ {
				s, err := b.ReadString('\n')
				if err != nil {
					break
				}
				done <- s
			}
			c.Close()
		}(c)
	}
}

func startServer(n, la string, done chan<- string) (addr string, sock io.Closer, wg *sync.WaitGroup) {
	if n == "udp" || n == "tcp" {
		la = "127.0.0.1:0"
	} else {
		// unix and unixgram: choose an address if none given
		if la == "" {
			// use ioutil.TempFile to get a name that is unique
			f, err := ioutil.TempFile("", "syslogtest")
			if err != nil {
				log.Fatal("TempFile: ", err)
			}
			f.Close()
			la = f.Name()
		}
		os.Remove(la)
	}

	wg = new(sync.WaitGroup)
	if n == "udp" || n == "unixgram" {
		l, e := net.ListenPacket(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.LocalAddr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runPktSyslog(l, done)
		}()
	} else {
		l, e := net.Listen(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.Addr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runStreamSyslog(l, done, wg)
		}()
	}
	return
}

func TestWithSimulated(t *testing.T) {
	msg := "Test 123"
	transport := []string{"unix", "unixgram", "udp", "tcp"}

	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm" {
		transport = []string{"udp", "tcp"}
	}

	for _, tr := range transport {
		done := make(chan string)
		addr, sock, srvWG := startServer(tr, "", done)
		defer srvWG.Wait()
		defer sock.Close()
		if tr == "unix" || tr == "unixgram" {
			defer os.Remove(addr)
		}
		s, err := Dial(tr, addr, LOG_INFO|LOG_USER, "syslog_test")
		if err != nil {
			t.Fatalf("Dial() failed: %v", err)
		}
		err = s.Info(msg)
		if err != nil {
			t.Fatalf("log failed: %v", err)
		}
		check(t, msg, <-done)
		s.Close()
	}
}

func TestFlap(t *testing.T) {
	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm" {
		t.Skipf("skipping on %s/%s", runtime.GOOS, runtime.GOARCH)
	}

	net := "unix"
	done := make(chan string)
	addr, sock, srvWG := startServer(net, "", done)
	defer srvWG.Wait()
	defer os.Remove(addr)
	defer sock.Close()

	s, err := Dial(net, addr, LOG_INFO|LOG_USER, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %v", err)
	}
	msg := "Moo 2"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	// restart the server
	_, sock2, srvWG2 := startServer(net, addr, done)
	defer srvWG2.Wait()
	defer sock2.Close()

	// and try retransmitting
	msg = "Moo 3"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	s.Close()
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Skip("skipping syslog test during -short")
	}

	s, err := New(LOG_INFO|LOG_USER, "the_tag")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func check(t *testing.T, in, out string) {
	tmpl := fmt.Sprintf("<%d>%%s %%s syslog_test[%%d]: %s\n", LOG_USER+LOG_INFO, in)
	if hostname, err := os.Hostname(); err != nil {
		t.Error("Error retrieving hostname")
	} else {
		var parsedHostname, timestamp string
		var pid int
		if n, err := fmt.Sscanf(out, tmpl, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
			t.Errorf("Got %q, does not match template %q (%d %s)", out, tmpl, n, err)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			addr, sock, srvWG := startServer("udp", "", done)
			defer srvWG.Wait()
			defer sock.Close()
			l, err := Dial("udp", addr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer l.Close()
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %v", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
				t.Errorf("s.Info() = '%q', didn't match '%q' (%d %s)", rcvd, test.exp, n, err)
			}
		}
	}
}

func TestConcurrentWrite(t *testing.T) {
	addr, sock, srvWG := startServer("udp", "", make(chan string, 1))
	defer srvWG.Wait()
	defer sock.Close()
	w, err := Dial("udp", addr, LOG_USER|LOG_ERR, "how's it going?")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %v", err)
	}
	var wg sync.WaitGroup
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			err := w.Info("test")
			if err != nil {
				t.Errorf("Info() failed: %v", err)
				return
			}
		}()
	}
	wg.Wait()
}

func TestConcurrentReconnect(t *testing.T) {
	crashy = true
	defer func() { crashy = false }()

	const N = 10
	const M = 100
	net := "unix"
	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm" {
		net = "tcp"
	}
	done := make(chan string, N*M)
	addr, sock, srvWG := startServer(net, "", done)
	if net == "unix" {
		defer os.Remove(addr)
	}

	// count all the messages arriving
	count := make(chan int)
	go func() {
		ct := 0
		for range done {
			ct++
			// we are looking for 500 out of 1000 events
			// here because lots of log messages are lost
			// in buffers (kernel and/or bufio)
			if ct > N*M/2 {
				break
			}
		}
		count <- ct
	}()

	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			w, err := Dial(net, addr, LOG_USER|LOG_ERR, "tag")
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer w.Close()
			for i := 0; i < M; i++ {
				err := w.Info("test")
				if err != nil {
					t.Errorf("Info() failed: %v", err)
					return
				}
			}
		}()
	}
	wg.Wait()
	sock.Close()
	srvWG.Wait()
	close(done)

	select {
	case <-count:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout in concurrent reconnect")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"

	"github.com/k4rtik/gopl/ch2/tempconv"
	"github.com/k4rtik/gopl/ch2/unitconv"
)

func main() {
	for _, arg := range os.Args[1:] {
		t, err := strconv.ParseFloat(arg, 64)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unitconv: %v\n", err)
			os.Exit(1)
		}
		f := tempconv.Fahrenheit(t)
		c := tempconv.Celsius(t)
		ft := unitconv.Feet(t)
		m := unitconv.Meters(t)
		p := unitconv.Pounds(t)
		k := unitconv.Kilograms(t)
		fmt.Printf("%s = %s, %s = %s\n", f, tempconv.FToC(f), c, tempconv.CToF(c))
		fmt.Printf("%s = %s, %s = %s\n", ft, unitconv.FToM(ft), m, unitconv.MToF(m))
		fmt.Printf("%s = %s, %s = %s\n", p, unitconv.PToK(p), k, unitconv.KToP(k))
	}
}
<commit_msg>add support for reading from stdin ex 2.2<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/k4rtik/gopl/ch2/tempconv"
	"github.com/k4rtik/gopl/ch2/unitconv"
)

func main() {
	input := strings.Join(os.Args[1:], " ")
	if len(os.Args) == 1 {
		iscanner := bufio.NewScanner(os.Stdin)
		iscanner.Scan()
		input = iscanner.Text()
	}
	for _, arg := range strings.Split(input, " ") {
		t, err := strconv.ParseFloat(arg, 64)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unitconv: %v\n", err)
			os.Exit(1)
		}
		f := tempconv.Fahrenheit(t)
		c := tempconv.Celsius(t)
		ft := unitconv.Feet(t)
		m := unitconv.Meters(t)
		p := unitconv.Pounds(t)
		k := unitconv.Kilograms(t)
		fmt.Printf("%s = %s, %s = %s\n", f, tempconv.FToC(f), c, tempconv.CToF(c))
		fmt.Printf("%s = %s, %s = %s\n", ft, unitconv.FToM(ft), m, unitconv.MToF(m))
		fmt.Printf("%s = %s, %s = %s\n", p, unitconv.PToK(p), k, unitconv.KToP(k))
	}
}
<|endoftext|>


<commit_before>package cwl

import (
	"testing"

	. "github.com/otiai10/mint"
)

func TestDecode_scatter_valuefrom_wf3(t *testing.T) {
	f := cwl("scatter-valuefrom-wf3.cwl")
	root := NewCWL()
	err := root.Decode(f)
	Expect(t, err).ToBe(err)

	Expect(t, root.Version).ToBe("v1.0")
	Expect(t, root.Graphs[0].ID).ToBe("echo")
	Expect(t, root.Graphs[0].Class).ToBe("CommandLineTool")
	Expect(t, root.Graphs[0].Inputs).ToBe("first")
	Expect(t, root.Graphs[0].Inputs[0].ID).ToBe("first")
	Expect(t, root.Graphs[0].Inputs[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[0].Binding.Position).ToBe(1)
	Expect(t, root.Graphs[0].Inputs[1].ID).ToBe("echo_in1")
	Expect(t, root.Graphs[0].Inputs[1].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[1].Binding.Position).ToBe(2)
	Expect(t, root.Graphs[0].Inputs[1].ID).ToBe("echo_in2")
	Expect(t, root.Graphs[0].Inputs[1].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[1].Binding.Position).ToBe(3)
	Expect(t, root.Graphs[0].Outputs[0].ID).ToBe("echo_out")
	Expect(t, root.Graphs[0].Outputs[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Outputs[0].Binding.Glob[0]).ToBe("step1_out")
	Expect(t, root.Graphs[0].Outputs[0].Binding.LoadContents).ToBe(true)
	Expect(t, root.Graphs[0].Outputs[0].Binding.Eval).ToBe("$(self[0].contents)")
	Expect(t, root.Graphs[0].BaseCommands[0]).ToBe("echo")
	Expect(t, root.Graphs[0].Arguments[0].Value).ToBe("-n")
	Expect(t, root.Graphs[0].Arguments[1].Value).ToBe("foo")
	Expect(t, root.Graphs[0].Stdout).ToBe("step1_out")

	Expect(t, root.Graphs[1].ID).ToBe("main")
	Expect(t, root.Graphs[1].Class).ToBe("Workflow")
	Expect(t, root.Graphs[1].Inputs[0].ID).ToBe("inp1")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Type).ToBe("record")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Fields[0].Name).ToBe("instr")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Fields[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[1].Inputs[1].ID).ToBe("inp2")
	Expect(t, root.Graphs[1].Inputs[1].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Inputs[1].Types[0].Items[0].Type).ToBe("string")
	Expect(t, root.Graphs[1].Requirements[0].Class).ToBe("ScatterFeatureRequirement")
	Expect(t, root.Graphs[1].Requirements[1].Class).ToBe("StepInputExpressionRequirement")
	Expect(t, root.Graphs[1].Steps[0].Scatter[0]).ToBe("echo_in1")
	Expect(t, root.Graphs[1].Steps[0].Scatter[1]).ToBe("echo_in2")
	Expect(t, root.Graphs[1].Steps[0].ScatterMethod).ToBe("flat_crossproduct")
	Expect(t, root.Graphs[1].Steps[0].ID).ToBe("step1")
	Expect(t, root.Graphs[1].Steps[0].In[0].ID).ToBe("echo_in1")
	Expect(t, root.Graphs[1].Steps[0].In[0].Source[0]).ToBe("inp1")
	Expect(t, root.Graphs[1].Steps[0].In[0].ValueFrom).ToBe("$(self.instr)")
	Expect(t, root.Graphs[1].Steps[0].In[1].ID).ToBe("echo_in2")
	Expect(t, root.Graphs[1].Steps[0].In[1].Source[0]).ToBe("inp2")
	Expect(t, root.Graphs[1].Steps[0].In[2].ID).ToBe("first")
	Expect(t, root.Graphs[1].Steps[0].In[2].Source[0]).ToBe("inp1")
	Expect(t, root.Graphs[1].Steps[0].In[2].ValueFrom).ToBe("$(self[0].instr)")
	Expect(t, root.Graphs[1].Steps[0].Out[0].ID).ToBe("echo_out")
	Expect(t, root.Graphs[1].Steps[0].Run.Value).ToBe("#echo")
	Expect(t, root.Graphs[1].Outputs[0].ID).ToBe("out")
	Expect(t, root.Graphs[1].Outputs[0].Source[0]).ToBe("step1/echo_out")
	Expect(t, root.Graphs[1].Outputs[0].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Outputs[0].Types[0].Items[0].Type).ToBe("string")
}
<commit_msg>Update test case<commit_after>package cwl

import (
	"testing"

	. "github.com/otiai10/mint"
)

func TestDecode_scatter_valuefrom_wf3(t *testing.T) {
	f := cwl("scatter-valuefrom-wf3.cwl")
	root := NewCWL()
	err := root.Decode(f)
	Expect(t, err).ToBe(err)

	Expect(t, root.Version).ToBe("v1.0")
	Expect(t, root.Graphs[0].ID).ToBe("echo")
	Expect(t, root.Graphs[0].Class).ToBe("CommandLineTool")
	Expect(t, root.Graphs[0].Inputs[0].ID).ToBe("first")
	Expect(t, root.Graphs[0].Inputs[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[0].Binding.Position).ToBe(1)
	Expect(t, root.Graphs[0].Inputs[1].ID).ToBe("echo_in1")
	Expect(t, root.Graphs[0].Inputs[1].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[1].Binding.Position).ToBe(2)
	Expect(t, root.Graphs[0].Inputs[1].ID).ToBe("echo_in2")
	Expect(t, root.Graphs[0].Inputs[1].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Inputs[1].Binding.Position).ToBe(3)
	Expect(t, root.Graphs[0].Outputs[0].ID).ToBe("echo_out")
	Expect(t, root.Graphs[0].Outputs[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[0].Outputs[0].Binding.Glob[0]).ToBe("step1_out")
	Expect(t, root.Graphs[0].Outputs[0].Binding.LoadContents).ToBe(true)
	Expect(t, root.Graphs[0].Outputs[0].Binding.Eval).ToBe("$(self[0].contents)")
	Expect(t, root.Graphs[0].BaseCommands[0]).ToBe("echo")
	Expect(t, root.Graphs[0].Arguments[0].Value).ToBe("-n")
	Expect(t, root.Graphs[0].Arguments[1].Value).ToBe("foo")
	Expect(t, root.Graphs[0].Stdout).ToBe("step1_out")

	Expect(t, root.Graphs[1].ID).ToBe("main")
	Expect(t, root.Graphs[1].Class).ToBe("Workflow")
	Expect(t, root.Graphs[1].Inputs[0].ID).ToBe("inp1")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Type).ToBe("record")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Fields[0].Name).ToBe("instr")
	Expect(t, root.Graphs[1].Inputs[0].Types[0].Items[0].Fields[0].Types[0].Type).ToBe("string")
	Expect(t, root.Graphs[1].Inputs[1].ID).ToBe("inp2")
	Expect(t, root.Graphs[1].Inputs[1].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Inputs[1].Types[0].Items[0].Type).ToBe("string")
	Expect(t, root.Graphs[1].Requirements[0].Class).ToBe("ScatterFeatureRequirement")
	Expect(t, root.Graphs[1].Requirements[1].Class).ToBe("StepInputExpressionRequirement")
	Expect(t, root.Graphs[1].Steps[0].Scatter[0]).ToBe("echo_in1")
	Expect(t, root.Graphs[1].Steps[0].Scatter[1]).ToBe("echo_in2")
	Expect(t, root.Graphs[1].Steps[0].ScatterMethod).ToBe("flat_crossproduct")
	Expect(t, root.Graphs[1].Steps[0].ID).ToBe("step1")
	Expect(t, root.Graphs[1].Steps[0].In[0].ID).ToBe("echo_in1")
	Expect(t, root.Graphs[1].Steps[0].In[0].Source[0]).ToBe("inp1")
	Expect(t, root.Graphs[1].Steps[0].In[0].ValueFrom).ToBe("$(self.instr)")
	Expect(t, root.Graphs[1].Steps[0].In[1].ID).ToBe("echo_in2")
	Expect(t, root.Graphs[1].Steps[0].In[1].Source[0]).ToBe("inp2")
	Expect(t, root.Graphs[1].Steps[0].In[2].ID).ToBe("first")
	Expect(t, root.Graphs[1].Steps[0].In[2].Source[0]).ToBe("inp1")
	Expect(t, root.Graphs[1].Steps[0].In[2].ValueFrom).ToBe("$(self[0].instr)")
	Expect(t, root.Graphs[1].Steps[0].Out[0].ID).ToBe("echo_out")
	Expect(t, root.Graphs[1].Steps[0].Run.Value).ToBe("#echo")
	Expect(t, root.Graphs[1].Outputs[0].ID).ToBe("out")
	Expect(t, root.Graphs[1].Outputs[0].Source[0]).ToBe("step1/echo_out")
	Expect(t, root.Graphs[1].Outputs[0].Types[0].Type).ToBe("array")
	Expect(t, root.Graphs[1].Outputs[0].Types[0].Items[0].Type).ToBe("string")
}
<|endoftext|>


<commit_before>// Obdi - a REST interface and GUI for deploying software
// Copyright (C) 2014  Mark Clarkson
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

// All api calls have the username and GUID to be sent as part of the request

import (
	"fmt"
	"github.com/mclarkson/obdi/external/ant0ine/go-json-rest/rest"
	"strconv"
)

func (api *Api) GetAllWorkers(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Anyone can view envs

	//session := Session{}
	var errl error = nil
	//if session,errl = api.CheckLogin( login, guid ); errl != nil {
	if _, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	qs := r.URL.Query() // Query string - map[string][]string

	jsonobjects := []JsonObject{}
	if len(qs["env_id"]) > 0 && len(qs["env_cap_id"]) > 0 {
		mutex.Lock()
		err := api.db.Order("env_id,env_cap_id").Find(&jsonobjects,
			"env_id = ? and env_cap_id = ?", qs["env_id"][0],
			qs["env_cap_id"][0])
		mutex.Unlock()
		if err.Error != nil {
			if !err.RecordNotFound() {
				rest.Error(w, err.Error.Error(), 500)
				return
			}
		}
	} else {
		mutex.Lock()
		err := api.db.Order("env_id,env_cap_id").Find(&jsonobjects)
		mutex.Unlock()
		if err.Error != nil {
			if !err.RecordNotFound() {
				rest.Error(w, err.Error.Error(), 500)
				return
			}
		}
	}

	// Create a slice of maps from users struct
	// to selectively copy database fields for display

	u := make([]map[string]interface{}, len(jsonobjects))
	for i := range jsonobjects {
		u[i] = make(map[string]interface{})
		u[i]["Id"] = jsonobjects[i].Id
		u[i]["EnvId"] = jsonobjects[i].EnvId
		u[i]["EnvCapId"] = jsonobjects[i].EnvCapId
		u[i]["Json"] = jsonobjects[i].Json

		env_caps := EnvCap{}
		mutex.Lock()
		api.db.Model(&jsonobjects[i]).Related(&env_caps)
		mutex.Unlock()

		u[i]["EnvCapCode"] = env_caps.Code
		u[i]["EnvCapDesc"] = env_caps.Desc
		u[i]["EnvCapId"] = env_caps.Id
	}

	// Too much noise
	//api.LogActivity( session.Id, "Sent list of users" )
	w.WriteJson(&u)
}

func (api *Api) AddWorker(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Can't add if it exists already

	jsonobject := JsonObject{}

	if err := r.DecodeJsonPayload(&jsonobject); err != nil {
		rest.Error(w, "Invalid data format received.", 400)
		return
	} else if len(jsonobject.Json) == 0 {
		rest.Error(w, "Incorrect data format received."+
			" Json is unset.", 400)
		return
	}
	{
		tmpjsonobject := JsonObject{}
		mutex.Lock()
		if !api.db.Find(&tmpjsonobject, "env_id = ? and env_cap_id = ?",
			jsonobject.EnvId, jsonobject.EnvCapId).RecordNotFound() {
			mutex.Unlock()
			rest.Error(w, "Record exists.", 400)
			return
		}
		mutex.Unlock()
	}

	// Add jsonobject

	mutex.Lock()
	if err := api.db.Save(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Added new jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)
	w.WriteJson(jsonobject)
}

func (api *Api) UpdateWorker(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Ensure user exists

	id := r.PathParam("id")

	// Check that the id string is a number
	if _, err := strconv.Atoi(id); err != nil {
		rest.Error(w, "Invalid id.", 400)
		return
	}

	// Load data from db, then ...
	jsonobject := JsonObject{}
	mutex.Lock()
	if api.db.Find(&jsonobject, id).RecordNotFound() {
		mutex.Unlock()
		rest.Error(w, "Record not found.", 400)
		return
	}
	mutex.Unlock()

	// ... overwrite any sent fields
	if err := r.DecodeJsonPayload(&jsonobject); err != nil {
		//rest.Error(w, err.Error(), 400)
		rest.Error(w, "Invalid data format received.", 400)
		return
	}

	// Force the use of the path id over an id in the payload
	Id, _ := strconv.Atoi(id)
	jsonobject.Id = int64(Id)

	mutex.Lock()
	if err := api.db.Save(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Updated jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)

	w.WriteJson("Success")
}

func (api *Api) DeleteWorker(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Delete

	id := 0
	if id, errl = strconv.Atoi(r.PathParam("id")); errl != nil {
		rest.Error(w, "Invalid id.", 400)
		return
	}

	jsonobject := JsonObject{}
	mutex.Lock()
	if api.db.First(&jsonobject, id).RecordNotFound() {
		mutex.Unlock()
		rest.Error(w, "Record not found.", 400)
		return
	}
	mutex.Unlock()

	mutex.Lock()
	if err := api.db.Delete(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Deleted jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)

	w.WriteJson("Success")
}
<commit_msg>Json Objects by capability (fixes)<commit_after>// Obdi - a REST interface and GUI for deploying software
// Copyright (C) 2014  Mark Clarkson
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

// All api calls have the username and GUID to be sent as part of the request

import (
	"fmt"
	"github.com/mclarkson/obdi/external/ant0ine/go-json-rest/rest"
	"strconv"
)

func (api *Api) GetAllJsonObjects(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Anyone can view envs

	//session := Session{}
	var errl error = nil
	//if session,errl = api.CheckLogin( login, guid ); errl != nil {
	if _, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	qs := r.URL.Query() // Query string - map[string][]string

	jsonobjects := []JsonObject{}
	if len(qs["env_id"]) > 0 && len(qs["env_cap_id"]) > 0 {
		mutex.Lock()
		err := api.db.Order("env_id,env_cap_id").Find(&jsonobjects,
			"env_id = ? and env_cap_id = ?", qs["env_id"][0],
			qs["env_cap_id"][0])
		mutex.Unlock()
		if err.Error != nil {
			if !err.RecordNotFound() {
				rest.Error(w, err.Error.Error(), 500)
				return
			}
		}
	} else {
		mutex.Lock()
		err := api.db.Order("env_id,env_cap_id").Find(&jsonobjects)
		mutex.Unlock()
		if err.Error != nil {
			if !err.RecordNotFound() {
				rest.Error(w, err.Error.Error(), 500)
				return
			}
		}
	}

	// Create a slice of maps from users struct
	// to selectively copy database fields for display

	u := make([]map[string]interface{}, len(jsonobjects))
	for i := range jsonobjects {
		u[i] = make(map[string]interface{})
		u[i]["Id"] = jsonobjects[i].Id
		u[i]["EnvId"] = jsonobjects[i].EnvId
		u[i]["EnvCapId"] = jsonobjects[i].EnvCapId
		u[i]["Json"] = jsonobjects[i].Json

		env_caps := EnvCap{}
		mutex.Lock()
		api.db.Model(&jsonobjects[i]).Related(&env_caps)
		mutex.Unlock()

		u[i]["EnvCapCode"] = env_caps.Code
		u[i]["EnvCapDesc"] = env_caps.Desc
		u[i]["EnvCapId"] = env_caps.Id
	}

	// Too much noise
	//api.LogActivity( session.Id, "Sent list of users" )
	w.WriteJson(&u)
}

func (api *Api) AddJsonObject(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Can't add if it exists already

	jsonobject := JsonObject{}

	if err := r.DecodeJsonPayload(&jsonobject); err != nil {
		rest.Error(w, "Invalid data format received.", 400)
		return
	} else if len(jsonobject.Json) == 0 {
		rest.Error(w, "Incorrect data format received."+
			" Json is unset.", 400)
		return
	}
	{
		tmpjsonobject := JsonObject{}
		mutex.Lock()
		if !api.db.Find(&tmpjsonobject, "env_id = ? and env_cap_id = ?",
			jsonobject.EnvId, jsonobject.EnvCapId).RecordNotFound() {
			mutex.Unlock()
			rest.Error(w, "Record exists.", 400)
			return
		}
		mutex.Unlock()
	}

	// Add jsonobject

	mutex.Lock()
	if err := api.db.Save(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Added new jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)
	w.WriteJson(jsonobject)
}

func (api *Api) UpdateJsonObject(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Ensure user exists

	id := r.PathParam("id")

	// Check that the id string is a number
	if _, err := strconv.Atoi(id); err != nil {
		rest.Error(w, "Invalid id.", 400)
		return
	}

	// Load data from db, then ...
	jsonobject := JsonObject{}
	mutex.Lock()
	if api.db.Find(&jsonobject, id).RecordNotFound() {
		mutex.Unlock()
		rest.Error(w, "Record not found.", 400)
		return
	}
	mutex.Unlock()

	// ... overwrite any sent fields
	if err := r.DecodeJsonPayload(&jsonobject); err != nil {
		//rest.Error(w, err.Error(), 400)
		rest.Error(w, "Invalid data format received.", 400)
		return
	}

	// Force the use of the path id over an id in the payload
	Id, _ := strconv.Atoi(id)
	jsonobject.Id = int64(Id)

	mutex.Lock()
	if err := api.db.Save(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Updated jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)

	w.WriteJson("Success")
}

func (api *Api) DeleteJsonObject(w rest.ResponseWriter, r *rest.Request) {

	// Check credentials

	login := r.PathParam("login")
	guid := r.PathParam("GUID")

	// Only admin is allowed

	if login != "admin" {
		rest.Error(w, "Not allowed", 400)
		return
	}

	session := Session{}
	var errl error
	if session, errl = api.CheckLogin(login, guid); errl != nil {
		rest.Error(w, errl.Error(), 401)
		return
	}

	defer api.TouchSession(guid)

	// Delete

	id := 0
	if id, errl = strconv.Atoi(r.PathParam("id")); errl != nil {
		rest.Error(w, "Invalid id.", 400)
		return
	}

	jsonobject := JsonObject{}
	mutex.Lock()
	if api.db.First(&jsonobject, id).RecordNotFound() {
		mutex.Unlock()
		rest.Error(w, "Record not found.", 400)
		return
	}
	mutex.Unlock()

	mutex.Lock()
	if err := api.db.Delete(&jsonobject).Error; err != nil {
		mutex.Unlock()
		rest.Error(w, err.Error(), 400)
		return
	}
	mutex.Unlock()

	env_cap := EnvCap{}
	mutex.Lock()
	api.db.First(&env_cap, jsonobject.EnvCapId)
	mutex.Unlock()

	text := fmt.Sprintf("Deleted jsonobject '%s->%s'.",
		env_cap.Desc, jsonobject.Json)

	api.LogActivity(session.Id, text)

	w.WriteJson("Success")
}
<|endoftext|>