<commit_before>package redis

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"

	"github.com/pivotal-cf/cf-redis-broker/system"
)

type fakeProcessChecker struct {
	alive          bool
	lastCheckedPid int
}

func (fakeProcessChecker *fakeProcessChecker) Alive(pid int) bool {
	fakeProcessChecker.lastCheckedPid = pid
	return fakeProcessChecker.alive
}

type fakeProcessKiller struct {
	killed        bool
	lastPidKilled int
}

func (fakeProcessKiller *fakeProcessKiller) Kill(pid int) error {
	fakeProcessKiller.lastPidKilled = pid
	fakeProcessKiller.killed = true
	return nil
}

type fakeInstanceInformer struct{}

func (*fakeInstanceInformer) InstancePid(instanceID string) (int, error) {
	return 123, nil
}

var _ = Describe("Redis Process Controller", func() {
	var (
		processController    *OSProcessController
		instance             *Instance = new(Instance)
		instanceInformer     *fakeInstanceInformer
		logger               *lagertest.TestLogger
		fakeProcessChecker   *fakeProcessChecker = new(fakeProcessChecker)
		fakeProcessKiller    *fakeProcessKiller  = new(fakeProcessKiller)
		commandRunner        *system.FakeCommandRunner
		connectionTimeoutErr error
	)

	BeforeEach(func() {
		connectionTimeoutErr = nil
		instanceInformer = new(fakeInstanceInformer)
		logger = lagertest.NewTestLogger("process-controller")
		commandRunner = new(system.FakeCommandRunner)
	})

	JustBeforeEach(func() {
		processController = &OSProcessController{
			Logger:           logger,
			InstanceInformer: instanceInformer,
			CommandRunner:    commandRunner,
			ProcessChecker:   fakeProcessChecker,
			ProcessKiller:    fakeProcessKiller,
			PingFunc: func(instance *Instance) error {
				return errors.New("what")
			},
			WaitUntilConnectableFunc: func(*net.TCPAddr, time.Duration) error {
				return connectionTimeoutErr
			},
		}
	})

	itStartsARedisProcess := func(executablePath string) {
		Ω(commandRunner.Commands).To(Equal([]string{
			fmt.Sprintf("%s configFilePath --pidfile pidFilePath --dir instanceDataDir --logfile logFilePath", executablePath),
		}))
	}

	Describe("StartAndWaitUntilReady", func() {
		It("runs the right command to start redis", func() {
			processController.StartAndWaitUntilReady(instance, "configFilePath", "instanceDataDir", "pidFilePath", "logFilePath", time.Second*1)
			itStartsARedisProcess("redis-server")
		})

		It("returns no error", func() {
			err := processController.StartAndWaitUntilReady(instance, "", "", "", "", time.Second*1)
			Ω(err).NotTo(HaveOccurred())
		})

		Context("when the redis process fails to start", func() {
			BeforeEach(func() {
				connectionTimeoutErr = errors.New("oops")
			})

			It("returns the same error that the WaitUntilConnectableFunc returns", func() {
				err := processController.StartAndWaitUntilReady(instance, "", "", "", "", time.Second*1)
				Ω(err).To(Equal(connectionTimeoutErr))
			})
		})
	})

	Describe("StartAndWaitUntilReadyWithConfig", func() {

		Context("When using a custom redis-server executable", func() {
			It("runs the right command to start redis", func() {
				processController.RedisServerExecutablePath = "custom/path/to/redis"

				args := []string{
					"configFilePath",
					"--pidfile", "pidFilePath",
					"--dir", "instanceDataDir",
					"--logfile", "logFilePath",
				}
				processController.StartAndWaitUntilReadyWithConfig(instance, args, time.Second*1)
				itStartsARedisProcess("custom/path/to/redis")
			})
		})

		It("runs the right command to start redis", func() {
			args := []string{
				"configFilePath",
				"--pidfile", "pidFilePath",
				"--dir", "instanceDataDir",
				"--logfile", "logFilePath",
			}
			processController.StartAndWaitUntilReadyWithConfig(instance, args, time.Second*1)
			itStartsARedisProcess("redis-server")
		})

		It("returns no error", func() {
			err := processController.StartAndWaitUntilReadyWithConfig(instance, []string{}, time.Second*1)
			Ω(err).NotTo(HaveOccurred())
		})

		Context("when the redis process fails to start", func() {
			BeforeEach(func() {
				connectionTimeoutErr = errors.New("oops")
			})

			It("returns the same error that the WaitUntilConnectableFunc returns", func() {
				err := processController.StartAndWaitUntilReadyWithConfig(instance, []string{}, time.Second*1)
				Ω(err).To(Equal(connectionTimeoutErr))
			})
		})
	})

	Describe("Kill", func() {
		It("kills the correct process", func() {
			err := processController.Kill(instance)
			Ω(err).NotTo(HaveOccurred())

			Ω(fakeProcessKiller.killed).Should(BeTrue())
			Ω(fakeProcessKiller.lastPidKilled).Should(Equal(123))
		})
	})

	Describe("EnsureRunning", func() {
		Context("if the process is already running", func() {
			var controller *OSProcessController
			var log *gbytes.Buffer

			BeforeEach(func() {
				fakeProcessChecker.alive = true
			})

			JustBeforeEach(func() {
				controller = processController
				log = gbytes.NewBuffer()
				controller.Logger.RegisterSink(lager.NewWriterSink(log, lager.DEBUG))
			})

			Context("and is a redis server", func() {
				Context("and is the correct redis instance", func() {
					var err error

					JustBeforeEach(func() {
						processController.PingFunc = func(instance *Instance) error {
							return nil
						}
						err = controller.EnsureRunning(instance, "", "", "", "")
					})

					It("does not return an error", func() {
						Expect(err).NotTo(HaveOccurred())
					})

					It("logs success", func() {
						Eventually(log).Should(gbytes.Say("redis instance already running"))
					})
				})

				Context("and is not the correct redis instance", func() {
					var err error
					var file *os.File

					BeforeEach(func() {
						var statErr error

						file, statErr = ioutil.TempFile("/tmp", "brokerTest")
						Expect(statErr).NotTo(HaveOccurred())
					})

					JustBeforeEach(func() {
						err = controller.EnsureRunning(instance, "/my/lovely/config", "", file.Name(), "")
					})

					It("does not return an error", func() {
						Expect(err).NotTo(HaveOccurred())
					})

					It("deletes the pid file", func() {
						_, statErr := os.Stat(file.Name())
						Expect(statErr).To(HaveOccurred())
					})

					It("logs the pidfile deletion", func() {
						Eventually(log).Should(gbytes.Say("removed stale pidfile"))
					})

					It("should restart the redis-server", func() {
						ran := false
						for _, command := range commandRunner.Commands {
							if strings.Contains(command, "redis-server /my/lovely/config") {
								ran = true
								break
							}
						}
						Expect(ran).To(BeTrue())
					})

					Context("and failed to delete pidfile", func() {
						JustBeforeEach(func() {
							err = controller.EnsureRunning(instance, "", "", "Pikachu", "")
						})

						It("returns an error", func() {
							Expect(err).To(HaveOccurred())
						})

						It("logs the error", func() {
							Eventually(log).Should(gbytes.Say("failed to delete stale pidfile"))
						})
					})
				})
			})

			Context("and is not a redis server", func() {
				var err error
				var file *os.File

				BeforeEach(func() {
					var statErr error

					file, statErr = ioutil.TempFile("/tmp", "brokerTest")
					Expect(statErr).NotTo(HaveOccurred())
				})

				JustBeforeEach(func() {
					err = controller.EnsureRunning(instance, "/my/config", "", file.Name(), "")
				})

				It("deletes the pidfile", func() {
					_, statErr := os.Stat(file.Name())
					Expect(statErr).To(HaveOccurred())
				})

				It("does not return an error", func() {
					Expect(err).NotTo(HaveOccurred())
				})

				It("logs the pidfile deletion", func() {
					Eventually(log).Should(gbytes.Say("removed stale pidfile"))
				})

				It("should restart the redis-server", func() {
					ran := false
					for _, command := range commandRunner.Commands {
						if strings.Contains(command, "redis-server /my/config") {
							ran = true
							break
						}
					}
					Expect(ran).To(BeTrue())
				})

				Context("and failed to delete pidfile", func() {
					JustBeforeEach(func() {
						err = controller.EnsureRunning(instance, "", "", "Pikachu", "")
					})

					It("returns an error", func() {
						Expect(err).To(HaveOccurred())
					})

					It("logs the error", func() {
						Eventually(log).Should(gbytes.Say("failed to delete stale pidfile"))
					})
				})
			})
		})

		Context("if the process is not already running", func() {
			BeforeEach(func() {
				fakeProcessChecker.alive = false
			})

			It("starts it", func() {
				err := processController.EnsureRunning(instance, "configFilePath", "instanceDataDir", "pidFilePath", "logFilePath")
				Ω(err).ShouldNot(HaveOccurred())

				itStartsARedisProcess("redis-server")
			})

			Context("and it can not be started", func() {
				BeforeEach(func() {
					commandRunner.RunError = errors.New("run error")
				})

				It("should return error", func() {
					err := processController.EnsureRunning(instance, "", "", "", "")
					Ω(err).Should(HaveOccurred())
				})
			})
		})
	})
})
<commit_msg>use consistent test language in process controller<commit_after>package redis

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"

	"github.com/pivotal-cf/cf-redis-broker/system"
)

type fakeProcessChecker struct {
	alive          bool
	lastCheckedPid int
}

func (fakeProcessChecker *fakeProcessChecker) Alive(pid int) bool {
	fakeProcessChecker.lastCheckedPid = pid
	return fakeProcessChecker.alive
}

type fakeProcessKiller struct {
	killed        bool
	lastPidKilled int
}

func (fakeProcessKiller *fakeProcessKiller) Kill(pid int) error {
	fakeProcessKiller.lastPidKilled = pid
	fakeProcessKiller.killed = true
	return nil
}

type fakeInstanceInformer struct{}

func (*fakeInstanceInformer) InstancePid(instanceID string) (int, error) {
	return 123, nil
}

var _ = Describe("Redis Process Controller", func() {
	var (
		processController    *OSProcessController
		instance             *Instance = new(Instance)
		instanceInformer     *fakeInstanceInformer
		logger               *lagertest.TestLogger
		fakeProcessChecker   *fakeProcessChecker = new(fakeProcessChecker)
		fakeProcessKiller    *fakeProcessKiller  = new(fakeProcessKiller)
		commandRunner        *system.FakeCommandRunner
		connectionTimeoutErr error
	)

	BeforeEach(func() {
		connectionTimeoutErr = nil
		instanceInformer = new(fakeInstanceInformer)
		logger = lagertest.NewTestLogger("process-controller")
		commandRunner = new(system.FakeCommandRunner)
	})

	JustBeforeEach(func() {
		processController = &OSProcessController{
			Logger:           logger,
			InstanceInformer: instanceInformer,
			CommandRunner:    commandRunner,
			ProcessChecker:   fakeProcessChecker,
			ProcessKiller:    fakeProcessKiller,
			PingFunc: func(instance *Instance) error {
				return errors.New("what")
			},
			WaitUntilConnectableFunc: func(*net.TCPAddr, time.Duration) error {
				return connectionTimeoutErr
			},
		}
	})

	itStartsARedisProcess := func(executablePath string) {
		Expect(commandRunner.Commands).To(Equal([]string{
			fmt.Sprintf("%s configFilePath --pidfile pidFilePath --dir instanceDataDir --logfile logFilePath", executablePath),
		}))
	}

	Describe("StartAndWaitUntilReady", func() {
		It("runs the right command to start redis", func() {
			processController.StartAndWaitUntilReady(instance, "configFilePath", "instanceDataDir", "pidFilePath", "logFilePath", time.Second*1)
			itStartsARedisProcess("redis-server")
		})

		It("returns no error", func() {
			err := processController.StartAndWaitUntilReady(instance, "", "", "", "", time.Second*1)
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when the redis process fails to start", func() {
			BeforeEach(func() {
				connectionTimeoutErr = errors.New("oops")
			})

			It("returns the same error that the WaitUntilConnectableFunc returns", func() {
				err := processController.StartAndWaitUntilReady(instance, "", "", "", "", time.Second*1)
				Expect(err).To(Equal(connectionTimeoutErr))
			})
		})
	})

	Describe("StartAndWaitUntilReadyWithConfig", func() {
		Context("When using a custom redis-server executable", func() {
			It("runs the right command to start redis", func() {
				processController.RedisServerExecutablePath = "custom/path/to/redis"

				args := []string{
					"configFilePath",
					"--pidfile", "pidFilePath",
					"--dir", "instanceDataDir",
					"--logfile", "logFilePath",
				}
				processController.StartAndWaitUntilReadyWithConfig(instance, args, time.Second*1)
				itStartsARedisProcess("custom/path/to/redis")
			})
		})

		It("runs the right command to start redis", func() {
			args := []string{
				"configFilePath",
				"--pidfile", "pidFilePath",
				"--dir", "instanceDataDir",
				"--logfile", "logFilePath",
			}
			processController.StartAndWaitUntilReadyWithConfig(instance, args, time.Second*1)
			itStartsARedisProcess("redis-server")
		})

		It("returns no error", func() {
			err := processController.StartAndWaitUntilReadyWithConfig(instance, []string{}, time.Second*1)
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when the redis process fails to start", func() {
			BeforeEach(func() {
				connectionTimeoutErr = errors.New("oops")
			})

			It("returns the same error that the WaitUntilConnectableFunc returns", func() {
				err := processController.StartAndWaitUntilReadyWithConfig(instance, []string{}, time.Second*1)
				Expect(err).To(Equal(connectionTimeoutErr))
			})
		})
	})

	Describe("Kill", func() {
		It("kills the correct process", func() {
			err := processController.Kill(instance)
			Expect(err).NotTo(HaveOccurred())

			Expect(fakeProcessKiller.killed).To(BeTrue())
			Expect(fakeProcessKiller.lastPidKilled).To(Equal(123))
		})
	})

	Describe("EnsureRunning", func() {
		Context("if the process is already running", func() {
			var (
				controller *OSProcessController
				log        *gbytes.Buffer
			)

			BeforeEach(func() {
				fakeProcessChecker.alive = true
			})

			JustBeforeEach(func() {
				controller = processController
				log = gbytes.NewBuffer()
				controller.Logger.RegisterSink(lager.NewWriterSink(log, lager.DEBUG))
			})

			Context("and is a redis server", func() {
				Context("and is the correct redis instance", func() {
					var err error

					JustBeforeEach(func() {
						processController.PingFunc = func(instance *Instance) error {
							return nil
						}
						err = controller.EnsureRunning(instance, "", "", "", "")
					})

					It("does not return an error", func() {
						Expect(err).NotTo(HaveOccurred())
					})

					It("logs success", func() {
						Eventually(log).Should(gbytes.Say("redis instance already running"))
					})
				})

				Context("and is not the correct redis instance", func() {
					var (
						err  error
						file *os.File
					)

					BeforeEach(func() {
						var statErr error

						file, statErr = ioutil.TempFile("/tmp", "brokerTest")
						Expect(statErr).NotTo(HaveOccurred())
					})

					JustBeforeEach(func() {
						err = controller.EnsureRunning(instance, "/my/lovely/config", "", file.Name(), "")
					})

					It("does not return an error", func() {
						Expect(err).NotTo(HaveOccurred())
					})

					It("deletes the pid file", func() {
						_, statErr := os.Stat(file.Name())
						Expect(statErr).To(HaveOccurred())
					})

					It("logs the pidfile deletion", func() {
						Eventually(log).Should(gbytes.Say("removed stale pidfile"))
					})

					It("should restart the redis-server", func() {
						ran := false
						for _, command := range commandRunner.Commands {
							if strings.Contains(command, "redis-server /my/lovely/config") {
								ran = true
								break
							}
						}
						Expect(ran).To(BeTrue())
					})

					Context("and failed to delete pidfile", func() {
						JustBeforeEach(func() {
							err = controller.EnsureRunning(instance, "", "", "Pikachu", "")
						})

						It("returns an error", func() {
							Expect(err).To(HaveOccurred())
						})

						It("logs the error", func() {
							Eventually(log).Should(gbytes.Say("failed to delete stale pidfile"))
						})
					})
				})
			})

			Context("and is not a redis server", func() {
				var (
					err  error
					file *os.File
				)

				BeforeEach(func() {
					var statErr error

					file, statErr = ioutil.TempFile("/tmp", "brokerTest")
					Expect(statErr).NotTo(HaveOccurred())
				})

				JustBeforeEach(func() {
					err = controller.EnsureRunning(instance, "/my/config", "", file.Name(), "")
				})

				It("deletes the pidfile", func() {
					_, statErr := os.Stat(file.Name())
					Expect(statErr).To(HaveOccurred())
				})

				It("does not return an error", func() {
					Expect(err).NotTo(HaveOccurred())
				})

				It("logs the pidfile deletion", func() {
					Eventually(log).Should(gbytes.Say("removed stale pidfile"))
				})

				It("should restart the redis-server", func() {
					ran := false
					for _, command := range commandRunner.Commands {
						if strings.Contains(command, "redis-server /my/config") {
							ran = true
							break
						}
					}
					Expect(ran).To(BeTrue())
				})

				Context("and failed to delete pidfile", func() {
					JustBeforeEach(func() {
						err = controller.EnsureRunning(instance, "", "", "Pikachu", "")
					})

					It("returns an error", func() {
						Expect(err).To(HaveOccurred())
					})

					It("logs the error", func() {
						Eventually(log).Should(gbytes.Say("failed to delete stale pidfile"))
					})
				})
			})
		})

		Context("if the process is not already running", func() {
			BeforeEach(func() {
				fakeProcessChecker.alive = false
			})

			It("starts it", func() {
				err := processController.EnsureRunning(instance, "configFilePath", "instanceDataDir", "pidFilePath", "logFilePath")
				Expect(err).NotTo(HaveOccurred())

				itStartsARedisProcess("redis-server")
			})

			Context("and it can not be started", func() {
				BeforeEach(func() {
					commandRunner.RunError = errors.New("run error")
				})

				It("should return error", func() {
					err := processController.EnsureRunning(instance, "", "", "", "")
					Expect(err).To(HaveOccurred())
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright ©2013 The bíogo.external Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package kmeans implements interaction with the kmeans clustering tool.
// The kmeans tool is available from http://www.cs.cmu.edu/~dpelleg/kmeans.html.
package kmeans

import (
	"code.google.com/p/biogo.external"

	"bufio"
	"errors"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var (
	ErrMissingRequired = errors.New("kmeans: missing required argument")
	ErrBadInput        = errors.New("kmeans: bad input")
	ErrNoUniverse      = errors.New("kmeans: no universe file")
)

type MakeUniverse struct {
	// Usage: kmeans makeuni in <infile>
	//
	Cmd string `buildarg:"{{if .}}{{.}}{{else}}kmeans{{end}}"` // kmeans

	// Make Universe:
	Kmeans struct{} `buildarg:"makeuni"` // makeuni

	// Files:
	Infile string `buildarg:"{{if .}}in{{split}}{{.}}{{end}}"` // in <file>
}

func (u MakeUniverse) BuildCommand() (*exec.Cmd, error) {
	if u.Infile == "" {
		return nil, ErrMissingRequired
	}
	cl := external.Must(external.Build(u))
	return exec.Command(cl[0], cl[1:]...), nil
}

type Xmeans struct {
	// Usage: kmeans kmeans [options] -in <infile>
	//
	Cmd string `buildarg:"{{if .}}{{.}}{{else}}kmeans{{end}}"` // kmeans

	// Kmeans:
	Kmeans struct{} `buildarg:"kmeans"` // kmeans

	// Files:
	InFile           string `buildarg:"{{if .}}-in{{split}}{{.}}{{end}}"`                     // -in <file>
	Constraints      string `buildarg:"{{if .}}-cons{{split}}{{.}}{{end}}"`                   // -cons <file>
	InitCenters      string `buildarg:"{{if .}}-init_ctrs{{split}}{{.}}{{end}}"`              // -init_ctrs <file>
	SaveCenters      string `buildarg:"{{if .}}-save_ctrs{{split}}{{.}}{{end}}"`              // -save_ctrs <file>
	PrintClusters    string `buildarg:"{{if .}}-printclusters{{split}}{{.}}{{end}}"`          // -printclusters <file>
	PrintNonClusters string `buildarg:"{{if .}}-print_no_cons_clusters{{split}}{{.}}{{end}}"` // -print_no_cons_clusters <file>

	// Options:
	InitialK         int     `buildarg:"{{if .}}-k{{split}}{{.}}{{end}}"`                     // -k <int>
	MaxCenters       int     `buildarg:"{{if .}}-max_ctrs{{split}}{{.}}{{end}}"`              // -max_ctrs <int>
	Method           string  `buildarg:"{{if .}}-method{{split}}{{.}}{{end}}"`                // -method <string>
	Splits           int     `buildarg:"{{if .}}-num_splits{{split}}{{.}}{{end}}"`            // -num_splits <int>
	DelSteps         int     `buildarg:"{{if .}}-del_steps_ratio{{split}}{{.}}{{end}}"`       // -del_steps_ratio <int>
	MaxIterations    int     `buildarg:"{{if .}}-max_iter{{split}}{{.}}{{end}}"`              // -max_iter <int>
	CutoffFactor     float64 `buildarg:"{{if .}}-cutoff_factor{{split}}{{.}}{{end}}"`         // -cutoff_factor <float>
	MaxLeafSize      int     `buildarg:"{{if .}}-max_leaf_size{{split}}{{.}}{{end}}"`         // -max_leaf_size <int>
	MinBoxWidth      float64 `buildarg:"{{if .}}-min_box_width{{split}}{{.}}{{end}}"`         // -min_box_width <float>
	CreateUniverse   bool    `buildarg:"{{if .}}-create_universe{{split}}true{{end}}"`        // -create_universe <bool>
	NeverKillCenters bool    `buildarg:"{{if .}}-never_kill_ctrs{{split}}true{{end}}"`        // -never_kill_ctrs <bool>
	SplitStatistic   string  `buildarg:"{{if .}}-S{{split}}{{.}}{{end}}"`                     // -S <string>
	Seed             int     `buildarg:"{{if .}}-seed{{split}}{{.}}{{end}}"`                  // -seed <int>
	RandStart        bool    `buildarg:"{{if .}}-randstart{{split}}true{{end}}"`              // -randstart <bool>
	ForceSplitFrac   float64 `buildarg:"{{if .}}-forced_split_fraction{{split}}{{.}}{{end}}"` // -forced_split_fraction <float>
	SplitConfLevel   float64 `buildarg:"{{if .}}-split_conf_level{{split}}{{.}}{{end}}"`      // -split_conf_level <float>

	// Display options:
	ShowEndCenters bool `buildarg:"{{if .}}-D_SHOW_END_CENTERS{{end}}"` // -D_SHOW_END_CENTERS
	DrawPoints     bool `buildarg:"{{if .}}-D_DRAWPOINTS{{end}}"`       // -D_DRAWPOINTS
	Interactive    bool `buildarg:"{{if .}}-D_INTERACTIVE{{end}}"`      // -D_INTERACTIVE
	ShowBValue     bool `buildarg:"{{if .}}-D_SHOW_BVALUE{{end}}"`      // -D_SHOW_BVALUE
}

func (x Xmeans) BuildCommand() (*exec.Cmd, error) {
	if x.InFile == "" {
		return nil, ErrMissingRequired
	}

	// We should be able to let kmeans fail this, but it waits for user input so
	// we just don't go there. Sane behaviour would require people patch their
	// kmeans source:
	//
	//  change void my_error_default(const char *string) in kmeans/utils/ambs.c
	//  to something less drastic.
	//
	_, err := os.Open(x.InFile + ".universe")
	if err != nil {
		return nil, ErrNoUniverse
	}

	cl := external.Must(external.Build(x))
	return exec.Command(cl[0], cl[1:]...), nil
}

func Membership(r io.Reader) ([]int, error) {
	var (
		currId  = -1
		lastID  = -1
		seen    = make(map[int]int)
		classes = make(map[int]int)
		pnts    int
	)

	buf := bufio.NewReader(r)
	for {
		l, err := buf.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				return nil, err
			}
			break
		}
		l = strings.TrimSpace(l)

		if len(l) < 2 || l[0] != '+' {
			return nil, ErrBadInput
		}

		if l[1] == '#' {
			l = strings.TrimSpace(l[2:])
			cname, err := strconv.Atoi(l)
			if err != nil {
				return nil, err
			}
			var ok bool
			if currId, ok = seen[cname]; !ok {
				lastID++
				currId = lastID
				seen[cname] = currId
			}
			continue
		}

		if currId < 0 {
			return nil, ErrBadInput
		}
		l = strings.TrimSpace(l[1:])
		pnt, err := strconv.Atoi(l)
		if err != nil {
			return nil, err
		}
		pnts++
		classes[pnt] = currId
	}

	if lastID <= 0 {
		return nil, nil
	}

	mi := make([]int, pnts)
	for i := range mi {
		v, ok := classes[i]
		if ok {
			mi[i] = v
		} else {
			mi[i] = -1
		}
	}

	return mi, nil
}
<commit_msg>Allow client to actually use the CreateUniverse option<commit_after>// Copyright ©2013 The bíogo.external Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package kmeans implements interaction with the kmeans clustering tool.
// The kmeans tool is available from http://www.cs.cmu.edu/~dpelleg/kmeans.html.
package kmeans

import (
	"code.google.com/p/biogo.external"

	"bufio"
	"errors"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var (
	ErrMissingRequired = errors.New("kmeans: missing required argument")
	ErrBadInput        = errors.New("kmeans: bad input")
	ErrNoUniverse      = errors.New("kmeans: no universe file")
)

type MakeUniverse struct {
	// Usage: kmeans makeuni in <infile>
	//
	Cmd string `buildarg:"{{if .}}{{.}}{{else}}kmeans{{end}}"` // kmeans

	// Make Universe:
	Kmeans struct{} `buildarg:"makeuni"` // makeuni

	// Files:
	Infile string `buildarg:"{{if .}}in{{split}}{{.}}{{end}}"` // in <file>
}

func (u MakeUniverse) BuildCommand() (*exec.Cmd, error) {
	if u.Infile == "" {
		return nil, ErrMissingRequired
	}
	cl := external.Must(external.Build(u))
	return exec.Command(cl[0], cl[1:]...), nil
}

type Xmeans struct {
	// Usage: kmeans kmeans [options] -in <infile>
	//
	Cmd string `buildarg:"{{if .}}{{.}}{{else}}kmeans{{end}}"` // kmeans

	// Kmeans:
	Kmeans struct{} `buildarg:"kmeans"` // kmeans

	// Files:
	InFile           string `buildarg:"{{if .}}-in{{split}}{{.}}{{end}}"`                     // -in <file>
	Constraints      string `buildarg:"{{if .}}-cons{{split}}{{.}}{{end}}"`                   // -cons <file>
	InitCenters      string `buildarg:"{{if .}}-init_ctrs{{split}}{{.}}{{end}}"`              // -init_ctrs <file>
	SaveCenters      string `buildarg:"{{if .}}-save_ctrs{{split}}{{.}}{{end}}"`              // -save_ctrs <file>
	PrintClusters    string `buildarg:"{{if .}}-printclusters{{split}}{{.}}{{end}}"`          // -printclusters <file>
	PrintNonClusters string `buildarg:"{{if .}}-print_no_cons_clusters{{split}}{{.}}{{end}}"` // -print_no_cons_clusters <file>

	// Options:
	InitialK         int     `buildarg:"{{if .}}-k{{split}}{{.}}{{end}}"`                     // -k <int>
	MaxCenters       int     `buildarg:"{{if .}}-max_ctrs{{split}}{{.}}{{end}}"`              // -max_ctrs <int>
	Method           string  `buildarg:"{{if .}}-method{{split}}{{.}}{{end}}"`                // -method <string>
	Splits           int     `buildarg:"{{if .}}-num_splits{{split}}{{.}}{{end}}"`            // -num_splits <int>
	DelSteps         int     `buildarg:"{{if .}}-del_steps_ratio{{split}}{{.}}{{end}}"`       // -del_steps_ratio <int>
	MaxIterations    int     `buildarg:"{{if .}}-max_iter{{split}}{{.}}{{end}}"`              // -max_iter <int>
	CutoffFactor     float64 `buildarg:"{{if .}}-cutoff_factor{{split}}{{.}}{{end}}"`         // -cutoff_factor <float>
	MaxLeafSize      int     `buildarg:"{{if .}}-max_leaf_size{{split}}{{.}}{{end}}"`         // -max_leaf_size <int>
	MinBoxWidth      float64 `buildarg:"{{if .}}-min_box_width{{split}}{{.}}{{end}}"`         // -min_box_width <float>
	CreateUniverse   bool    `buildarg:"{{if .}}-create_universe{{split}}true{{end}}"`        // -create_universe <bool>
	NeverKillCenters bool    `buildarg:"{{if .}}-never_kill_ctrs{{split}}true{{end}}"`        // -never_kill_ctrs <bool>
	SplitStatistic   string  `buildarg:"{{if .}}-S{{split}}{{.}}{{end}}"`                     // -S <string>
	Seed             int     `buildarg:"{{if .}}-seed{{split}}{{.}}{{end}}"`                  // -seed <int>
	RandStart        bool    `buildarg:"{{if .}}-randstart{{split}}true{{end}}"`              // -randstart <bool>
	ForceSplitFrac   float64 `buildarg:"{{if .}}-forced_split_fraction{{split}}{{.}}{{end}}"` // -forced_split_fraction <float>
	SplitConfLevel   float64 `buildarg:"{{if .}}-split_conf_level{{split}}{{.}}{{end}}"`      // -split_conf_level <float>

	// Display options:
	ShowEndCenters bool `buildarg:"{{if .}}-D_SHOW_END_CENTERS{{end}}"` // -D_SHOW_END_CENTERS
	DrawPoints     bool `buildarg:"{{if .}}-D_DRAWPOINTS{{end}}"`       // -D_DRAWPOINTS
	Interactive    bool `buildarg:"{{if .}}-D_INTERACTIVE{{end}}"`      // -D_INTERACTIVE
	ShowBValue     bool `buildarg:"{{if .}}-D_SHOW_BVALUE{{end}}"`      // -D_SHOW_BVALUE
}

func (x Xmeans) BuildCommand() (*exec.Cmd, error) {
	if x.InFile == "" {
		return nil, ErrMissingRequired
	}

	// We should be able to let kmeans fail this, but it waits for user input so
	// we just don't go there. Sane behaviour would require people patch their
	// kmeans source:
	//
	//  change void my_error_default(const char *string) in kmeans/utils/ambs.c
	//  to something less drastic.
	//
	if !x.CreateUniverse {
		_, err := os.Open(x.InFile + ".universe")
		if err != nil {
			return nil, ErrNoUniverse
		}
	}

	cl := external.Must(external.Build(x))
	return exec.Command(cl[0], cl[1:]...), nil
}

func Membership(r io.Reader) ([]int, error) {
	var (
		currId  = -1
		lastID  = -1
		seen    = make(map[int]int)
		classes = make(map[int]int)
		pnts    int
	)

	buf := bufio.NewReader(r)
	for {
		l, err := buf.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				return nil, err
			}
			break
		}
		l = strings.TrimSpace(l)

		if len(l) < 2 || l[0] != '+' {
			return nil, ErrBadInput
		}

		if l[1] == '#' {
			l = strings.TrimSpace(l[2:])
			cname, err := strconv.Atoi(l)
			if err != nil {
				return nil, err
			}
			var ok bool
			if currId, ok = seen[cname]; !ok {
				lastID++
				currId = lastID
				seen[cname] = currId
			}
			continue
		}

		if currId < 0 {
			return nil, ErrBadInput
		}
		l = strings.TrimSpace(l[1:])
		pnt, err := strconv.Atoi(l)
		if err != nil {
			return nil, err
		}
		pnts++
		classes[pnt] = currId
	}

	if lastID <= 0 {
		return nil, nil
	}

	mi := make([]int, pnts)
	for i := range mi {
		v, ok := classes[i]
		if ok {
			mi[i] = v
		} else {
			mi[i] = -1
		}
	}

	return mi, nil
}
<|endoftext|>


<commit_before>package adapter

import (
	"fmt"
	"github.com/stitchfix/flotilla-os/state"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"time"
)

type EKSAdapter interface {
	AdaptJobToFlotillaRun(job *batchv1.Job, run state.Run) (state.Run, error)
	AdaptFlotillaDefinitionAndRunToJob(td state.Definition, run state.Run, sa string) (batchv1.Job, error)
}
type eksAdapter struct{}

//
// NewEKSAdapter configures and returns an eks adapter for translating
// from EKS api specific objects to our representation
//
func NewEKSAdapter() (EKSAdapter, error) {
	adapter := eksAdapter{}

	return &adapter, nil
}

// TODO: figure this out later.
func (a *eksAdapter) AdaptJobToFlotillaRun(job *batchv1.Job, run state.Run) (state.Run, error) {
	updated := run
	if job.Status.Active == 1 {
		updated.Status = state.StatusRunning
	} else if job.Status.Succeeded == 1 {
		var exitCode int64 = 0
		updated.Status = state.StatusStopped
		updated.ExitCode = &exitCode
	} else if job.Status.Failed == 1 {
		var exitCode int64 = 1
		updated.Status = state.StatusStopped
		updated.ExitCode = &exitCode
	}

	if job != nil && job.Status.StartTime != nil {
		updated.StartedAt = &job.Status.StartTime.Time
	}

	if updated.Status == state.StatusStopped {
		if job != nil && job.Status.CompletionTime != nil {
			updated.FinishedAt = &job.Status.CompletionTime.Time
		} else {
			finishedAt := time.Now()
			updated.FinishedAt = &finishedAt
		}
	}
	return updated, nil
}

func (a *eksAdapter) AdaptFlotillaDefinitionAndRunToJob(definition state.Definition, run state.Run, sa string) (batchv1.Job, error) {
	resourceRequirements := a.constructResourceRequirements(definition, run)

	cmd := definition.Command
	if run.Command != nil {
		cmd = *run.Command
	}

	container := corev1.Container{
		Name:      run.RunID,
		Image:     run.Image,
		Command:   a.constructCmdSlice(cmd),
		Resources: resourceRequirements,
		Env:       a.envOverrides(definition, run),
	}

	nodeLifecycle := state.SpotLifecycle

	if run.NodeLifecycle != nil {
		nodeLifecycle = *run.NodeLifecycle
	}

	lifecycle := "kubernetes.io/lifecycle"

	jobSpec := batchv1.JobSpec{
		TTLSecondsAfterFinished: &state.TTLSecondsAfterFinished,
		ActiveDeadlineSeconds:   &state.ActiveDeadlineSeconds,
		BackoffLimit:            &state.EKSBackoffLimit,

		Template: corev1.PodTemplateSpec{
			Spec: corev1.PodSpec{
				Containers:         []corev1.Container{container},
				RestartPolicy:      corev1.RestartPolicyNever,
				ServiceAccountName: sa,
				NodeSelector: map[string]string{
					lifecycle: nodeLifecycle,
				},
			},
		},
	}

	eksJob := batchv1.Job{
		Spec: jobSpec,
		ObjectMeta: v1.ObjectMeta{
			Name: run.RunID,
		},
	}

	return eksJob, nil
}

func (a *eksAdapter) constructResourceRequirements(definition state.Definition, run state.Run) corev1.ResourceRequirements {
	limits := make(corev1.ResourceList)
	cpu := *definition.Cpu

	if run.Cpu != nil {
		cpu = *run.Cpu
	}
	if cpu < state.MinCPU {
		cpu = state.MinCPU
	}

	mem := *definition.Memory
	if run.Memory != nil {
		mem = *run.Memory
	}
	if mem < state.MinMem {
		mem = state.MinMem

	}

	cpuQuantity := resource.MustParse(fmt.Sprintf("%dm", cpu))
	memoryQuantity := resource.MustParse(fmt.Sprintf("%dMi", mem))

	if definition.Gpu != nil {
		limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", *definition.Gpu))
	}
	if run.EphemeralStorage != nil {
		limits[corev1.ResourceEphemeralStorage] =
			resource.MustParse(fmt.Sprintf("%dGi", *run.EphemeralStorage))
	}
	limits[corev1.ResourceCPU] = cpuQuantity
	limits[corev1.ResourceMemory] = memoryQuantity
	resourceRequirements := corev1.ResourceRequirements{
		Limits: limits,
	}
	return resourceRequirements
}

func (a *eksAdapter) constructCmdSlice(cmdString string) []string {
	bashCmd := "bash"
	optLogin := "-l"
	optStr := "-cex"
	return []string{bashCmd, optLogin, optStr, cmdString}
}

func (a *eksAdapter) envOverrides(definition state.Definition, run state.Run) []corev1.EnvVar {
	pairs := make(map[string]string)
	for _, ev := range *definition.Env {
		name := ev.Name
		value := ev.Value
		pairs[name] = value
	}

	for _, ev := range *run.Env {
		name := ev.Name
		value := ev.Value
		pairs[name] = value
	}

	var res []corev1.EnvVar
	for key := range pairs {
		if len(key) > 0 {
			res = append(res, corev1.EnvVar{
				Name:  key,
				Value: pairs[key],
			})
		}
	}
	return res
}
<commit_msg>remove Node selector (#179)<commit_after>package adapter

import (
	"fmt"
	"github.com/stitchfix/flotilla-os/state"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"time"
)

type EKSAdapter interface {
	AdaptJobToFlotillaRun(job *batchv1.Job, run state.Run) (state.Run, error)
	AdaptFlotillaDefinitionAndRunToJob(td state.Definition, run state.Run, sa string) (batchv1.Job, error)
}
type eksAdapter struct{}

//
// NewEKSAdapter configures and returns an eks adapter for translating
// from EKS api specific objects to our representation
//
func NewEKSAdapter() (EKSAdapter, error) {
	adapter := eksAdapter{}

	return &adapter, nil
}

// TODO: figure this out later.
func (a *eksAdapter) AdaptJobToFlotillaRun(job *batchv1.Job, run state.Run) (state.Run, error) {
	updated := run
	if job.Status.Active == 1 {
		updated.Status = state.StatusRunning
	} else if job.Status.Succeeded == 1 {
		var exitCode int64 = 0
		updated.Status = state.StatusStopped
		updated.ExitCode = &exitCode
	} else if job.Status.Failed == 1 {
		var exitCode int64 = 1
		updated.Status = state.StatusStopped
		updated.ExitCode = &exitCode
	}

	if job != nil && job.Status.StartTime != nil {
		updated.StartedAt = &job.Status.StartTime.Time
	}

	if updated.Status == state.StatusStopped {
		if job != nil && job.Status.CompletionTime != nil {
			updated.FinishedAt = &job.Status.CompletionTime.Time
		} else {
			finishedAt := time.Now()
			updated.FinishedAt = &finishedAt
		}
	}
	return updated, nil
}

func (a *eksAdapter) AdaptFlotillaDefinitionAndRunToJob(definition state.Definition, run state.Run, sa string) (batchv1.Job, error) {
	resourceRequirements := a.constructResourceRequirements(definition, run)

	cmd := definition.Command
	if run.Command != nil {
		cmd = *run.Command
	}

	container := corev1.Container{
		Name:      run.RunID,
		Image:     run.Image,
		Command:   a.constructCmdSlice(cmd),
		Resources: resourceRequirements,
		Env:       a.envOverrides(definition, run),
	}

	//nodeLifecycle := state.SpotLifecycle

	//if run.NodeLifecycle != nil {
	//	nodeLifecycle = *run.NodeLifecycle
	//}

	//lifecycle := "kubernetes.io/lifecycle"

	jobSpec := batchv1.JobSpec{
		TTLSecondsAfterFinished: &state.TTLSecondsAfterFinished,
		ActiveDeadlineSeconds:   &state.ActiveDeadlineSeconds,
		BackoffLimit:            &state.EKSBackoffLimit,

		Template: corev1.PodTemplateSpec{
			Spec: corev1.PodSpec{
				Containers:         []corev1.Container{container},
				RestartPolicy:      corev1.RestartPolicyNever,
				ServiceAccountName: sa,
				//NodeSelector: map[string]string{
				//	lifecycle: nodeLifecycle,
				//},
			},
		},
	}

	eksJob := batchv1.Job{
		Spec: jobSpec,
		ObjectMeta: v1.ObjectMeta{
			Name: run.RunID,
		},
	}

	return eksJob, nil
}

func (a *eksAdapter) constructResourceRequirements(definition state.Definition, run state.Run) corev1.ResourceRequirements {
	limits := make(corev1.ResourceList)
	cpu := *definition.Cpu

	if run.Cpu != nil {
		cpu = *run.Cpu
	}
	if cpu < state.MinCPU {
		cpu = state.MinCPU
	}

	mem := *definition.Memory
	if run.Memory != nil {
		mem = *run.Memory
	}
	if mem < state.MinMem {
		mem = state.MinMem

	}

	cpuQuantity := resource.MustParse(fmt.Sprintf("%dm", cpu))
	memoryQuantity := resource.MustParse(fmt.Sprintf("%dMi", mem))

	if definition.Gpu != nil {
		limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", *definition.Gpu))
	}
	if run.EphemeralStorage != nil {
		limits[corev1.ResourceEphemeralStorage] =
			resource.MustParse(fmt.Sprintf("%dGi", *run.EphemeralStorage))
	}
	limits[corev1.ResourceCPU] = cpuQuantity
	limits[corev1.ResourceMemory] = memoryQuantity
	resourceRequirements := corev1.ResourceRequirements{
		Limits: limits,
	}
	return resourceRequirements
}

func (a *eksAdapter) constructCmdSlice(cmdString string) []string {
	bashCmd := "bash"
	optLogin := "-l"
	optStr := "-cex"
	return []string{bashCmd, optLogin, optStr, cmdString}
}

func (a *eksAdapter) envOverrides(definition state.Definition, run state.Run) []corev1.EnvVar {
	pairs := make(map[string]string)
	for _, ev := range *definition.Env {
		name := ev.Name
		value := ev.Value
		pairs[name] = value
	}

	for _, ev := range *run.Env {
		name := ev.Name
		value := ev.Value
		pairs[name] = value
	}

	var res []corev1.EnvVar
	for key := range pairs {
		if len(key) > 0 {
			res = append(res, corev1.EnvVar{
				Name:  key,
				Value: pairs[key],
			})
		}
	}
	return res
}
<|endoftext|>


<commit_before>package langserver

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"sync/atomic"

	opentracing "github.com/opentracing/opentracing-go"

	"github.com/sourcegraph/ctxvfs"
	"github.com/sourcegraph/go-langserver/pkg/lsp"
	"github.com/sourcegraph/jsonrpc2"
)

// IsFileSystemRequest returns if this is an LSP method whose sole
// purpose is modifying the contents of the overlay file system.
func IsFileSystemRequest(method string) bool {
	return method == "textDocument/didOpen" ||
		method == "textDocument/didChange" ||
		method == "textDocument/didClose" ||
		method == "textDocument/didSave"
}

func (h *HandlerShared) HandleFileSystemRequest(ctx context.Context, req *jsonrpc2.Request) error {
	span := opentracing.SpanFromContext(ctx)

	switch req.Method {
	case "textDocument/didOpen":
		var params lsp.DidOpenTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		span.SetTag("uri", params.TextDocument.URI)
		h.addOverlayFile(params.TextDocument.URI, []byte(params.TextDocument.Text))
		return nil

	case "textDocument/didChange":
		var params lsp.DidChangeTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		contents, found := h.readOverlayFile(params.TextDocument.URI)
		if !found {
			return fmt.Errorf("received textDocument/didChange for unknown file %q", params.TextDocument.URI)
		}
		for _, change := range params.ContentChanges {
			switch {
			case change.Range == nil && change.RangeLength == 0:
				contents = []byte(change.Text) // new full content

			default:
				return fmt.Errorf("incremental updates in textDocument/didChange not supported for file %q", params.TextDocument.URI)
			}
		}
		h.addOverlayFile(params.TextDocument.URI, contents)
		return nil

	case "textDocument/didClose":
		var params lsp.DidCloseTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		h.removeOverlayFile(params.TextDocument.URI)
		return nil

	case "textDocument/didSave":
		// no-op
		return nil

	default:
		panic("unexpected file system request method: " + req.Method)
	}
}

func (h *HandlerShared) FilePath(uri string) string {
	path := uriToPath(uri)
	if !strings.HasPrefix(path, "/") {
		panic(fmt.Sprintf("bad uri %q (path %q MUST have leading slash; it can't be relative)", uri, path))
	}
	return path
}

func (h *HandlerShared) readFile(ctx context.Context, uri string) ([]byte, error) {
	h.Mu.Lock()
	fs := h.FS
	h.Mu.Unlock()
	path := h.FilePath(uri)
	contents, err := ctxvfs.ReadFile(ctx, fs, path)
	if os.IsNotExist(err) {
		if _, ok := err.(*os.PathError); !ok {
			err = &os.PathError{Op: "Open", Path: path, Err: err}
		}
	}
	return contents, err
}

func uriToOverlayPath(uri string) string {
	if isURI(uri) {
		return strings.TrimPrefix(uriToPath(uri), "/")
	}
	return uri
}

func (h *HandlerShared) addOverlayFile(uri string, contents []byte) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	h.overlayFS[uriToOverlayPath(uri)] = contents
}

func (h *HandlerShared) removeOverlayFile(uri string) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	delete(h.overlayFS, uriToOverlayPath(uri))
}

func (h *HandlerShared) readOverlayFile(uri string) (contents []byte, found bool) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	contents, found = h.overlayFS[uriToOverlayPath(uri)]
	return
}

// AtomicFS wraps a ctxvfs.NameSpace but is safe for concurrent calls to Bind
// while doing FS operations. It is optimized for "ReadMostly" use-case. IE
// Bind is a relatively rare call compared to actual FS operations.
type AtomicFS struct {
	mu sync.Mutex   // serialize calls to Bind (ie only used by writers)
	v  atomic.Value // stores the ctxvfs.NameSpace
}

// NewAtomicFS returns an AtomicFS with an empty wrapped ctxvfs.NameSpace
func NewAtomicFS() *AtomicFS {
	fs := &AtomicFS{}
	fs.v.Store(make(ctxvfs.NameSpace))
	return fs
}

// Bind wraps ctxvfs.NameSpace.Bind
func (a *AtomicFS) Bind(old string, newfs ctxvfs.FileSystem, new string, mode ctxvfs.BindMode) {
	// We do copy-on-write
	a.mu.Lock()
	defer a.mu.Unlock()

	fs1 := a.v.Load().(ctxvfs.NameSpace)
	fs2 := make(ctxvfs.NameSpace)
	for k, v := range fs1 {
		fs2[k] = v
	}
	fs2.Bind(old, newfs, new, mode)
	a.v.Store(fs2)
}

func (*AtomicFS) String() string {
	return "atomicfs"
}

// Open wraps ctxvfs.NameSpace.Open
func (a *AtomicFS) Open(ctx context.Context, path string) (ctxvfs.ReadSeekCloser, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Open(ctx, path)
}

// Stat wraps ctxvfs.NameSpace.Stat
func (a *AtomicFS) Stat(ctx context.Context, path string) (os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Stat(ctx, path)
}

// Lstat wraps ctxvfs.NameSpace.Lstat
func (a *AtomicFS) Lstat(ctx context.Context, path string) (os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Lstat(ctx, path)
}

// ReadDir wraps ctxvfs.NameSpace.ReadDir
func (a *AtomicFS) ReadDir(ctx context.Context, path string) ([]os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.ReadDir(ctx, path)
}
<commit_msg>fs: Support incremental file updates (#142)<commit_after>package langserver

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"sync/atomic"

	opentracing "github.com/opentracing/opentracing-go"

	"github.com/sourcegraph/ctxvfs"
	"github.com/sourcegraph/go-langserver/pkg/lsp"
	"github.com/sourcegraph/jsonrpc2"
)

// IsFileSystemRequest returns if this is an LSP method whose sole
// purpose is modifying the contents of the overlay file system.
func IsFileSystemRequest(method string) bool {
	return method == "textDocument/didOpen" ||
		method == "textDocument/didChange" ||
		method == "textDocument/didClose" ||
		method == "textDocument/didSave"
}

func (h *HandlerShared) HandleFileSystemRequest(ctx context.Context, req *jsonrpc2.Request) error {
	span := opentracing.SpanFromContext(ctx)

	switch req.Method {
	case "textDocument/didOpen":
		var params lsp.DidOpenTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		span.SetTag("uri", params.TextDocument.URI)
		h.addOverlayFile(params.TextDocument.URI, []byte(params.TextDocument.Text))
		return nil

	case "textDocument/didChange":
		var params lsp.DidChangeTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		contents, found := h.readOverlayFile(params.TextDocument.URI)
		if !found {
			return fmt.Errorf("received textDocument/didChange for unknown file %q", params.TextDocument.URI)
		}

		for _, change := range params.ContentChanges {
			if change.Range == nil && change.RangeLength == 0 {
				contents = []byte(change.Text) // new full content
				continue
			}
			start, ok, why := offsetForPosition(contents, change.Range.Start)
			if !ok {
				return fmt.Errorf("received textDocument/didChange for invalid position %q on %q: %s", change.Range.Start, params.TextDocument.URI, why)
			}
			var end int
			if change.RangeLength != 0 {
				end = start + int(change.RangeLength) - 1
			} else {
				// RangeLength not specified, work it out from Range.End
				end, ok, why = offsetForPosition(contents, change.Range.End)
				if !ok {
					return fmt.Errorf("received textDocument/didChange for invalid position %q on %q: %s", change.Range.Start, params.TextDocument.URI, why)
				}
			}
			if start < 0 || end >= len(contents) || end < start {
				return fmt.Errorf("received textDocument/didChange for out of range position %q on %q", change.Range, params.TextDocument.URI)
			}
			// Try avoid doing too many allocations, so use bytes.Buffer
			b := &bytes.Buffer{}
			b.Grow(start + len(change.Text) + len(contents) - end - 1)
			b.Write(contents[:start])
			b.WriteString(change.Text)
			b.Write(contents[end+1:])
			contents = b.Bytes()
		}
		h.addOverlayFile(params.TextDocument.URI, contents)
		return nil

	case "textDocument/didClose":
		var params lsp.DidCloseTextDocumentParams
		if err := json.Unmarshal(*req.Params, &params); err != nil {
			return err
		}
		h.removeOverlayFile(params.TextDocument.URI)
		return nil

	case "textDocument/didSave":
		// no-op
		return nil

	default:
		panic("unexpected file system request method: " + req.Method)
	}
}

func (h *HandlerShared) FilePath(uri string) string {
	path := uriToPath(uri)
	if !strings.HasPrefix(path, "/") {
		panic(fmt.Sprintf("bad uri %q (path %q MUST have leading slash; it can't be relative)", uri, path))
	}
	return path
}

func (h *HandlerShared) readFile(ctx context.Context, uri string) ([]byte, error) {
	h.Mu.Lock()
	fs := h.FS
	h.Mu.Unlock()
	path := h.FilePath(uri)
	contents, err := ctxvfs.ReadFile(ctx, fs, path)
	if os.IsNotExist(err) {
		if _, ok := err.(*os.PathError); !ok {
			err = &os.PathError{Op: "Open", Path: path, Err: err}
		}
	}
	return contents, err
}

func uriToOverlayPath(uri string) string {
	if isURI(uri) {
		return strings.TrimPrefix(uriToPath(uri), "/")
	}
	return uri
}

func (h *HandlerShared) addOverlayFile(uri string, contents []byte) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	h.overlayFS[uriToOverlayPath(uri)] = contents
}

func (h *HandlerShared) removeOverlayFile(uri string) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	delete(h.overlayFS, uriToOverlayPath(uri))
}

func (h *HandlerShared) readOverlayFile(uri string) (contents []byte, found bool) {
	h.Mu.Lock()
	defer h.Mu.Unlock()
	h.overlayFSMu.Lock()
	defer h.overlayFSMu.Unlock()
	contents, found = h.overlayFS[uriToOverlayPath(uri)]
	return
}

// AtomicFS wraps a ctxvfs.NameSpace but is safe for concurrent calls to Bind
// while doing FS operations. It is optimized for "ReadMostly" use-case. IE
// Bind is a relatively rare call compared to actual FS operations.
type AtomicFS struct {
	mu sync.Mutex   // serialize calls to Bind (ie only used by writers)
	v  atomic.Value // stores the ctxvfs.NameSpace
}

// NewAtomicFS returns an AtomicFS with an empty wrapped ctxvfs.NameSpace
func NewAtomicFS() *AtomicFS {
	fs := &AtomicFS{}
	fs.v.Store(make(ctxvfs.NameSpace))
	return fs
}

// Bind wraps ctxvfs.NameSpace.Bind
func (a *AtomicFS) Bind(old string, newfs ctxvfs.FileSystem, new string, mode ctxvfs.BindMode) {
	// We do copy-on-write
	a.mu.Lock()
	defer a.mu.Unlock()

	fs1 := a.v.Load().(ctxvfs.NameSpace)
	fs2 := make(ctxvfs.NameSpace)
	for k, v := range fs1 {
		fs2[k] = v
	}
	fs2.Bind(old, newfs, new, mode)
	a.v.Store(fs2)
}

func (*AtomicFS) String() string {
	return "atomicfs"
}

// Open wraps ctxvfs.NameSpace.Open
func (a *AtomicFS) Open(ctx context.Context, path string) (ctxvfs.ReadSeekCloser, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Open(ctx, path)
}

// Stat wraps ctxvfs.NameSpace.Stat
func (a *AtomicFS) Stat(ctx context.Context, path string) (os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Stat(ctx, path)
}

// Lstat wraps ctxvfs.NameSpace.Lstat
func (a *AtomicFS) Lstat(ctx context.Context, path string) (os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.Lstat(ctx, path)
}

// ReadDir wraps ctxvfs.NameSpace.ReadDir
func (a *AtomicFS) ReadDir(ctx context.Context, path string) ([]os.FileInfo, error) {
	fs := a.v.Load().(ctxvfs.NameSpace)
	return fs.ReadDir(ctx, path)
}
<|endoftext|>


<commit_before>package application

import (
	"flag"
	"github.com/photoshelf/photoshelf-storage/application/service"
	"github.com/photoshelf/photoshelf-storage/infrastructure/container"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/boltdb_storage"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/file_storage"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/leveldb_storage"
	"github.com/photoshelf/photoshelf-storage/presentation/controller"
	"github.com/stretchr/testify/assert"
	"os"
	"path"
	"testing"
)

func TestLoad(t *testing.T) {
	for _, testcase := range []struct {
		name     string
		function func(t *testing.T)
	}{
		{
			"with no args, can load default",
			func(t *testing.T) {
				configuration := load()
				assert.EqualValues(t, 1213, configuration.Server.Port)
				assert.EqualValues(t, "leveldb", configuration.Storage.Type)
				assert.EqualValues(t, "./photos", configuration.Storage.Path)
			},
		}, {
			"with specify c flag, can load from file",
			func(t *testing.T) {
				configurationPath := path.Join(os.Getenv("GOPATH"), "src/github.com/photoshelf/photoshelf-storage", "testdata", "test.yml")
				os.Args = append(os.Args, "-c", configurationPath)

				configuration := load()
				assert.EqualValues(t, configuration.Server.Port, 12345)
				assert.EqualValues(t, configuration.Storage.Type, "hoge")
				assert.EqualValues(t, configuration.Storage.Path, "fuga")
			},
		}, {
			"with flags, can parse from flags",
			func(t *testing.T) {
				os.Args = append(os.Args, "-p", "54321", "-t", "foo", "-s", "bar")

				configuration := load()
				assert.EqualValues(t, configuration.Server.Port, 54321)
				assert.EqualValues(t, configuration.Storage.Type, "foo")
				assert.EqualValues(t, configuration.Storage.Path, "bar")
			},
		},
	} {
		resetFlag()
		t.Run(testcase.name, testcase.function)
	}
}

func TestConfigure(t *testing.T) {
	for _, testcase := range []struct {
		name     string
		function func(t *testing.T)
	}{
		{
			"with leveldb type, returns instance specify",
			func(t *testing.T) {
				os.Args = append(os.Args, "-t", "leveldb")

				_, err := Configure()
				if assert.NoError(t, err) {
					assert.IsType(t, new(leveldb_storage.LeveldbStorage), actualRepository())
				}
			},
		}, {
			"when fail to load leveldb, returns error",
			func(t *testing.T) {
				file, err := os.Create(path.Join(os.TempDir(), "readonly"))
				assert.NoError(t, err)
				file.Close()

				os.Args = append(os.Args, "-t", "leveldb", "-s", path.Join(os.TempDir(), "readonly"))

				_, err = Configure()
				assert.Error(t, err)
			},
		}, {
			"with file type, returns instance specify",
			func(t *testing.T) {
				os.Args = append(os.Args, "-t", "file")

				_, err := Configure()
				if assert.NoError(t, err) {
					assert.IsType(t, new(file_storage.FileStorage), actualRepository())
				}
			},
		}, {
			"with boltdb type, returns instance specify",
			func(t *testing.T) {
				dbPath := path.Join(os.TempDir(), "boltdb")
				os.RemoveAll(dbPath)
				os.Args = append(os.Args, "-t", "boltdb", "-s", dbPath)

				_, err := Configure()
				if assert.NoError(t, err) {
					assert.IsType(t, new(boltdb_storage.BoltdbStorage), actualRepository())
				}
			},
		}, {
			"when fail to load boltdb, returns error",
			func(t *testing.T) {
				dbPath := path.Join(os.TempDir(), "err_boltdb")
				os.RemoveAll(dbPath)
				os.MkdirAll(dbPath, 0600)

				os.Args = append(os.Args, "-t", "boltdb", "-s", dbPath)

				_, err := Configure()
				assert.Error(t, err)
			},
		}, {
			"with unknown type, returns error",
			func(t *testing.T) {
				os.Args = append(os.Args, "-t", "unknown")

				_, err := Configure()
				assert.Error(t, err)
			},
		},
	} {
		resetFlag()
		t.Run(testcase.name, testcase.function)
	}
}

func resetFlag() {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	os.Args = []string{os.Args[0]}
}

func actualRepository() interface{} {
	var photoController controller.PhotoController
	container.Get(&photoController)

	return photoController.Service.(*service.PhotoServiceImpl).Repository
}
<commit_msg>Clean test code<commit_after>package application

import (
	"flag"
	"github.com/photoshelf/photoshelf-storage/application/service"
	"github.com/photoshelf/photoshelf-storage/infrastructure/container"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/boltdb_storage"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/file_storage"
	"github.com/photoshelf/photoshelf-storage/infrastructure/datastore/leveldb_storage"
	"github.com/photoshelf/photoshelf-storage/presentation/controller"
	"github.com/stretchr/testify/assert"
	"os"
	"path"
	"testing"
)

func TestLoad(t *testing.T) {
	t.Run("with no args, can load default", func(t *testing.T) {
		resetFlag()

		configuration := load()
		assert.EqualValues(t, 1213, configuration.Server.Port)
		assert.EqualValues(t, "leveldb", configuration.Storage.Type)
		assert.EqualValues(t, "./photos", configuration.Storage.Path)
	})

	t.Run("with specify c flag, can load from file", func(t *testing.T) {
		resetFlag()

		configurationPath := path.Join(os.Getenv("GOPATH"), "src/github.com/photoshelf/photoshelf-storage", "testdata", "test.yml")
		os.Args = append(os.Args, "-c", configurationPath)

		configuration := load()
		assert.EqualValues(t, configuration.Server.Port, 12345)
		assert.EqualValues(t, configuration.Storage.Type, "hoge")
		assert.EqualValues(t, configuration.Storage.Path, "fuga")
	})

	t.Run("with flags, can parse from flags", func(t *testing.T) {
		resetFlag()

		os.Args = append(os.Args, "-p", "54321", "-t", "foo", "-s", "bar")

		configuration := load()
		assert.EqualValues(t, configuration.Server.Port, 54321)
		assert.EqualValues(t, configuration.Storage.Type, "foo")
		assert.EqualValues(t, configuration.Storage.Path, "bar")
	})
}

func TestConfigure(t *testing.T) {
	t.Run("with leveldb type, returns instance specify", func(t *testing.T) {
		resetFlag()

		os.Args = append(os.Args, "-t", "leveldb")

		_, err := Configure()
		if assert.NoError(t, err) {
			assert.IsType(t, new(leveldb_storage.LeveldbStorage), actualRepository())
		}
	})

	t.Run("when fail to load leveldb, returns error", func(t *testing.T) {
		resetFlag()
		file, err := os.Create(path.Join(os.TempDir(), "readonly"))
		if err != nil {
			t.Fatal(err)
		}
		file.Close()

		os.Args = append(os.Args, "-t", "leveldb", "-s", path.Join(os.TempDir(), "readonly"))

		_, err = Configure()
		assert.Error(t, err)
	})

	t.Run("with file type, returns instance specify", func(t *testing.T) {
		resetFlag()

		os.Args = append(os.Args, "-t", "file")

		_, err := Configure()
		if assert.NoError(t, err) {
			assert.IsType(t, new(file_storage.FileStorage), actualRepository())
		}
	})

	t.Run("with boltdb type, returns instance specify", func(t *testing.T) {
		resetFlag()
		dbPath := path.Join(os.TempDir(), "boltdb")
		os.RemoveAll(dbPath)

		os.Args = append(os.Args, "-t", "boltdb", "-s", dbPath)

		_, err := Configure()
		if assert.NoError(t, err) {
			assert.IsType(t, new(boltdb_storage.BoltdbStorage), actualRepository())
		}
	})

	t.Run("when fail to load boltdb, returns error", func(t *testing.T) {
		resetFlag()
		dbPath := path.Join(os.TempDir(), "err_boltdb")
		os.RemoveAll(dbPath)
		os.MkdirAll(dbPath, 0600)

		os.Args = append(os.Args, "-t", "boltdb", "-s", dbPath)

		_, err := Configure()
		assert.Error(t, err)
	})

	t.Run("with unknown type, returns error", func(t *testing.T) {
		resetFlag()

		os.Args = append(os.Args, "-t", "unknown")

		_, err := Configure()
		assert.Error(t, err)
	})
}

func resetFlag() {
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	os.Args = []string{os.Args[0]}
}

func actualRepository() interface{} {
	var photoController controller.PhotoController
	container.Get(&photoController)

	return photoController.Service.(*service.PhotoServiceImpl).Repository
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	"image/color"
	"image/png"
	"math"
	"math/cmplx"
	"os"

	"github.com/krasoffski/gomill/htcmap"
)

func main() {
	const (
		xmin, ymin    = -2.2, -1.2
		xmax, ymax    = +1.2, +1.2
		width, height = 1536, 1024
	)

	img := image.NewRGBA(image.Rect(0, 0, width, height))
	for py := 0; py < height; py++ {
		y := float64(py)/height*(ymax-ymin) + ymin
		for px := 0; px < width; px++ {
			x := float64(px)/width*(xmax-xmin) + xmin
			z := complex(x, y)
			img.Set(px, py, mandelbrot(z))
		}
	}
	if err := png.Encode(os.Stdout, img); err != nil {
		fmt.Fprintf(os.Stderr, "error encoding png: %s", err)
	}
}

func mandelbrot(z complex128) color.Color {
	const iterations = 255
	const contrast = 15

	var v complex128
	for n := uint8(0); n < iterations; n++ {
		v = v*v + z
		vAbs := cmplx.Abs(v)
		if vAbs > 2 && n > 5 {
			smooth := float64(n) + 1 - math.Log(math.Log(vAbs))/math.Log(2)
			r, g, b := htcmap.AsUInt8(float64(smooth*contrast), 0, iterations)
			return color.RGBA{r, g, b, 255}
		}
	}
	return color.Black
}
<commit_msg>Playing with supersampling.<commit_after>package main

import (
	"fmt"
	"image"
	"image/color"
	"image/png"
	"math/cmplx"
	"os"

	"github.com/krasoffski/gomill/htcmap"
)

func main() {
	const (
		xmin, ymin    = -2.3, -1.2
		xmax, ymax    = +1.2, +1.2
		width, height = 4000, 3000
		// step          = 1
	)

	// stepx := step * (xmax - xmin) / width
	// stepy := step * (ymax - ymin) / height
	img := image.NewRGBA(image.Rect(0, 0, width, height))
	for py := 0; py < height; py++ {
		y0 := float64(py)/height*(ymax-ymin) + ymin
		for px := 0; px < width; px++ {
			x0 := float64(px)/width*(xmax-xmin) + xmin
			c := mandelbrot(complex(x0, y0))
			// r0, g0, b0, _ := mandelbrot(complex(x0, y0)).RGBA()
			// r1, g1, b1, _ := mandelbrot(complex(x0-stepx, y0-stepy)).RGBA()
			// r2, g2, b2, _ := mandelbrot(complex(x0+stepx, y0-stepy)).RGBA()
			// r3, g3, b3, _ := mandelbrot(complex(x0+stepx, y0+stepy)).RGBA()
			// r4, g4, b4, _ := mandelbrot(complex(x0-stepx, y0+stepy)).RGBA()
			// r := uint16(math.Sqrt(float64(r0*r0+r1*r1+r2*r2+r3*r3+r4*r4) / 5))
			// g := uint16(math.Sqrt(float64(g0*g0+g1*g1+g2*g2+g3*g3+g4*g4) / 5))
			// b := uint16(math.Sqrt(float64(b0*b0+b1*b1+b2*b2+b3*b3+b4*b4) / 5))
			// r := uint16((r0 + r1 + r2 + r3 + r4) / 5)
			// g := uint16((g0 + g1 + g2 + g3 + g4) / 5)
			// b := uint16((b0 + b1 + b2 + b3 + b4) / 5)
			img.Set(px, py, c)
		}
	}
	if err := png.Encode(os.Stdout, img); err != nil {
		fmt.Fprintf(os.Stderr, "error encoding png: %s", err)
	}
}

func mandelbrot(z complex128) color.Color {
	const iterations = 255
	const contrast = 15

	var v complex128
	for n := uint8(0); n < iterations; n++ {
		v = v*v + z
		vAbs := cmplx.Abs(v)
		if vAbs > 2 {
			// smooth := float64(n) + 1 - math.Log(math.Log(vAbs))/math.Log(2)
			r, g, b := htcmap.AsUInt8(float64(n*contrast), 0, iterations)
			return color.RGBA{r, g, b, 255}
		}
	}
	return color.Black
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/mholt/caddy/middleware"
)

// dirFunc is a type of parsing function which processes
// a particular directive and populates the config.
type dirFunc func(*parser) error

// validDirectives is a map of valid, built-in directive names
// to their parsing function. Built-in directives cannot be
// ordered, so they should only be used for internal server
// configuration; not directly handling requests.
var validDirectives map[string]dirFunc

func init() {
	// This has to be in the init function
	// to avoid an initialization loop error because
	// the 'import' directive (key) in this map
	// invokes a method that uses this map.
	validDirectives = map[string]dirFunc{
		"root": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}
			p.cfg.Root = p.tkn()

			// Ensure root folder exists
			_, err := os.Open(p.cfg.Root)
			if err != nil {
				if os.IsNotExist(err) {
					// Allow this, because the folder might appear later.
					// But make sure the user knows!
					log.Printf("Warning: Root path %s does not exist", p.cfg.Root)
				} else {
					return p.err("Path", fmt.Sprintf("Unable to access root path '%s': %s", p.cfg.Root, err.Error()))
				}
			}
			return nil
		},
		"import": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}

			filename := p.tkn()
			file, err := os.Open(filename)
			if err != nil {
				return p.err("Parse", err.Error())
			}
			defer file.Close()
			p2, err := newParser(file)
			if err != nil {
				return p.err("Parse", "Could not import "+filename+"; "+err.Error())
			}

			p2.cfg = p.cfg
			err = p2.directives()
			if err != nil {
				return err
			}
			p.cfg = p2.cfg

			return nil
		},
		"tls": func(p *parser) error {
			tls := TLSConfig{Enabled: true}

			if !p.nextArg() {
				return p.argErr()
			}
			tls.Certificate = p.tkn()

			if !p.nextArg() {
				return p.argErr()
			}
			tls.Key = p.tkn()

			p.cfg.TLS = tls
			return nil
		},
		"startup": func(p *parser) error {
			// TODO: This code is duplicated with the shutdown directive below

			if !p.nextArg() {
				return p.argErr()
			}

			command, args, err := middleware.SplitCommandAndArgs(p.tkn())
			if err != nil {
				return p.err("Parse", err.Error())
			}

			startupfn := func() error {
				cmd := exec.Command(command, args...)
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				err := cmd.Run()
				if err != nil {
					return err
				}
				return nil
			}

			p.cfg.Startup = append(p.cfg.Startup, startupfn)
			return nil
		},
		"shutdown": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}

			command, args, err := middleware.SplitCommandAndArgs(p.tkn())
			if err != nil {
				return p.err("Parse", err.Error())
			}

			shutdownfn := func() error {
				cmd := exec.Command(command, args...)
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				err := cmd.Run()
				if err != nil {
					return err
				}
				return nil
			}

			p.cfg.Shutdown = append(p.cfg.Shutdown, shutdownfn)
			return nil
		},
	}
}
<commit_msg>Replace Open with Stat<commit_after>package config

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/mholt/caddy/middleware"
)

// dirFunc is a type of parsing function which processes
// a particular directive and populates the config.
type dirFunc func(*parser) error

// validDirectives is a map of valid, built-in directive names
// to their parsing function. Built-in directives cannot be
// ordered, so they should only be used for internal server
// configuration; not directly handling requests.
var validDirectives map[string]dirFunc

func init() {
	// This has to be in the init function
	// to avoid an initialization loop error because
	// the 'import' directive (key) in this map
	// invokes a method that uses this map.
	validDirectives = map[string]dirFunc{
		"root": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}
			p.cfg.Root = p.tkn()

			// Ensure root folder exists
			_, err := os.Stat(p.cfg.Root)
			if err != nil {
				if os.IsNotExist(err) {
					// Allow this, because the folder might appear later.
					// But make sure the user knows!
					log.Printf("Warning: Root path %s does not exist", p.cfg.Root)
				} else {
					return p.err("Path", fmt.Sprintf("Unable to access root path '%s': %s", p.cfg.Root, err.Error()))
				}
			}
			return nil
		},
		"import": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}

			filename := p.tkn()
			file, err := os.Open(filename)
			if err != nil {
				return p.err("Parse", err.Error())
			}
			defer file.Close()
			p2, err := newParser(file)
			if err != nil {
				return p.err("Parse", "Could not import "+filename+"; "+err.Error())
			}

			p2.cfg = p.cfg
			err = p2.directives()
			if err != nil {
				return err
			}
			p.cfg = p2.cfg

			return nil
		},
		"tls": func(p *parser) error {
			tls := TLSConfig{Enabled: true}

			if !p.nextArg() {
				return p.argErr()
			}
			tls.Certificate = p.tkn()

			if !p.nextArg() {
				return p.argErr()
			}
			tls.Key = p.tkn()

			p.cfg.TLS = tls
			return nil
		},
		"startup": func(p *parser) error {
			// TODO: This code is duplicated with the shutdown directive below

			if !p.nextArg() {
				return p.argErr()
			}

			command, args, err := middleware.SplitCommandAndArgs(p.tkn())
			if err != nil {
				return p.err("Parse", err.Error())
			}

			startupfn := func() error {
				cmd := exec.Command(command, args...)
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				err := cmd.Run()
				if err != nil {
					return err
				}
				return nil
			}

			p.cfg.Startup = append(p.cfg.Startup, startupfn)
			return nil
		},
		"shutdown": func(p *parser) error {
			if !p.nextArg() {
				return p.argErr()
			}

			command, args, err := middleware.SplitCommandAndArgs(p.tkn())
			if err != nil {
				return p.err("Parse", err.Error())
			}

			shutdownfn := func() error {
				cmd := exec.Command(command, args...)
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				err := cmd.Run()
				if err != nil {
					return err
				}
				return nil
			}

			p.cfg.Shutdown = append(p.cfg.Shutdown, shutdownfn)
			return nil
		},
	}
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/cloudfoundry/bosh-bootloader/storage"
	flags "github.com/jessevdk/go-flags"
)

type GlobalFlags struct {
	Help                 bool   `short:"h" long:"help"`
	Debug                bool   `short:"d" long:"debug"         env:"BBL_DEBUG"`
	Version              bool   `short:"v" long:"version"`
	StateDir             string `short:"s" long:"state-dir"     env:"BBL_STATE_DIR"`
	IAAS                 string `long:"iaas"                    env:"BBL_IAAS"`
	AWSAccessKeyID       string `long:"aws-access-key-id"       env:"BBL_AWS_ACCESS_KEY_ID"`
	AWSSecretAccessKey   string `long:"aws-secret-access-key"   env:"BBL_AWS_SECRET_ACCESS_KEY"`
	AWSRegion            string `long:"aws-region"              env:"BBL_AWS_REGION"`
	GCPServiceAccountKey string `long:"gcp-service-account-key" env:"BBL_GCP_SERVICE_ACCOUNT_KEY"`
	GCPProjectID         string `long:"gcp-project-id"          env:"BBL_GCP_PROJECT_ID"`
	GCPZone              string `long:"gcp-zone"                env:"BBL_GCP_ZONE"`
	GCPRegion            string `long:"gcp-region"              env:"BBL_GCP_REGION"`
}

type ParsedFlags struct {
	State         storage.State
	RemainingArgs []string
	Help          bool
	Debug         bool
	Version       bool
	StateDir      string
}

func NewConfig(getState func(string) (storage.State, error)) Config {
	return Config{
		getState: getState,
	}
}

type Config struct {
	getState func(string) (storage.State, error)
}

func (c Config) Bootstrap(args []string) (ParsedFlags, error) {
	var globalFlags GlobalFlags

	parser := flags.NewParser(&globalFlags, flags.IgnoreUnknown)

	remainingArgs, err := parser.ParseArgs(args[1:])
	if err != nil {
		return ParsedFlags{}, err
	}

	stateDir := globalFlags.StateDir
	if stateDir == "" {
		stateDir, err = os.Getwd()
		if err != nil {
			// not tested
			return ParsedFlags{}, err
		}
	}

	state, err := c.getState(stateDir)
	if err != nil {
		return ParsedFlags{}, err
	}

	if globalFlags.IAAS != "" {
		if state.IAAS != "" && globalFlags.IAAS != state.IAAS {
			iaasMismatch := fmt.Sprintf("The iaas type cannot be changed for an existing environment. The current iaas type is %s.", state.IAAS)
			return ParsedFlags{}, errors.New(iaasMismatch)
		}
		state.IAAS = globalFlags.IAAS
	}

	if globalFlags.AWSAccessKeyID != "" {
		state.AWS.AccessKeyID = globalFlags.AWSAccessKeyID
	}
	if globalFlags.AWSSecretAccessKey != "" {
		state.AWS.SecretAccessKey = globalFlags.AWSSecretAccessKey
	}
	if globalFlags.AWSRegion != "" {
		if state.AWS.Region != "" && globalFlags.AWSRegion != state.AWS.Region {
			regionMismatch := fmt.Sprintf("The region cannot be changed for an existing environment. The current region is %s.", state.AWS.Region)
			return ParsedFlags{}, errors.New(regionMismatch)
		}
		state.AWS.Region = globalFlags.AWSRegion
	}

	if globalFlags.GCPServiceAccountKey != "" {
		serviceAccountKey, err := parseServiceAccountKey(globalFlags.GCPServiceAccountKey)
		if err != nil {
			return ParsedFlags{}, err
		}
		state.GCP.ServiceAccountKey = serviceAccountKey
	}
	if globalFlags.GCPProjectID != "" {
		state.GCP.ProjectID = globalFlags.GCPProjectID
	}
	if globalFlags.GCPZone != "" {
		state.GCP.Zone = globalFlags.GCPZone
	}
	if globalFlags.GCPRegion != "" {
		if state.GCP.Region != "" && globalFlags.GCPRegion != state.GCP.Region {
			regionMismatch := fmt.Sprintf("The region cannot be changed for an existing environment. The current region is %s.", state.GCP.Region)
			return ParsedFlags{}, errors.New(regionMismatch)
		}
		state.GCP.Region = globalFlags.GCPRegion
	}

	nonStatefulCommand := len(remainingArgs) == 0 || (remainingArgs[0] == "help" || remainingArgs[0] == "version" || remainingArgs[0] == "latest-error")
	ignoreMissingFlags := globalFlags.Help || globalFlags.Version || nonStatefulCommand

	if !ignoreMissingFlags {
		err := validate(state)
		if err != nil {
			return ParsedFlags{}, err
		}
	}

	return ParsedFlags{State: state, RemainingArgs: remainingArgs, Help: globalFlags.Help, Debug: globalFlags.Debug, Version: globalFlags.Version, StateDir: globalFlags.StateDir}, nil
}

func validate(state storage.State) error {
	if state.IAAS == "" || (state.IAAS != "gcp" && state.IAAS != "aws") {
		return errors.New("--iaas [gcp, aws] must be provided or BBL_IAAS must be set")
	}
	if state.IAAS == "aws" {
		err := validateAWSFlags(state.AWS)
		if err != nil {
			return err
		}
	}
	if state.IAAS == "gcp" {
		err := validateGCPFlags(state.GCP)
		if err != nil {
			return err
		}
	}
	return nil
}

func validateAWSFlags(awsFlags storage.AWS) error {
	if awsFlags.AccessKeyID == "" {
		return errors.New("AWS access key ID must be provided")
	}
	if awsFlags.SecretAccessKey == "" {
		return errors.New("AWS secret access key must be provided")
	}
	if awsFlags.Region == "" {
		return errors.New("AWS region must be provided")
	}
	return nil
}

func validateGCPFlags(gcpFlags storage.GCP) error {
	if gcpFlags.ServiceAccountKey == "" {
		return errors.New("GCP service account key must be provided")
	}
	if gcpFlags.ProjectID == "" {
		return errors.New("GCP project ID must be provided")
	}
	if gcpFlags.Zone == "" {
		return errors.New("GCP zone must be provided")
	}
	if gcpFlags.Region == "" {
		return errors.New("GCP region must be provided")
	}
	return nil
}

func parseServiceAccountKey(serviceAccountKey string) (string, error) {
	var key string

	if _, err := os.Stat(serviceAccountKey); err != nil {
		key = serviceAccountKey
	} else {
		rawServiceAccountKey, err := ioutil.ReadFile(serviceAccountKey)
		if err != nil {
			return "", fmt.Errorf("error reading service account key from file: %v", err)
		}

		key = string(rawServiceAccountKey)
	}

	var tmp interface{}
	err := json.Unmarshal([]byte(key), &tmp)
	if err != nil {
		return "", fmt.Errorf("error unmarshalling service account key (must be valid json): %v", err)
	}

	return key, err
}
<commit_msg>No longer exposing GlobalFlags struct<commit_after>package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/cloudfoundry/bosh-bootloader/storage"
	flags "github.com/jessevdk/go-flags"
)

type globalFlags struct {
	Help                 bool   `short:"h" long:"help"`
	Debug                bool   `short:"d" long:"debug"         env:"BBL_DEBUG"`
	Version              bool   `short:"v" long:"version"`
	StateDir             string `short:"s" long:"state-dir"     env:"BBL_STATE_DIR"`
	IAAS                 string `long:"iaas"                    env:"BBL_IAAS"`
	AWSAccessKeyID       string `long:"aws-access-key-id"       env:"BBL_AWS_ACCESS_KEY_ID"`
	AWSSecretAccessKey   string `long:"aws-secret-access-key"   env:"BBL_AWS_SECRET_ACCESS_KEY"`
	AWSRegion            string `long:"aws-region"              env:"BBL_AWS_REGION"`
	GCPServiceAccountKey string `long:"gcp-service-account-key" env:"BBL_GCP_SERVICE_ACCOUNT_KEY"`
	GCPProjectID         string `long:"gcp-project-id"          env:"BBL_GCP_PROJECT_ID"`
	GCPZone              string `long:"gcp-zone"                env:"BBL_GCP_ZONE"`
	GCPRegion            string `long:"gcp-region"              env:"BBL_GCP_REGION"`
}

type ParsedFlags struct {
	State         storage.State
	RemainingArgs []string
	Help          bool
	Debug         bool
	Version       bool
	StateDir      string
}

func NewConfig(getState func(string) (storage.State, error)) Config {
	return Config{
		getState: getState,
	}
}

type Config struct {
	getState func(string) (storage.State, error)
}

func (c Config) Bootstrap(args []string) (ParsedFlags, error) {
	var globalFlags globalFlags

	parser := flags.NewParser(&globalFlags, flags.IgnoreUnknown)

	remainingArgs, err := parser.ParseArgs(args[1:])
	if err != nil {
		return ParsedFlags{}, err
	}

	stateDir := globalFlags.StateDir
	if stateDir == "" {
		stateDir, err = os.Getwd()
		if err != nil {
			// not tested
			return ParsedFlags{}, err
		}
	}

	state, err := c.getState(stateDir)
	if err != nil {
		return ParsedFlags{}, err
	}

	if globalFlags.IAAS != "" {
		if state.IAAS != "" && globalFlags.IAAS != state.IAAS {
			iaasMismatch := fmt.Sprintf("The iaas type cannot be changed for an existing environment. The current iaas type is %s.", state.IAAS)
			return ParsedFlags{}, errors.New(iaasMismatch)
		}
		state.IAAS = globalFlags.IAAS
	}

	if globalFlags.AWSAccessKeyID != "" {
		state.AWS.AccessKeyID = globalFlags.AWSAccessKeyID
	}
	if globalFlags.AWSSecretAccessKey != "" {
		state.AWS.SecretAccessKey = globalFlags.AWSSecretAccessKey
	}
	if globalFlags.AWSRegion != "" {
		if state.AWS.Region != "" && globalFlags.AWSRegion != state.AWS.Region {
			regionMismatch := fmt.Sprintf("The region cannot be changed for an existing environment. The current region is %s.", state.AWS.Region)
			return ParsedFlags{}, errors.New(regionMismatch)
		}
		state.AWS.Region = globalFlags.AWSRegion
	}

	if globalFlags.GCPServiceAccountKey != "" {
		serviceAccountKey, err := parseServiceAccountKey(globalFlags.GCPServiceAccountKey)
		if err != nil {
			return ParsedFlags{}, err
		}
		state.GCP.ServiceAccountKey = serviceAccountKey
	}
	if globalFlags.GCPProjectID != "" {
		state.GCP.ProjectID = globalFlags.GCPProjectID
	}
	if globalFlags.GCPZone != "" {
		state.GCP.Zone = globalFlags.GCPZone
	}
	if globalFlags.GCPRegion != "" {
		if state.GCP.Region != "" && globalFlags.GCPRegion != state.GCP.Region {
			regionMismatch := fmt.Sprintf("The region cannot be changed for an existing environment. The current region is %s.", state.GCP.Region)
			return ParsedFlags{}, errors.New(regionMismatch)
		}
		state.GCP.Region = globalFlags.GCPRegion
	}

	nonStatefulCommand := len(remainingArgs) == 0 || (remainingArgs[0] == "help" || remainingArgs[0] == "version" || remainingArgs[0] == "latest-error")
	ignoreMissingFlags := globalFlags.Help || globalFlags.Version || nonStatefulCommand

	if !ignoreMissingFlags {
		err := validate(state)
		if err != nil {
			return ParsedFlags{}, err
		}
	}

	return ParsedFlags{State: state, RemainingArgs: remainingArgs, Help: globalFlags.Help, Debug: globalFlags.Debug, Version: globalFlags.Version, StateDir: globalFlags.StateDir}, nil
}

func validate(state storage.State) error {
	if state.IAAS == "" || (state.IAAS != "gcp" && state.IAAS != "aws") {
		return errors.New("--iaas [gcp, aws] must be provided or BBL_IAAS must be set")
	}
	if state.IAAS == "aws" {
		err := validateAWSFlags(state.AWS)
		if err != nil {
			return err
		}
	}
	if state.IAAS == "gcp" {
		err := validateGCPFlags(state.GCP)
		if err != nil {
			return err
		}
	}
	return nil
}

func validateAWSFlags(awsFlags storage.AWS) error {
	if awsFlags.AccessKeyID == "" {
		return errors.New("AWS access key ID must be provided")
	}
	if awsFlags.SecretAccessKey == "" {
		return errors.New("AWS secret access key must be provided")
	}
	if awsFlags.Region == "" {
		return errors.New("AWS region must be provided")
	}
	return nil
}

func validateGCPFlags(gcpFlags storage.GCP) error {
	if gcpFlags.ServiceAccountKey == "" {
		return errors.New("GCP service account key must be provided")
	}
	if gcpFlags.ProjectID == "" {
		return errors.New("GCP project ID must be provided")
	}
	if gcpFlags.Zone == "" {
		return errors.New("GCP zone must be provided")
	}
	if gcpFlags.Region == "" {
		return errors.New("GCP region must be provided")
	}
	return nil
}

func parseServiceAccountKey(serviceAccountKey string) (string, error) {
	var key string

	if _, err := os.Stat(serviceAccountKey); err != nil {
		key = serviceAccountKey
	} else {
		rawServiceAccountKey, err := ioutil.ReadFile(serviceAccountKey)
		if err != nil {
			return "", fmt.Errorf("error reading service account key from file: %v", err)
		}

		key = string(rawServiceAccountKey)
	}

	var tmp interface{}
	err := json.Unmarshal([]byte(key), &tmp)
	if err != nil {
		return "", fmt.Errorf("error unmarshalling service account key (must be valid json): %v", err)
	}

	return key, err
}
<|endoftext|>


<commit_before>package regsrc

import (
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/svchost"
)

var (
	// InvalidHostString is a placeholder returned when a raw host can't be
	// converted by IDNA spec. It will never be returned for any host for which
	// Valid() is true.
	InvalidHostString = "<invalid host>"

	// urlLabelEndSubRe is a sub-expression that matches any character that's
	// allowed at the start or end of a URL label according to RFC1123.
	urlLabelEndSubRe = "[0-9A-Za-z]"

	// urlLabelEndSubRe is a sub-expression that matches any character that's
	// allowed at in a non-start or end of a URL label according to RFC1123.
	urlLabelMidSubRe = "[0-9A-Za-z-]"

	// urlLabelUnicodeSubRe is a sub-expression that matches any non-ascii char
	// in an IDN (Unicode) display URL. It's not strict - there are only ~15k
	// valid Unicode points in IDN RFC (some with conditions). We are just going
	// with being liberal with matching and then erroring if we fail to convert
	// to punycode later (which validates chars fully). This at least ensures
	// ascii chars dissalowed by the RC1123 parts above don't become legal
	// again.
	urlLabelUnicodeSubRe = "[^[:ascii:]]"

	// hostLabelSubRe is the sub-expression that matches a valid hostname label.
	// It does not anchor the start or end so it can be composed into more
	// complex RegExps below. Note that for sanity we don't handle disallowing
	// raw punycode in this regexp (esp. since re2 doesn't support negative
	// lookbehind, but we can capture it's presence here to check later).
	hostLabelSubRe = "" +
		// Match valid initial char, or unicode char
		"(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" +
		// Optionally, match 0 to 61 valid URL or Unicode chars,
		// followed by one valid end char or unicode char
		"(?:" +
		"(?:" + urlLabelMidSubRe + "|" + urlLabelUnicodeSubRe + "){0,61}" +
		"(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" +
		")?"

	// hostSubRe is the sub-expression that matches a valid host prefix.
	// Allows custom port.
	hostSubRe = hostLabelSubRe + "(?:\\." + hostLabelSubRe + ")+(?::\\d+)?"

	// hostRe is a regexp that matches a valid host prefix. Additional
	// validation of unicode strings is needed for matches.
	hostRe = regexp.MustCompile("^" + hostSubRe + "$")
)

// FriendlyHost describes a registry instance identified in source strings by a
// simple bare hostname like registry.terraform.io.
type FriendlyHost struct {
	Raw string
}

func NewFriendlyHost(host string) *FriendlyHost {
	return &FriendlyHost{Raw: host}
}

// ParseFriendlyHost attempts to parse a valid "friendly host" prefix from the
// given string. If no valid prefix is found, host will be nil and rest will
// contain the full source string. The host prefix must terminate at the end of
// the input or at the first / character. If one or more characters exist after
// the first /, they will be returned as rest (without the / delimiter).
// Hostnames containing punycode WILL be parsed successfully since they may have
// come from an internal normalized source string, however should be considered
// invalid if the string came from a user directly. This must be checked
// explicitly for user-input strings by calling Valid() on the
// returned host.
func ParseFriendlyHost(source string) (host *FriendlyHost, rest string) {
	parts := strings.SplitN(source, "/", 2)

	if hostRe.MatchString(parts[0]) {
		host = &FriendlyHost{Raw: parts[0]}
		if len(parts) == 2 {
			rest = parts[1]
		}
		return
	}

	// No match, return whole string as rest along with nil host
	rest = source
	return
}

// Valid returns whether the host prefix is considered valid in any case.
// Example of invalid prefixes might include ones that don't conform to the host
// name specifications. Not that IDN prefixes containing punycode are not valid
// input which we expect to always be in user-input or normalised display form.
func (h *FriendlyHost) Valid() bool {
	if h.Display() == InvalidHostString {
		return false
	}
	if h.Normalized() == InvalidHostString {
		return false
	}
	if containsPuny(h.Raw) {
		return false
	}
	return true
}

// Display returns the host formatted for display to the user in CLI or web
// output.
func (h *FriendlyHost) Display() string {
	hostname, err := svchost.ForComparison(h.Raw)
	if err != nil {
		return InvalidHostString
	}
	return hostname.ForDisplay()
}

// Normalized returns the host formatted for internal reference or comparison.
func (h *FriendlyHost) Normalized() string {
	hostname, err := svchost.ForComparison(h.Raw)
	if err != nil {
		return InvalidHostString
	}
	return hostname.String()
}

// String returns the host formatted as the user originally typed it assuming it
// was parsed from user input.
func (h *FriendlyHost) String() string {
	return h.Raw
}

// Equal compares the FriendlyHost against another instance taking normalization
// into account.
func (h *FriendlyHost) Equal(other *FriendlyHost) bool {
	if other == nil {
		return false
	}
	return h.Normalized() == other.Normalized()
}

func containsPuny(host string) bool {
	for _, lbl := range strings.Split(host, ".") {
		if strings.HasPrefix(strings.ToLower(lbl), "xn--") {
			return true
		}
	}
	return false
}
<commit_msg>use svchost.IsValid for FriendlyHost.Valid<commit_after>package regsrc

import (
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/svchost"
)

var (
	// InvalidHostString is a placeholder returned when a raw host can't be
	// converted by IDNA spec. It will never be returned for any host for which
	// Valid() is true.
	InvalidHostString = "<invalid host>"

	// urlLabelEndSubRe is a sub-expression that matches any character that's
	// allowed at the start or end of a URL label according to RFC1123.
	urlLabelEndSubRe = "[0-9A-Za-z]"

	// urlLabelEndSubRe is a sub-expression that matches any character that's
	// allowed at in a non-start or end of a URL label according to RFC1123.
	urlLabelMidSubRe = "[0-9A-Za-z-]"

	// urlLabelUnicodeSubRe is a sub-expression that matches any non-ascii char
	// in an IDN (Unicode) display URL. It's not strict - there are only ~15k
	// valid Unicode points in IDN RFC (some with conditions). We are just going
	// with being liberal with matching and then erroring if we fail to convert
	// to punycode later (which validates chars fully). This at least ensures
	// ascii chars dissalowed by the RC1123 parts above don't become legal
	// again.
	urlLabelUnicodeSubRe = "[^[:ascii:]]"

	// hostLabelSubRe is the sub-expression that matches a valid hostname label.
	// It does not anchor the start or end so it can be composed into more
	// complex RegExps below. Note that for sanity we don't handle disallowing
	// raw punycode in this regexp (esp. since re2 doesn't support negative
	// lookbehind, but we can capture it's presence here to check later).
	hostLabelSubRe = "" +
		// Match valid initial char, or unicode char
		"(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" +
		// Optionally, match 0 to 61 valid URL or Unicode chars,
		// followed by one valid end char or unicode char
		"(?:" +
		"(?:" + urlLabelMidSubRe + "|" + urlLabelUnicodeSubRe + "){0,61}" +
		"(?:" + urlLabelEndSubRe + "|" + urlLabelUnicodeSubRe + ")" +
		")?"

	// hostSubRe is the sub-expression that matches a valid host prefix.
	// Allows custom port.
	hostSubRe = hostLabelSubRe + "(?:\\." + hostLabelSubRe + ")+(?::\\d+)?"

	// hostRe is a regexp that matches a valid host prefix. Additional
	// validation of unicode strings is needed for matches.
	hostRe = regexp.MustCompile("^" + hostSubRe + "$")
)

// FriendlyHost describes a registry instance identified in source strings by a
// simple bare hostname like registry.terraform.io.
type FriendlyHost struct {
	Raw string
}

func NewFriendlyHost(host string) *FriendlyHost {
	return &FriendlyHost{Raw: host}
}

// ParseFriendlyHost attempts to parse a valid "friendly host" prefix from the
// given string. If no valid prefix is found, host will be nil and rest will
// contain the full source string. The host prefix must terminate at the end of
// the input or at the first / character. If one or more characters exist after
// the first /, they will be returned as rest (without the / delimiter).
// Hostnames containing punycode WILL be parsed successfully since they may have
// come from an internal normalized source string, however should be considered
// invalid if the string came from a user directly. This must be checked
// explicitly for user-input strings by calling Valid() on the
// returned host.
func ParseFriendlyHost(source string) (host *FriendlyHost, rest string) {
	parts := strings.SplitN(source, "/", 2)

	if hostRe.MatchString(parts[0]) {
		host = &FriendlyHost{Raw: parts[0]}
		if len(parts) == 2 {
			rest = parts[1]
		}
		return
	}

	// No match, return whole string as rest along with nil host
	rest = source
	return
}

// Valid returns whether the host prefix is considered valid in any case.
// Example of invalid prefixes might include ones that don't conform to the host
// name specifications. Not that IDN prefixes containing punycode are not valid
// input which we expect to always be in user-input or normalised display form.
func (h *FriendlyHost) Valid() bool {
	return svchost.IsValid(h.Raw)
}

// Display returns the host formatted for display to the user in CLI or web
// output.
func (h *FriendlyHost) Display() string {
	hostname, err := svchost.ForComparison(h.Raw)
	if err != nil {
		return InvalidHostString
	}
	return hostname.ForDisplay()
}

// Normalized returns the host formatted for internal reference or comparison.
func (h *FriendlyHost) Normalized() string {
	hostname, err := svchost.ForComparison(h.Raw)
	if err != nil {
		return InvalidHostString
	}
	return hostname.String()
}

// String returns the host formatted as the user originally typed it assuming it
// was parsed from user input.
func (h *FriendlyHost) String() string {
	return h.Raw
}

// Equal compares the FriendlyHost against another instance taking normalization
// into account.
func (h *FriendlyHost) Equal(other *FriendlyHost) bool {
	if other == nil {
		return false
	}
	return h.Normalized() == other.Normalized()
}

func containsPuny(host string) bool {
	for _, lbl := range strings.Split(host, ".") {
		if strings.HasPrefix(strings.ToLower(lbl), "xn--") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"syscall"

	dockerclient "github.com/control-center/serviced/commons/docker"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
)

// TODO: what to do about logging?

// GetServiceInstances returns all instances running on a service
func (a *api) GetServiceInstances(serviceID string) ([]service.Instance, error) {
	client, err := a.connectMaster()
	if err != nil {
		return nil, err
	}
	return client.GetServiceInstances(serviceID)
}

// StopServiceInstance stops a running instance of a service.
func (a *api) StopServiceInstance(serviceID string, instanceID int) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.StopServiceInstance(serviceID, instanceID)
}

// AttachServiceInstance locates and attaches to a running instance of a service
func (a *api) AttachServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// attach to the container
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "attach", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command == "" {
			cmd = append(cmd, "/bin/bash")
		} else {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return utils.AttachAndExec(location.ContainerID, cmd)
	}
}

// LogsForServiceInstance returns the logs for the service instance
func (a *api) LogsForServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// report container logs
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "logs", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return dockerclient.Logs(location.ContainerID, cmd)
	}
}

// SendDockerAction submits an action to a running service instance
func (a *api) SendDockerAction(serviceID string, instanceID int, action string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	return client.SendDockerAction(serviceID, instanceID, action, args)
}
<commit_msg>fixed docker logging<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"syscall"

	dockerclient "github.com/control-center/serviced/commons/docker"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
)

// TODO: what to do about logging?

// GetServiceInstances returns all instances running on a service
func (a *api) GetServiceInstances(serviceID string) ([]service.Instance, error) {
	client, err := a.connectMaster()
	if err != nil {
		return nil, err
	}
	return client.GetServiceInstances(serviceID)
}

// StopServiceInstance stops a running instance of a service.
func (a *api) StopServiceInstance(serviceID string, instanceID int) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.StopServiceInstance(serviceID, instanceID)
}

// AttachServiceInstance locates and attaches to a running instance of a service
func (a *api) AttachServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// attach to the container
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "attach", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command == "" {
			cmd = append(cmd, "/bin/bash")
		} else {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return utils.AttachAndExec(location.ContainerID, cmd)
	}
}

// LogsForServiceInstance returns the logs for the service instance
func (a *api) LogsForServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// report container logs
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "logs", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return dockerclient.Logs(location.ContainerID, cmd)
	}
}

// SendDockerAction submits an action to a running service instance
func (a *api) SendDockerAction(serviceID string, instanceID int, action string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	return client.SendDockerAction(serviceID, instanceID, action, args)
}
<|endoftext|>


<commit_before>package keycloak

import (
	"testing"
)

func TestGetToken(t *testing.T) {
	err := GetToken()
	if err != nil {
		t.Errorf("Failed test because: %s", err.Error())
	}
}

func TestGetUser(t *testing.T) {
	users, err := GetUser("user1@havengrc.com")
	if err != nil {
		t.Errorf("Failed test because: %s and %s", err.Error(), users)
	}
	if len(users) == 0 {
		t.Errorf("Failed and found %s", users)
	}
}

func TestResetPassword(t *testing.T) {
	err := ResetPassword("user1@havengrc.com")
	if err != nil {
		t.Errorf("Failed test because: %s", err.Error())
	}
}
<commit_msg>Fix test<commit_after>package keycloak

import (
	"testing"
)

func TestGetToken(t *testing.T) {
	err := GetToken()
	if err != nil {
		t.Errorf("Failed test because: %s", err.Error())
	}
}

func TestGetUser(t *testing.T) {
	users, err := GetUser("user1@havengrc.com")
	if err != nil {
		t.Errorf("Failed test because: %s and %s", err.Error(), users)
	}
	if len(users) == 0 {
		t.Errorf("Failed and found %s", users)
	}
}

func TestVerifyEmail(t *testing.T) {
	err := SendVerificationEmail("user1@havengrc.com")
	if err != nil {
		t.Errorf("Failed test because: %s", err.Error())
	}
}
<|endoftext|>