<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kilt

import (
	"encoding/json"
	"errors"
	"io/ioutil"

	log "github.com/golang/glog"

	"github.com/google/kilt/pkg/dependency"
	"github.com/google/kilt/pkg/patchset"
	"github.com/google/kilt/pkg/repo"

	"github.com/spf13/cobra"
)

var addDepCmd = &cobra.Command{
	Use:   "add-dep <patchset> <p1> [p2...]",
	Short: "Add a dependency to a patchset",
	Long: `Add one or more dependencies to a patchset. Pass in multiple patchset names to
include multiple dependencies.`,
	Args: argsDep,
	Run:  runAdd,
}

var rmDepCmd = &cobra.Command{
	Use:   "rm-dep <patchset> <p1> [p2...]",
	Short: "Remove a dependency from a patchset",
	Long: `Remove one or more dependencies to a patchset. Pass in multiple patchset names to
include multiple dependencies.`,
	Args: argsDep,
	Run:  runRm,
}

var dependencyFile = "dependencies.json"

func init() {
	rootCmd.AddCommand(addDepCmd)
	rootCmd.AddCommand(rmDepCmd)
}

func argsDep(cmd *cobra.Command, args []string) error {
	if len(args) < 2 {
		return errors.New("Patchset name and at least one dependency required")
	}
	return nil
}

func runAdd(cmd *cobra.Command, args []string) {
	runDep(dependency.Graph.Add, cmd, args)
}

func runRm(cmd *cobra.Command, args []string) {
	runDep(dependency.Graph.Remove, cmd, args)
}

func runDep(op func(d dependency.Graph, ps, dep *patchset.Patchset) error, cmd *cobra.Command, args []string) {
	repo, err := repo.Open()
	if err != nil {
		log.Exitf("Init failed: %s", err)
	}
	patchsets, err := repo.Patchsets()
	if err != nil {
		log.Exitf("Error loading patchsets: %v", err)
	}
	deps := dependency.NewStruct(patchsets)
	b, err := ioutil.ReadFile(dependencyFile)
	if err != nil {
		log.Exitf("Failed to read %q: %v", dependencyFile, err)
	}
	err = json.Unmarshal(b, deps)
	if err != nil {
		log.Exitf("Failed to load %q: %v", dependencyFile, err)
	}
	ps, err := repo.FindPatchset(args[0])
	if err != nil {
		log.Exitf("Error finding patchset %q: %v", args[0], err)
	}
	if ps == nil {
		log.Exitf("Patchset %q not found", args[0])
	}
	for _, d := range args[1:] {
		dep, err := repo.FindPatchset(d)
		if err != nil {
			log.Exitf("Error finding dependency %q: %v", args[0], err)
		}
		if dep == nil {
			log.Exitf("Patchset %q not found", d)
		}
		if err = op(deps, ps, dep); err != nil {
			log.Exitf("Operation failed: %v", err)
		}
	}
	if err = deps.Validate(); err != nil {
		log.Exitf("Invalid graph: %v", err)
	}
	b, err = json.Marshal(deps)
	if err != nil {
		log.Exitf("Failed to marshal dependencies: %v", err)
	}
	err = ioutil.WriteFile(dependencyFile, b, 0666)
	if err != nil {
		log.Exitf("Failed to write file %q: %v", dependencyFile, err)
	}
}
<commit_msg>kilt: Fix up dependency file handling<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kilt

import (
	"encoding/json"
	"errors"
	"io/ioutil"

	log "github.com/golang/glog"

	"github.com/google/kilt/pkg/dependency"
	"github.com/google/kilt/pkg/patchset"
	"github.com/google/kilt/pkg/repo"

	"github.com/spf13/cobra"
)

var addDepCmd = &cobra.Command{
	Use:   "add-dep <patchset> <p1> [p2...]",
	Short: "Add a dependency to a patchset",
	Long: `Add one or more dependencies to a patchset. Pass in multiple patchset names to
include multiple dependencies.`,
	Args: argsDep,
	Run:  runAdd,
}

var rmDepCmd = &cobra.Command{
	Use:   "rm-dep <patchset> <p1> [p2...]",
	Short: "Remove a dependency from a patchset",
	Long: `Remove one or more dependencies to a patchset. Pass in multiple patchset names to
include multiple dependencies.`,
	Args: argsDep,
	Run:  runRm,
}

var dependencyFile = "dependencies.json"

func init() {
	rootCmd.AddCommand(addDepCmd)
	rootCmd.AddCommand(rmDepCmd)
}

func argsDep(cmd *cobra.Command, args []string) error {
	if len(args) < 2 {
		return errors.New("Patchset name and at least one dependency required")
	}
	return nil
}

func runAdd(cmd *cobra.Command, args []string) {
	runDep(dependency.Graph.Add, cmd, args)
}

func runRm(cmd *cobra.Command, args []string) {
	runDep(dependency.Graph.Remove, cmd, args)
}

func runDep(op func(d dependency.Graph, ps, dep *patchset.Patchset) error, cmd *cobra.Command, args []string) {
	repo, err := repo.Open()
	if err != nil {
		log.Exitf("Init failed: %s", err)
	}
	patchsets, err := repo.Patchsets()
	if err != nil {
		log.Exitf("Error loading patchsets: %v", err)
	}
	deps := dependency.NewStruct(patchsets)
	b, err := ioutil.ReadFile(dependencyFile)
	if err == nil {
		err = json.Unmarshal(b, deps)
		if err != nil {
			log.Exitf("Failed to load %q: %v", dependencyFile, err)
		}
	}
	ps, err := repo.FindPatchset(args[0])
	if err != nil {
		log.Exitf("Error finding patchset %q: %v", args[0], err)
	}
	if ps == nil {
		log.Exitf("Patchset %q not found", args[0])
	}
	for _, d := range args[1:] {
		dep, err := repo.FindPatchset(d)
		if err != nil {
			log.Exitf("Error finding dependency %q: %v", args[0], err)
		}
		if dep == nil {
			log.Exitf("Patchset %q not found", d)
		}
		if err = op(deps, ps, dep); err != nil {
			log.Exitf("Operation failed: %v", err)
		}
	}
	if err = deps.Validate(); err != nil {
		log.Exitf("Invalid graph: %v", err)
	}
	b, err = json.MarshalIndent(deps, "", "  ")
	if err != nil {
		log.Exitf("Failed to marshal dependencies: %v", err)
	}
	b = append(b, "\n"...)
	err = ioutil.WriteFile(dependencyFile, b, 0666)
	if err != nil {
		log.Exitf("Failed to write file %q: %v", dependencyFile, err)
	}
}
<|endoftext|>


<commit_before>package kubectl

import (
	"github.com/spf13/cobra"
	"k8s.io/kubernetes/pkg/kubectl/cmd"
)

// NewKubectlCmd creates a new embedded kubectl command.
func NewKubectlCmd() *cobra.Command {
	return cmd.NewDefaultKubectlCommand()
}
<commit_msg>chore: Add aliases for the alauda kubectl command<commit_after>package kubectl

import (
	"github.com/spf13/cobra"
	"k8s.io/kubernetes/pkg/kubectl/cmd"
)

// NewKubectlCmd creates a new embedded kubectl command.
func NewKubectlCmd() *cobra.Command {
	kubectlCmd := cmd.NewDefaultKubectlCommand()

	kubectlCmd.Aliases = []string{"k", "k8s"}

	return kubectlCmd
}
<|endoftext|>


<commit_before>package executor

import (
	. "github.com/smartystreets/goconvey/convey"
	"os/exec"
	"syscall"
	"testing"
)

const (
	fifoTestDirTemplate = "/tmp/swan_local_test.XXXXXXXXXXX"
	fifoTestName        = "swan_fifo"
)

// TestLocal
func TestLocal(t *testing.T) {
	// Prepare unique tmp directory for the following tests.
	cmd := exec.Command("mktemp", "-d", fifoTestDirTemplate)
	// Parse unique dir output.
	dirBytes, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}

	// Remove last element - it's newline.
	fifoDir := string(dirBytes[:len(dirBytes)-1])

	fifoPath := fifoDir + "/" + fifoTestName

	// Create fifo for the following tests. Making sure it has proper permissions.
	err = syscall.Mkfifo(fifoPath, syscall.S_IFIFO| 0666)
	if err != nil {
		t.Fatal(err)
	}

	Convey("Using Local Shell", t, func() {
		l := NewLocal()

		Convey("When command waiting for signal in fifo "+
			"is executed and we wait for it with timeout 1ms", func() {
			task, err := l.Execute("read -n 1 <" + fifoPath)

			if task != nil {
				taskNotTimeouted := task.Wait(1)

				taskState, _ := task.Status()

				Convey("The task should be still running", func() {
					So(taskState, ShouldEqual, RUNNING)
				})

				Convey("And the timeout should exceed", func() {
					So(taskNotTimeouted, ShouldBeFalse)
				})

			}

			Convey("Error is nil", func() {
				So(err, ShouldBeNil)
			})

			task.Stop()
		})

		Convey("When command waiting for signal in fifo "+
			"is executed and we stop it after start", func() {
			task, err := l.Execute("read -n 1 <" + fifoPath)

			if task != nil {
				task.Stop()

				taskState, taskStatus := task.Status()

				Convey("The task should be not running", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be -1", func() {
					So(taskStatus.ExitCode, ShouldEqual, -1)
				})
			}

			Convey("Error is nil", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("When command `echo output` is executed and we wait for it", func() {
			task, err := l.Execute("echo output")

			if task != nil {
				taskNotTimeouted := task.Wait(500)

				taskState, taskStatus := task.Status()

				Convey("The task should be not running", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 0", func() {
					So(taskStatus.ExitCode, ShouldEqual, 0)
				})

				Convey("And command stdout needs to match 'output", func() {
					So(taskStatus.Stdout, ShouldEqual, "output\n")
				})

				Convey("And the timeout should NOT exceed", func() {
					So(taskNotTimeouted, ShouldBeTrue)
				})
			}

			Convey("And error is nil", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("When command which does not exists is executed and we wait for it", func() {
			task, err := l.Execute("commandThatDoesNotExists")

			if task != nil {
				taskNotTimeouted := task.Wait(500)

				taskState, taskStatus := task.Status()

				Convey("The task should be not running", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 127", func() {
					So(taskStatus.ExitCode, ShouldEqual, 127)
				})

				Convey("And the timeout should NOT exceed", func() {
					So(taskNotTimeouted, ShouldBeTrue)
				})
			}

			Convey("Error is nil", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("When we execute two tasks in the same time", func() {
			task, err := l.Execute("echo output1")
			task2, err2 := l.Execute("echo output2")

			if task != nil && task2 != nil {
				task.Wait(0)
				task2.Wait(0)

				taskState1, taskStatus1 := task.Status()
				taskState2, taskStatus2 := task2.Status()

				Convey("The tasks should be not running", func() {
					So(taskState1, ShouldEqual, TERMINATED)
					So(taskState2, ShouldEqual, TERMINATED)
				})

				Convey("The commands stdouts needs to match 'output1' & 'output2'", func() {
					So(taskStatus1.Stdout, ShouldEqual, "output1\n")
					So(taskStatus2.Stdout, ShouldEqual, "output2\n")
				})

				Convey("Both exit statuses should be 0", func() {
					So(taskStatus1.ExitCode, ShouldEqual, 0)
					So(taskStatus2.ExitCode, ShouldEqual, 0)
				})
			}

			Convey("Errors are nil", func() {
				So(err, ShouldBeNil)
				So(err2, ShouldBeNil)
			})
		})
	})

	//Clean up
	cmd = exec.Command("rm", "-rf", fifoDir)
	err = cmd.Run()

	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Refactored local tests.<commit_after>package executor

import (
	. "github.com/smartystreets/goconvey/convey"
	"os/exec"
	"syscall"
	"testing"
	log "github.com/Sirupsen/logrus"
)

const (
	fifoTestDirTemplate = "/tmp/swan_local_test.XXXXXXXXXXX"
	fifoTestName        = "swan_fifo"
)

// TestLocal
func TestLocal(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	// Prepare unique tmp directory for the following tests.
	cmd := exec.Command("mktemp", "-d", fifoTestDirTemplate)
	// Parse unique dir output.
	dirBytes, err := cmd.Output()
	if err != nil {
		t.Fatal(err)
	}

	// Remove last element - it's newline.
	fifoDir := string(dirBytes[:len(dirBytes)-1])

	fifoPath := fifoDir + "/" + fifoTestName

	// Create fifo for the following tests. Making sure it has proper permissions.
	err = syscall.Mkfifo(fifoPath, syscall.S_IFIFO| 0666)
	if err != nil {
		t.Fatal(err)
	}

	Convey("While using Local Shell", t, func() {
		l := NewLocal()

		Convey("When command `waiting for signal in fifo` "+
			"is executed", func() {
			task, err := l.Execute("read -n 1 <" + fifoPath)

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("Task should be still running and status should be nil", func() {
				taskState, taskStatus := task.Status()
				So(taskState, ShouldEqual, RUNNING)
				So(taskStatus, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for task with the 1ms timeout", func() {
				taskNotTimeouted := task.Wait(1)

				Convey("The timeout should exceed ", func() {
					So(taskNotTimeouted, ShouldBeFalse)
				})

				Convey("The task should be still running and status should be nil", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, RUNNING)
					So(taskStatus, ShouldBeNil)
				})

				task.Stop()
			})

			Convey("When we stop the task", func() {
				err := task.Stop()

				Convey("There should be no error", func() {
					So(err, ShouldBeNil)
				})

				Convey("The task should be NOT running and the task status should be -1", func() {
					taskState, taskStatus := task.Status()
					So(taskState, ShouldEqual, TERMINATED)
					So(taskStatus.ExitCode, ShouldEqual, -1)
				})
			})
		})

		Convey("When command `echo output` is executed", func() {
			task, err := l.Execute("echo output")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task", func() {
				taskNotTimeouted := task.Wait(500)

				Convey("The timeout should NOT exceed", func() {
					So(taskNotTimeouted, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be NOT running", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 0", func() {
					So(taskStatus.ExitCode, ShouldEqual, 0)
				})

				Convey("And command stdout needs to match 'output", func() {
					So(taskStatus.Stdout, ShouldEqual, "output\n")
				})
			})
		})

		Convey("When command which does not exists is executed", func() {
			task, err := l.Execute("commandThatDoesNotExists")

			Convey("There should be no error", func() {
				So(err, ShouldBeNil)

				task.Stop()
			})

			Convey("When we wait for the task", func() {
				taskNotTimeouted := task.Wait(500)

				Convey("The timeout should NOT exceed", func() {
					So(taskNotTimeouted, ShouldBeTrue)
				})

				taskState, taskStatus := task.Status()

				Convey("The task should be NOT running", func() {
					So(taskState, ShouldEqual, TERMINATED)
				})

				Convey("And the exit status should be 127", func() {
					So(taskStatus.ExitCode, ShouldEqual, 127)
				})
			})
		})

		Convey("When we execute two tasks in the same time", func() {
			task, err := l.Execute("echo output1")
			task2, err2 := l.Execute("echo output2")

			Convey("There should be no errors", func() {
				So(err, ShouldBeNil)
				So(err2, ShouldBeNil)
			})

			Convey("When we wait for the tasks", func() {
				taskNotTimeouted := task.Wait(0)
				taskNotTimeouted2 := task2.Wait(0)

				Convey("The timeouts should NOT exceed", func() {
					So(taskNotTimeouted, ShouldBeTrue)
					So(taskNotTimeouted2, ShouldBeTrue)
				})

				taskState1, taskStatus1 := task.Status()
				taskState2, taskStatus2 := task2.Status()

				Convey("The tasks should be not running", func() {
					So(taskState1, ShouldEqual, TERMINATED)
					So(taskState2, ShouldEqual, TERMINATED)
				})

				Convey("The commands stdouts needs to match 'output1' & 'output2'", func() {
					So(taskStatus1.Stdout, ShouldEqual, "output1\n")
					So(taskStatus2.Stdout, ShouldEqual, "output2\n")
				})

				Convey("Both exit statuses should be 0", func() {
					So(taskStatus1.ExitCode, ShouldEqual, 0)
					So(taskStatus2.ExitCode, ShouldEqual, 0)
				})
			})
		})
	})

	//Clean up
	cmd = exec.Command("rm", "-rf", fifoDir)
	err = cmd.Run()

	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package modules

import (
	"regexp"
	"time"

	"github.com/pajbot/pajbot2/pkg"
	mbase "github.com/pajbot/pajbot2/pkg/modules/base"
	"github.com/pajbot/pajbot2/pkg/twitchactions"
	xurls "mvdan.cc/xurls/v2"
)

func init() {
	Register("link_filter", func() pkg.ModuleSpec {
		relaxedRegexp := xurls.Relaxed()
		strictRegexp := xurls.Strict()

		return &Spec{
			id:   "link_filter",
			name: "Link filter",
			maker: func(b mbase.Base) pkg.Module {
				return newLinkFilter(b, relaxedRegexp, strictRegexp)
			},
		}
	})
}

type LinkFilter struct {
	mbase.Base

	relaxedRegexp *regexp.Regexp
	strictRegexp  *regexp.Regexp
}

func newLinkFilter(b mbase.Base, relaxedRegexp, strictRegexp *regexp.Regexp) pkg.Module {
	return &LinkFilter{
		Base: b,

		relaxedRegexp: relaxedRegexp,
		strictRegexp:  strictRegexp,
	}
}

func (m LinkFilter) OnMessage(event pkg.MessageEvent) pkg.Actions {
	if event.User.IsModerator() {
		return nil
	}

	links := m.relaxedRegexp.FindAllString(event.Message.GetText(), -1)
	if len(links) > 0 {
		actions := &twitchactions.Actions{}
		actions.Timeout(event.User, 180*time.Second).SetReason("No links allowed")
		return actions
	}

	return nil
}
<commit_msg>link filter now allows vips to post links (no setting for now KKona )<commit_after>package modules

import (
	"regexp"
	"time"

	"github.com/pajbot/pajbot2/pkg"
	mbase "github.com/pajbot/pajbot2/pkg/modules/base"
	"github.com/pajbot/pajbot2/pkg/twitchactions"
	xurls "mvdan.cc/xurls/v2"
)

func init() {
	Register("link_filter", func() pkg.ModuleSpec {
		relaxedRegexp := xurls.Relaxed()
		strictRegexp := xurls.Strict()

		return &Spec{
			id:   "link_filter",
			name: "Link filter",
			maker: func(b mbase.Base) pkg.Module {
				return newLinkFilter(b, relaxedRegexp, strictRegexp)
			},
		}
	})
}

type LinkFilter struct {
	mbase.Base

	relaxedRegexp *regexp.Regexp
	strictRegexp  *regexp.Regexp
}

func newLinkFilter(b mbase.Base, relaxedRegexp, strictRegexp *regexp.Regexp) pkg.Module {
	return &LinkFilter{
		Base: b,

		relaxedRegexp: relaxedRegexp,
		strictRegexp:  strictRegexp,
	}
}

func (m LinkFilter) OnMessage(event pkg.MessageEvent) pkg.Actions {
	if event.User.IsModerator() {
		return nil
	}

	if event.User.IsVIP() {
		return nil
	}

	links := m.relaxedRegexp.FindAllString(event.Message.GetText(), -1)
	if len(links) > 0 {
		actions := &twitchactions.Actions{}
		actions.Timeout(event.User, 180*time.Second).SetReason("No links allowed")
		return actions
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mon for the Ceph monitors.
package mon

import (
	"fmt"
	"time"

	"github.com/rook/rook/pkg/ceph/client"
	"github.com/rook/rook/pkg/ceph/mon"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// HealthCheckInterval interval to check the mons to be in quorum
	HealthCheckInterval = 20 * time.Second
	// MonOutTimeout the duration to wait before removing/failover to a new mon pod
	MonOutTimeout = 300 * time.Second
)

// HealthChecker check health for the monitors
type HealthChecker struct {
	monCluster *Cluster
}

// NewHealthChecker creates a new HealthChecker object
func NewHealthChecker(monCluster *Cluster) *HealthChecker {
	return &HealthChecker{
		monCluster: monCluster,
	}
}

// Check periodically the health of the monitors
func (hc *HealthChecker) Check(stopCh chan struct{}) {
	for {
		select {
		case <-stopCh:
			logger.Infof("Stopping monitoring of cluster in namespace %s", hc.monCluster.Namespace)
			return

		case <-time.After(HealthCheckInterval):
			logger.Debugf("checking health of mons")
			err := hc.monCluster.checkHealth()
			if err != nil {
				logger.Infof("failed to check mon health. %+v", err)
			}
		}
	}
}

func (c *Cluster) checkHealth() error {
	logger.Debugf("Checking health for mons. %+v", c.clusterInfo)

	// connect to the mons
	// get the status and check for quorum
	status, err := client.GetMonStatus(c.context, c.clusterInfo.Name, true)
	if err != nil {
		return fmt.Errorf("failed to get mon status. %+v", err)
	}
	logger.Debugf("Mon status: %+v", status)

	// failover the unhealthy mons
	for _, mon := range status.MonMap.Mons {
		inQuorum := monInQuorum(mon, status.Quorum)
		if inQuorum {
			logger.Debugf("mon %s found in quorum", mon.Name)
			// delete the "timeout" for a mon if the pod is in quorum again
			if _, ok := c.monTimeoutList[mon.Name]; ok {
				delete(c.monTimeoutList, mon.Name)
			}
		} else {
			logger.Warningf("mon %s NOT found in quorum. %+v", mon.Name, status)

			// If not yet set, add the current time, for the timeout
			// calculation, to the list
			if _, ok := c.monTimeoutList[mon.Name]; !ok {
				c.monTimeoutList[mon.Name] = time.Now()
			}

			// when the timeout for the mon has been reached, continue to the
			// normal failover/delete mon pod part of the code
			if time.Since(c.monTimeoutList[mon.Name]) > MonOutTimeout {
				logger.Warningf("mon %s NOT found in quorum, STILL in mon out timeout", mon.Name)
				continue
			}

			if len(status.MonMap.Mons) > c.Size {
				// no need to create a new mon since we have an extra
				err = c.removeMon(mon.Name)
				if err != nil {
					logger.Errorf("failed to remove mon %s. %+v", mon.Name, err)
				}
			} else {
				// bring up a new mon to replace the unhealthy mon
				err = c.failoverMon(mon.Name)
				if err != nil {
					logger.Errorf("failed to failover mon %s. %+v", mon.Name, err)
				}
			}
			// only deal with one unhealthy mon per health check
			return nil
		}
	}

	return nil
}

func (c *Cluster) failoverMon(name string) error {
	logger.Infof("Failing over monitor %s", name)

	// Start a new monitor
	m := &monConfig{Name: fmt.Sprintf("%s%d", appName, c.maxMonID+1), Port: int32(mon.Port)}
	logger.Infof("starting new mon %s", m.Name)

	// Create the service endpoint
	serviceIP, err := c.createService(m)
	if err != nil {
		return fmt.Errorf("failed to create mon service. %+v", err)
	}
	m.PublicIP = serviceIP
	c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.PublicIP)

	// Save the mon config
	err = c.saveMonConfig()
	if err != nil {
		return fmt.Errorf("failed to save mons. %+v", err)
	}

	// Start the pod
	err = c.startPods([]*monConfig{m})
	if err != nil {
		return fmt.Errorf("failed to start new mon %s. %+v", m.Name, err)
	}

	// Only increment the max mon id if the new pod started successfully
	c.maxMonID++

	return c.removeMon(name)
}

func (c *Cluster) removeMon(name string) error {
	logger.Infof("ensuring removal of unhealthy monitor %s", name)

	// Remove the mon pod if it is still there
	var gracePeriod int64
	propagation := metav1.DeletePropagationForeground
	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
	err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Delete(name, options)
	if err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	// Remove the bad monitor from quorum
	err = mon.RemoveMonitorFromQuorum(c.context, c.clusterInfo.Name, name)
	if err != nil {
		return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
	}
	delete(c.clusterInfo.Monitors, name)

	// Remove the service endpoint
	err = c.context.Clientset.CoreV1().Services(c.Namespace).Delete(name, options)
	if err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon service %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	err = c.saveMonConfig()
	if err != nil {
		return fmt.Errorf("failed to save mon config after failing over mon %s. %+v", name, err)
	}

	return nil
}
<commit_msg>Fix the condition: Time since added to list <= monOutTimeout<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mon for the Ceph monitors.
package mon

import (
	"fmt"
	"time"

	"github.com/rook/rook/pkg/ceph/client"
	"github.com/rook/rook/pkg/ceph/mon"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// HealthCheckInterval interval to check the mons to be in quorum
	HealthCheckInterval = 20 * time.Second
	// MonOutTimeout the duration to wait before removing/failover to a new mon pod
	MonOutTimeout = 300 * time.Second
)

// HealthChecker check health for the monitors
type HealthChecker struct {
	monCluster *Cluster
}

// NewHealthChecker creates a new HealthChecker object
func NewHealthChecker(monCluster *Cluster) *HealthChecker {
	return &HealthChecker{
		monCluster: monCluster,
	}
}

// Check periodically the health of the monitors
func (hc *HealthChecker) Check(stopCh chan struct{}) {
	for {
		select {
		case <-stopCh:
			logger.Infof("Stopping monitoring of cluster in namespace %s", hc.monCluster.Namespace)
			return

		case <-time.After(HealthCheckInterval):
			logger.Debugf("checking health of mons")
			err := hc.monCluster.checkHealth()
			if err != nil {
				logger.Infof("failed to check mon health. %+v", err)
			}
		}
	}
}

func (c *Cluster) checkHealth() error {
	logger.Debugf("Checking health for mons. %+v", c.clusterInfo)

	// connect to the mons
	// get the status and check for quorum
	status, err := client.GetMonStatus(c.context, c.clusterInfo.Name, true)
	if err != nil {
		return fmt.Errorf("failed to get mon status. %+v", err)
	}
	logger.Debugf("Mon status: %+v", status)

	// failover the unhealthy mons
	for _, mon := range status.MonMap.Mons {
		inQuorum := monInQuorum(mon, status.Quorum)
		if inQuorum {
			logger.Debugf("mon %s found in quorum", mon.Name)
			// delete the "timeout" for a mon if the pod is in quorum again
			if _, ok := c.monTimeoutList[mon.Name]; ok {
				delete(c.monTimeoutList, mon.Name)
			}
		} else {
			logger.Warningf("mon %s NOT found in quorum. %+v", mon.Name, status)

			// If not yet set, add the current time, for the timeout
			// calculation, to the list
			if _, ok := c.monTimeoutList[mon.Name]; !ok {
				c.monTimeoutList[mon.Name] = time.Now()
			}

			// when the timeout for the mon has been reached, continue to the
			// normal failover/delete mon pod part of the code
			if time.Since(c.monTimeoutList[mon.Name]) <= MonOutTimeout {
				logger.Warningf("mon %s NOT found in quorum, STILL in mon out timeout", mon.Name)
				continue
			}

			if len(status.MonMap.Mons) > c.Size {
				// no need to create a new mon since we have an extra
				err = c.removeMon(mon.Name)
				if err != nil {
					logger.Errorf("failed to remove mon %s. %+v", mon.Name, err)
				}
			} else {
				// bring up a new mon to replace the unhealthy mon
				err = c.failoverMon(mon.Name)
				if err != nil {
					logger.Errorf("failed to failover mon %s. %+v", mon.Name, err)
				}
			}
			// only deal with one unhealthy mon per health check
			return nil
		}
	}

	return nil
}

func (c *Cluster) failoverMon(name string) error {
	logger.Infof("Failing over monitor %s", name)

	// Start a new monitor
	m := &monConfig{Name: fmt.Sprintf("%s%d", appName, c.maxMonID+1), Port: int32(mon.Port)}
	logger.Infof("starting new mon %s", m.Name)

	// Create the service endpoint
	serviceIP, err := c.createService(m)
	if err != nil {
		return fmt.Errorf("failed to create mon service. %+v", err)
	}
	m.PublicIP = serviceIP
	c.clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.PublicIP)

	// Save the mon config
	err = c.saveMonConfig()
	if err != nil {
		return fmt.Errorf("failed to save mons. %+v", err)
	}

	// Start the pod
	err = c.startPods([]*monConfig{m})
	if err != nil {
		return fmt.Errorf("failed to start new mon %s. %+v", m.Name, err)
	}

	// Only increment the max mon id if the new pod started successfully
	c.maxMonID++

	return c.removeMon(name)
}

func (c *Cluster) removeMon(name string) error {
	logger.Infof("ensuring removal of unhealthy monitor %s", name)

	// Remove the mon pod if it is still there
	var gracePeriod int64
	propagation := metav1.DeletePropagationForeground
	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
	err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Delete(name, options)
	if err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	// Remove the bad monitor from quorum
	err = mon.RemoveMonitorFromQuorum(c.context, c.clusterInfo.Name, name)
	if err != nil {
		return fmt.Errorf("failed to remove mon %s from quorum. %+v", name, err)
	}
	delete(c.clusterInfo.Monitors, name)

	// Remove the service endpoint
	err = c.context.Clientset.CoreV1().Services(c.Namespace).Delete(name, options)
	if err != nil {
		if errors.IsNotFound(err) {
			logger.Infof("dead mon service %s was already gone", name)
		} else {
			return fmt.Errorf("failed to remove dead mon pod %s. %+v", name, err)
		}
	}

	err = c.saveMonConfig()
	if err != nil {
		return fmt.Errorf("failed to save mon config after failing over mon %s. %+v", name, err)
	}

	return nil
}
<|endoftext|>


<commit_before>package paxos

import (
	"borg/assert"
	"testing"
	"fmt"
)

func TestAcceptsInvite(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:INVITE:1")
	close(ins)

	exp := msgs("2:1:RSVP:1:0:")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewInvitesAreStaleAndIgnored(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:INVITE:2")
	ins <- m("1:*:INVITE:1")
	close(ins)

	exp := msgs("2:1:RSVP:2:0:")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestIgnoresMalformedMessages(t *testing.T) {
	totest := msgs(
		//m("x"),            // too few separators
		//m("x:x"),          // too few separators
		//m("x:x:x"),        // too few separators
		//m("x:x:x:x:x"),    // too many separators
		//m("1:x:INVITE:1"), // invalid to address
		//m("X:*:INVITE:1"), // invalid from address

		"1:*:INVITE:x", // invalid round number
		"1:*:x:1",      // unknown command
		"1:7:INVITE:1", // valid but incorrect to address

		"1:*:NOMINATE:x",     // too few separators in nominate body
		"1:*:NOMINATE:x:foo", // invalid round number
	)

	for _, test := range totest {
		ins := make(chan msg)
		outs := make(chan msg)

		go acceptor(2, ins, outs)
		ins <- test
		close(ins)

		exp := []msg{}

		// outs was closed; therefore all messages have been processed
		assert.Equal(t, exp, gather(outs), fmt.Sprintf("#v", test))
	}
}

func TestItVotes(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "foo"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- m("1:*:NOMINATE:1:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:1:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherValue(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- m("1:*:NOMINATE:1:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:1:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherRound(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- m("1:*:NOMINATE:2:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:2:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherSelf(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(3, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- m("1:*:NOMINATE:2:"+val)
	close(ins)

	exp := msgs("3:*:VOTE:2:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItIgnoresOldNominations(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(3, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- m("1:*:INVITE:2")
	<-outs // throw away RSVP message
	ins <- m("1:*:NOMINATE:1:"+val)
	close(ins)

	exp := []msg{}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewNominationsAreStaleAndIgnored(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:NOMINATE:2:v")
	<-outs // throw away VOTE message
	ins <- m("1:*:INVITE:1")
	close(ins)

	exp := []msg{}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotedRoundsAndValuesAreTracked(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:NOMINATE:1:v")
	<-outs // throw away VOTE message
	ins <- m("1:*:INVITE:2")
	close(ins)

	exp := msgs("2:1:RSVP:2:1:v")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotesOnlyOncePerRound(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:NOMINATE:1:v")
	ins <- m("1:*:NOMINATE:1:v")
	close(ins)

	exp := msgs("2:*:VOTE:1:v")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}
<commit_msg>paxos: acceptor: Refactor and rename tests for clarity.<commit_after>package paxos

import (
	"borg/assert"
	"testing"
	"fmt"
)

func TestIgnoreOldMessages(t *testing.T) {
	tests := [][]msg{
		msgs("1:*:INVITE:11", "1:*:NOMINATE:1:v"),
		msgs("1:*:NOMINATE:11:v", "1:*:INVITE:1"),
		msgs("1:*:INVITE:11", "1:*:INVITE:1"),
		msgs("1:*:NOMINATE:11:v", "1:*:NOMINATE:1:v"),
	}

	for _, test := range tests {
		ins := make(chan msg)
		outs := make(chan msg)

		go acceptor(2, ins, outs)
		ins <- test[0]
		<-outs // throw away first reply
		ins <- test[1]
		close(ins)

		// outs was closed; therefore all messages have been processed
		assert.Equal(t, []msg{}, gather(outs), fmt.Sprintf("%v", test))
	}
}

func TestAcceptsInvite(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:INVITE:1")
	close(ins)

	exp := msgs("2:1:RSVP:1:0:")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestIgnoresMalformedMessages(t *testing.T) {
	totest := msgs(
		//m("x"),            // too few separators
		//m("x:x"),          // too few separators
		//m("x:x:x"),        // too few separators
		//m("x:x:x:x:x"),    // too many separators
		//m("1:x:INVITE:1"), // invalid to address
		//m("X:*:INVITE:1"), // invalid from address

		"1:*:INVITE:x", // invalid round number
		"1:*:x:1",      // unknown command
		"1:7:INVITE:1", // valid but incorrect to address

		"1:*:NOMINATE:x",     // too few separators in nominate body
		"1:*:NOMINATE:x:foo", // invalid round number
	)

	for _, test := range totest {
		ins := make(chan msg)
		outs := make(chan msg)

		go acceptor(2, ins, outs)
		ins <- test
		close(ins)

		exp := []msg{}

		// outs was closed; therefore all messages have been processed
		assert.Equal(t, exp, gather(outs), fmt.Sprintf("#v", test))
	}
}

func TestItVotes(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "foo"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- m("1:*:NOMINATE:1:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:1:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherValue(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- m("1:*:NOMINATE:1:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:1:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherRound(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(2, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- m("1:*:NOMINATE:2:"+val)
	close(ins)

	exp := msgs("2:*:VOTE:2:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherSelf(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	val := "bar"

	go acceptor(3, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- m("1:*:NOMINATE:2:"+val)
	close(ins)

	exp := msgs("3:*:VOTE:2:" + val)

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotedRoundsAndValuesAreTracked(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:NOMINATE:1:v")
	<-outs // throw away VOTE message
	ins <- m("1:*:INVITE:2")
	close(ins)

	exp := msgs("2:1:RSVP:2:1:v")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotesOnlyOncePerRound(t *testing.T) {
	ins := make(chan msg)
	outs := make(chan msg)

	go acceptor(2, ins, outs)
	ins <- m("1:*:NOMINATE:1:v")
	ins <- m("1:*:NOMINATE:1:v")
	close(ins)

	exp := msgs("2:*:VOTE:1:v")

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2017, UPMC Enterprises
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package processor

import (
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/upmc-enterprises/elasticsearch-operator/pkg/snapshot"
	myspec "github.com/upmc-enterprises/elasticsearch-operator/pkg/spec"
	"github.com/upmc-enterprises/elasticsearch-operator/util/k8sutil"
)

// processorLock ensures that reconciliation and event processing does
// not happen at the same time.
var (
	processorLock = &sync.Mutex{}
)

// Processor object
type Processor struct {
	k8sclient *k8sutil.K8sutil
	baseImage string
	clusters  map[string]*myspec.ElasticSearchCluster
}

// New creates new instance of Processor
func New(kclient *k8sutil.K8sutil, baseImage string) (*Processor, error) {
	p := &Processor{
		k8sclient: kclient,
		baseImage: baseImage,
		clusters:  make(map[string]*myspec.ElasticSearchCluster),
	}

	return p, nil
}

// Run starts the processor
func (p *Processor) Run() error {

	p.refreshClusters()
	logrus.Infof("Found %d existing clusters ", len(p.clusters))

	return nil
}

// WatchElasticSearchClusterEvents watches for changes to tpr elasticsearch events
func (p *Processor) WatchElasticSearchClusterEvents(done chan struct{}, wg *sync.WaitGroup) {
	events, watchErrs := p.k8sclient.MonitorElasticSearchEvents()
	go func() {
		for {
			select {
			case event := <-events:
				err := p.processElasticSearchClusterEvent(event)
				if err != nil {
					logrus.Errorln(err)
				}
			case err := <-watchErrs:
				logrus.Errorln(err)
			case <-done:
				wg.Done()
				logrus.Println("Stopped elasticsearch event watcher.")
				return
			}
		}
	}()
}

func (p *Processor) refreshClusters() error {

	//Reset
	p.clusters = make(map[string]*myspec.ElasticSearchCluster)

	// Get existing clusters
	currentClusters, err := p.k8sclient.GetElasticSearchClusters()

	if err != nil {
		logrus.Error("Could not get list of clusters: ", err)
		return err
	}

	for _, cluster := range currentClusters {
		logrus.Infof("Found cluster: %s", cluster.Metadata["name"])

		p.clusters[cluster.Metadata["name"]] = &myspec.ElasticSearchCluster{
			Spec: myspec.ClusterSpec{
				ClientNodeReplicas: cluster.Spec.ClientNodeReplicas,
				MasterNodeReplicas: cluster.Spec.MasterNodeReplicas,
				DataNodeReplicas:   cluster.Spec.DataNodeReplicas,
				Zones:              cluster.Spec.Zones,
				DataDiskSize:       cluster.Spec.DataDiskSize,
				ElasticSearchImage: cluster.Spec.ElasticSearchImage,
				JavaOptions:        cluster.Spec.JavaOptions,
				Snapshot: myspec.Snapshot{
					SchedulerEnabled: cluster.Spec.Snapshot.SchedulerEnabled,
					BucketName:       cluster.Spec.Snapshot.BucketName,
					CronSchedule:     cluster.Spec.Snapshot.CronSchedule,
				},
				Storage: myspec.Storage{
					StorageType:            cluster.Spec.Storage.StorageType,
					StorageClassProvisoner: cluster.Spec.Storage.StorageClassProvisoner,
				},
				Scheduler: snapshot.New(
					cluster.Spec.Snapshot.BucketName,
					cluster.Spec.Snapshot.CronSchedule,
					cluster.Spec.Snapshot.SchedulerEnabled),
			},
		}
	}

	return nil
}

func (p *Processor) processElasticSearchClusterEvent(c k8sutil.ElasticSearchEvent) error {
	processorLock.Lock()
	defer processorLock.Unlock()
	switch {
	case c.Type == "ADDED" || c.Type == "MODIFIED":
		return p.processElasticSearchCluster(c.Object)
	case c.Type == "DELETED":
		return p.deleteElasticSearchCluster(c.Object)
	}
	return nil
}

func (p *Processor) processElasticSearchCluster(c myspec.ElasticSearchCluster) error {
	logrus.Println("--------> ElasticSearch Event!")

	// Refresh
	p.refreshClusters()

	// Is a base image defined in the custom cluster?
	var baseImage = p.calcBaseImage(p.baseImage, c.Spec.ElasticSearchImage)

	logrus.Infof("Using [%s] as image for es cluster", baseImage)

	// Create Services
	p.k8sclient.CreateDiscoveryService()
	p.k8sclient.CreateDataService()
	p.k8sclient.CreateClientService()

	p.k8sclient.CreateClientMasterDeployment("client", baseImage, &c.Spec.ClientNodeReplicas, c.Spec.JavaOptions)
	p.k8sclient.CreateClientMasterDeployment("master", baseImage, &c.Spec.MasterNodeReplicas, c.Spec.JavaOptions)

	zoneCount := 0
	if len(c.Spec.Zones) != 0 {
		zoneCount = len(c.Spec.Zones)

		// Create Storage Classes
		for _, sc := range c.Spec.Zones {
			p.k8sclient.CreateStorageClass(sc, c.Spec.Storage.StorageClassProvisoner, c.Spec.Storage.StorageType)
		}

		zoneDistribution := p.calculateZoneDistribution(c.Spec.DataNodeReplicas, zoneCount)

		for index, count := range zoneDistribution {
			p.k8sclient.CreateDataNodeDeployment(&count, p.baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize)
		}
	} else {
		// No zones defined, rely on current provisioning logic which may break. Other strategy is to use emptyDir?
		// NOTE: Issue with dynamic PV provisioning (https://github.com/kubernetes/kubernetes/issues/34583)
		p.k8sclient.CreateStorageClass("standard", c.Spec.Storage.StorageClassProvisoner, c.Spec.Storage.StorageType)
		p.k8sclient.CreateDataNodeDeployment(func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), p.baseImage, "standard", c.Spec.DataDiskSize)
	}

	// Setup CronSchedule
	p.clusters[c.Metadata["name"]].Spec.Scheduler.Run()

	return nil
}

func (p *Processor) deleteElasticSearchCluster(c myspec.ElasticSearchCluster) error {
	logrus.Println("--------> ElasticSearch Cluster deleted...removing all components...")

	err := p.k8sclient.DeleteClientMasterDeployment("client")
	if err != nil {
		logrus.Error("Could not delete client deployment:", err)
	}

	err = p.k8sclient.DeleteClientMasterDeployment("master")
	if err != nil {
		logrus.Error("Could not delete master deployment:", err)
	}

	err = p.k8sclient.DeleteStatefulSet()
	if err != nil {
		logrus.Error("Could not delete stateful set:", err)
	}

	p.k8sclient.DeleteServices()
	p.k8sclient.DeleteStorageClasses()

	// Leave PV + PVC's for now?
	return nil
}

func (p *Processor) calculateZoneDistribution(dataReplicas, zonesCount int) []int32 {
	if zonesCount == 0 {
		zonesCount = 1
	}

	zoneDistribution := make([]int32, zonesCount)

	index := 0
	for i := 0; i < dataReplicas; i++ {
		if index >= zonesCount {
			index = 0
		}

		zoneDistribution[index]++
		index++
	}

	return zoneDistribution
}

func (p *Processor) calcBaseImage(baseImage, customImage string) string {
	if len(customImage) > 0 {
		baseImage = customImage
	}

	return baseImage
}
<commit_msg>Fix bug with base image (#20)<commit_after>/*
Copyright (c) 2017, UPMC Enterprises
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package processor

import (
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/upmc-enterprises/elasticsearch-operator/pkg/snapshot"
	myspec "github.com/upmc-enterprises/elasticsearch-operator/pkg/spec"
	"github.com/upmc-enterprises/elasticsearch-operator/util/k8sutil"
)

// processorLock ensures that reconciliation and event processing does
// not happen at the same time.
var (
	processorLock = &sync.Mutex{}
)

// Processor object
type Processor struct {
	k8sclient *k8sutil.K8sutil
	baseImage string
	clusters  map[string]*myspec.ElasticSearchCluster
}

// New creates new instance of Processor
func New(kclient *k8sutil.K8sutil, baseImage string) (*Processor, error) {
	p := &Processor{
		k8sclient: kclient,
		baseImage: baseImage,
		clusters:  make(map[string]*myspec.ElasticSearchCluster),
	}

	return p, nil
}

// Run starts the processor
func (p *Processor) Run() error {

	p.refreshClusters()
	logrus.Infof("Found %d existing clusters ", len(p.clusters))

	return nil
}

// WatchElasticSearchClusterEvents watches for changes to tpr elasticsearch events
func (p *Processor) WatchElasticSearchClusterEvents(done chan struct{}, wg *sync.WaitGroup) {
	events, watchErrs := p.k8sclient.MonitorElasticSearchEvents()
	go func() {
		for {
			select {
			case event := <-events:
				err := p.processElasticSearchClusterEvent(event)
				if err != nil {
					logrus.Errorln(err)
				}
			case err := <-watchErrs:
				logrus.Errorln(err)
			case <-done:
				wg.Done()
				logrus.Println("Stopped elasticsearch event watcher.")
				return
			}
		}
	}()
}

func (p *Processor) refreshClusters() error {

	//Reset
	p.clusters = make(map[string]*myspec.ElasticSearchCluster)

	// Get existing clusters
	currentClusters, err := p.k8sclient.GetElasticSearchClusters()

	if err != nil {
		logrus.Error("Could not get list of clusters: ", err)
		return err
	}

	for _, cluster := range currentClusters {
		logrus.Infof("Found cluster: %s", cluster.Metadata["name"])

		p.clusters[cluster.Metadata["name"]] = &myspec.ElasticSearchCluster{
			Spec: myspec.ClusterSpec{
				ClientNodeReplicas: cluster.Spec.ClientNodeReplicas,
				MasterNodeReplicas: cluster.Spec.MasterNodeReplicas,
				DataNodeReplicas:   cluster.Spec.DataNodeReplicas,
				Zones:              cluster.Spec.Zones,
				DataDiskSize:       cluster.Spec.DataDiskSize,
				ElasticSearchImage: cluster.Spec.ElasticSearchImage,
				JavaOptions:        cluster.Spec.JavaOptions,
				Snapshot: myspec.Snapshot{
					SchedulerEnabled: cluster.Spec.Snapshot.SchedulerEnabled,
					BucketName:       cluster.Spec.Snapshot.BucketName,
					CronSchedule:     cluster.Spec.Snapshot.CronSchedule,
				},
				Storage: myspec.Storage{
					StorageType:            cluster.Spec.Storage.StorageType,
					StorageClassProvisoner: cluster.Spec.Storage.StorageClassProvisoner,
				},
				Scheduler: snapshot.New(
					cluster.Spec.Snapshot.BucketName,
					cluster.Spec.Snapshot.CronSchedule,
					cluster.Spec.Snapshot.SchedulerEnabled),
			},
		}
	}

	return nil
}

func (p *Processor) processElasticSearchClusterEvent(c k8sutil.ElasticSearchEvent) error {
	processorLock.Lock()
	defer processorLock.Unlock()
	switch {
	case c.Type == "ADDED" || c.Type == "MODIFIED":
		return p.processElasticSearchCluster(c.Object)
	case c.Type == "DELETED":
		return p.deleteElasticSearchCluster(c.Object)
	}
	return nil
}

func (p *Processor) processElasticSearchCluster(c myspec.ElasticSearchCluster) error {
	logrus.Println("--------> ElasticSearch Event!")

	// Refresh
	p.refreshClusters()

	// Is a base image defined in the custom cluster?
	var baseImage = p.calcBaseImage(p.baseImage, c.Spec.ElasticSearchImage)

	logrus.Infof("Using [%s] as image for es cluster", baseImage)

	// Create Services
	p.k8sclient.CreateDiscoveryService()
	p.k8sclient.CreateDataService()
	p.k8sclient.CreateClientService()

	p.k8sclient.CreateClientMasterDeployment("client", baseImage, &c.Spec.ClientNodeReplicas, c.Spec.JavaOptions)
	p.k8sclient.CreateClientMasterDeployment("master", baseImage, &c.Spec.MasterNodeReplicas, c.Spec.JavaOptions)

	zoneCount := 0
	if len(c.Spec.Zones) != 0 {
		zoneCount = len(c.Spec.Zones)

		// Create Storage Classes
		for _, sc := range c.Spec.Zones {
			p.k8sclient.CreateStorageClass(sc, c.Spec.Storage.StorageClassProvisoner, c.Spec.Storage.StorageType)
		}

		zoneDistribution := p.calculateZoneDistribution(c.Spec.DataNodeReplicas, zoneCount)

		for index, count := range zoneDistribution {
			p.k8sclient.CreateDataNodeDeployment(&count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize)
		}
	} else {
		// No zones defined, rely on current provisioning logic which may break. Other strategy is to use emptyDir?
		// NOTE: Issue with dynamic PV provisioning (https://github.com/kubernetes/kubernetes/issues/34583)
		p.k8sclient.CreateStorageClass("standard", c.Spec.Storage.StorageClassProvisoner, c.Spec.Storage.StorageType)
		p.k8sclient.CreateDataNodeDeployment(func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, "standard", c.Spec.DataDiskSize)
	}

	// Setup CronSchedule
	p.clusters[c.Metadata["name"]].Spec.Scheduler.Run()

	return nil
}

func (p *Processor) deleteElasticSearchCluster(c myspec.ElasticSearchCluster) error {
	logrus.Println("--------> ElasticSearch Cluster deleted...removing all components...")

	err := p.k8sclient.DeleteClientMasterDeployment("client")
	if err != nil {
		logrus.Error("Could not delete client deployment:", err)
	}

	err = p.k8sclient.DeleteClientMasterDeployment("master")
	if err != nil {
		logrus.Error("Could not delete master deployment:", err)
	}

	err = p.k8sclient.DeleteStatefulSet()
	if err != nil {
		logrus.Error("Could not delete stateful set:", err)
	}

	p.k8sclient.DeleteServices()
	p.k8sclient.DeleteStorageClasses()

	// Leave PV + PVC's for now?
	return nil
}

func (p *Processor) calculateZoneDistribution(dataReplicas, zonesCount int) []int32 {
	if zonesCount == 0 {
		zonesCount = 1
	}

	zoneDistribution := make([]int32, zonesCount)

	index := 0
	for i := 0; i < dataReplicas; i++ {
		if index >= zonesCount {
			index = 0
		}

		zoneDistribution[index]++
		index++
	}

	return zoneDistribution
}

func (p *Processor) calcBaseImage(baseImage, customImage string) string {
	if len(customImage) > 0 {
		baseImage = customImage
	}

	return baseImage
}
<|endoftext|>


<commit_before>package proxy

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

var (
	hosts = []*Target{
		&Target{Target: "127.0.0.1", Weight: 5},
		&Target{Target: "http://test.com", Weight: 10},
		&Target{Target: "http://example.com", Weight: 8},
	}
)

func TestRoundRobinBalancer(t *testing.T) {
	balancer := NewRoundrobinBalancer()

	electedHost, err := balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[0], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[1], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[2], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[0], electedHost)
}

func TestRoundRobinBalancerEmptyList(t *testing.T) {
	balancer := NewRoundrobinBalancer()

	_, err := balancer.Elect([]*Target{})
	assert.Error(t, err)
}

func TestWeightBalancer(t *testing.T) {
	balancer := NewWeightBalancer()

	electedHost, err := balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[1], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[2], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[2], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[0], electedHost)

	electedHost, err = balancer.Elect(hosts)
	assert.NoError(t, err)
	assert.Equal(t, hosts[1], electedHost)
}

func TestWeightBalancerEmptyList(t *testing.T) {
	balancer := NewWeightBalancer()

	_, err := balancer.Elect([]*Target{})
	assert.Error(t, err)
}

func TestWeightBalancerZeroWeight(t *testing.T) {
	balancer := NewWeightBalancer()

	_, err := balancer.Elect([]*Target{&Target{Target: "", Weight: 0}})
	assert.Error(t, err)
}
<commit_msg>Improving the tests<commit_after>package proxy

import (
	"testing"

	"github.com/stretchr/testify/suite"
)

type BalancerTestSuite struct {
	suite.Suite
	hosts []*Target
}

func (suite *BalancerTestSuite) SetupTest() {
	suite.hosts = []*Target{
		&Target{Target: "127.0.0.1", Weight: 5},
		&Target{Target: "http://test.com", Weight: 10},
		&Target{Target: "http://example.com", Weight: 8},
	}
}

func (suite *BalancerTestSuite) TestRoundRobinBalancerSuccessfulBalance() {
	balancer := NewRoundrobinBalancer()

	electedHost, err := balancer.Elect(suite.hosts)
	suite.NoError(err)
	suite.Equal(suite.hosts[0], electedHost)

	electedHost, err = balancer.Elect(suite.hosts)
	suite.NoError(err)
	suite.Equal(suite.hosts[1], electedHost)

	electedHost, err = balancer.Elect(suite.hosts)
	suite.NoError(err)
	suite.Equal(suite.hosts[2], electedHost)

	electedHost, err = balancer.Elect(suite.hosts)
	suite.NoError(err)
	suite.Equal(suite.hosts[0], electedHost)
}

func (suite *BalancerTestSuite) TestRoundRobinBalancerEmptyList() {
	balancer := NewRoundrobinBalancer()

	_, err := balancer.Elect([]*Target{})
	suite.Error(err)
}

func (suite *BalancerTestSuite) TestWeightBalancer() {
	balancer := NewWeightBalancer()

	electedHost, err := balancer.Elect(suite.hosts)
	suite.NoError(err)
	suite.NotNil(electedHost)
}

func (suite *BalancerTestSuite) TestWeightBalancerEmptyList() {
	balancer := NewWeightBalancer()

	_, err := balancer.Elect([]*Target{})
	suite.Error(err)
}

func (suite *BalancerTestSuite) TestWeightBalancerZeroWeight() {
	balancer := NewWeightBalancer()

	_, err := balancer.Elect([]*Target{&Target{Target: "", Weight: 0}})
	suite.Error(err)
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestBalancerTestSuite(t *testing.T) {
	suite.Run(t, new(BalancerTestSuite))
}
<|endoftext|>


<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package router

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/reborndb/reborn/pkg/utils"

	"github.com/reborndb/reborn/pkg/models"
	"github.com/reborndb/reborn/pkg/proxy/group"
	"github.com/reborndb/reborn/pkg/proxy/parser"
	"github.com/reborndb/reborn/pkg/proxy/router/topology"

	log "github.com/ngaut/logging"

	"github.com/juju/errors"
	topo "github.com/ngaut/go-zookeeper/zk"
	stats "github.com/ngaut/gostats"

	respcoding "github.com/ngaut/resp"
)

var blackList = []string{
	"KEYS", "MOVE", "OBJECT", "RENAME", "RENAMENX", "SORT", "SCAN", "BITOP" /*"MGET",*/ /* "MSET",*/, "MSETNX", "SCAN",
	"BLPOP", "BRPOP", "BRPOPLPUSH", "PSUBSCRIBE，PUBLISH", "PUNSUBSCRIBE", "SUBSCRIBE", "RANDOMKEY",
	"UNSUBSCRIBE", "DISCARD", "EXEC", "MULTI", "UNWATCH", "WATCH", "SCRIPT EXISTS", "SCRIPT FLUSH", "SCRIPT KILL",
	"SCRIPT LOAD" /*, "AUTH" , "ECHO"*/ /*"QUIT",*/ /*"SELECT",*/, "BGREWRITEAOF", "BGSAVE", "CLIENT KILL", "CLIENT LIST",
	"CONFIG GET", "CONFIG SET", "CONFIG RESETSTAT", "DBSIZE", "DEBUG OBJECT", "DEBUG SEGFAULT", "FLUSHALL", "FLUSHDB",
	"LASTSAVE", "MONITOR", "SAVE", "SHUTDOWN", "SLAVEOF", "SLOWLOG", "SYNC", "TIME", "SLOTSMGRTONE", "SLOTSMGRT",
	"SLOTSDEL",
}

var (
	blackListCommand = make(map[string]struct{})
	OK_BYTES         = []byte("+OK\r\n")
)

func init() {
	for _, k := range blackList {
		blackListCommand[k] = struct{}{}
	}
}

func allowOp(op string) bool {
	_, black := blackListCommand[op]
	return !black
}

func isMulOp(op string) bool {
	if op == "MGET" || op == "DEL" || op == "MSET" {
		return true
	}

	return false
}

func validSlot(i int) bool {
	if i < 0 || i >= models.DEFAULT_SLOT_NUM {
		return false
	}

	return true
}

func WriteMigrateKeyCmd(w io.Writer, addr string, timeoutMs int, keys ...[]byte) error {
	hostPort := strings.Split(addr, ":")
	if len(hostPort) != 2 {
		return errors.Errorf("invalid address " + addr)
	}
	respW := respcoding.NewRESPWriter(w)
	args := [][]byte{[]byte("slotsmgrttagone"), []byte(hostPort[0]), []byte(hostPort[1]),
		[]byte(strconv.Itoa(int(timeoutMs)))}
	args = append(args, keys...)
	err := respW.WriteCommand(args...)
	return errors.Trace(err)
}

type DeadlineReadWriter interface {
	io.ReadWriter
	SetWriteDeadline(t time.Time) error
	SetReadDeadline(t time.Time) error
}

func handleSpecCommand(cmd string, keys [][]byte, timeout int) ([]byte, bool, bool, error) {
	var b []byte
	shouldClose := false
	switch cmd {
	case "PING":
		b = []byte("+PONG\r\n")
	case "QUIT":
		b = OK_BYTES
		shouldClose = true
	case "SELECT":
		b = OK_BYTES
	case "AUTH":
		b = OK_BYTES
	case "ECHO":
		if len(keys) > 0 {
			var err error
			b, err = respcoding.Marshal(string(keys[0]))
			if err != nil {
				return nil, true, false, errors.Trace(err)
			}
		} else {
			return nil, true, false, nil
		}
	}

	if len(b) > 0 {
		return b, shouldClose, true, nil
	}

	return b, shouldClose, false, nil
}

func write2Client(redisReader *bufio.Reader, clientWriter io.Writer) (redisErr error, clientErr error) {
	resp, err := parser.Parse(redisReader)
	if err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	err = resp.WriteTo(clientWriter)
	return nil, errors.Trace(err)
}

type BufioDeadlineReadWriter interface {
	DeadlineReadWriter
	BufioReader() *bufio.Reader
}

func forward(c DeadlineReadWriter, redisConn BufioDeadlineReadWriter, resp *parser.Resp, timeout int) (redisErr error, clientErr error) {
	redisReader := redisConn.BufioReader()
	if err := redisConn.SetWriteDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}
	if err := resp.WriteTo(redisConn); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	if err := redisConn.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	if err := c.SetWriteDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return nil, errors.Trace(err)
	}

	// read and parse redis response
	return write2Client(redisReader, c)
}

func StringsContain(s []string, key string) bool {
	for _, val := range s {
		if val == key { //need our resopnse
			return true
		}
	}

	return false
}

func getRespOpKeys(c *session) (*parser.Resp, []byte, [][]byte, error) {
	resp, err := parser.Parse(c.r) // read client request
	if err != nil {
		return nil, nil, nil, errors.Trace(err)
	}

	op, keys, err := resp.GetOpKeys()
	if err != nil {
		return nil, nil, nil, errors.Trace(err)
	}

	if len(keys) == 0 {
		keys = [][]byte{[]byte("fakeKey")}
	}

	return resp, op, keys, nil
}

func filter(opstr string, keys [][]byte, c *session, timeoutSec int) (rawresp []byte, next bool, err error) {
	if !allowOp(opstr) {
		errmsg, err := respcoding.Marshal(fmt.Errorf("%s not allowed", opstr))
		if err != nil {
			log.Fatal("should never happend", opstr)
		}
		return errmsg, false, errors.New(string(errmsg))
	}

	buf, shouldClose, handled, err := handleSpecCommand(opstr, keys, timeoutSec)
	if shouldClose { //quit command
		return buf, false, errors.Trace(io.EOF)
	}
	if err != nil {
		return nil, false, errors.Trace(err)
	}

	if handled {
		return buf, false, nil
	}

	return nil, true, nil
}

func GetEventPath(evt interface{}) string {
	return evt.(topo.Event).Path
}

func CheckUlimit(min int) {
	ulimitN, err := exec.Command("/bin/sh", "-c", "ulimit -n").Output()
	if err != nil {
		log.Warning("get ulimit failed", err)
	}

	n, err := strconv.Atoi(strings.TrimSpace(string(ulimitN)))
	if err != nil || n < min {
		log.Fatalf("ulimit too small: %d, should be at least %d", n, min)
	}
}

func GetOriginError(err *errors.Err) error {
	if err != nil {
		if err.Cause() == nil && err.Underlying() == nil {
			return err
		} else {
			return err.Underlying()
		}
	}

	return err
}

func recordResponseTime(c *stats.Counters, d time.Duration) {
	switch {
	case d < 5:
		c.Add("0-5ms", 1)
	case d >= 5 && d < 10:
		c.Add("5-10ms", 1)
	case d >= 10 && d < 50:
		c.Add("10-50ms", 1)
	case d >= 50 && d < 200:
		c.Add("50-200ms", 1)
	case d >= 200 && d < 1000:
		c.Add("200-1000ms", 1)
	case d >= 1000 && d < 5000:
		c.Add("1000-5000ms", 1)
	case d >= 5000 && d < 10000:
		c.Add("5000-10000ms", 1)
	default:
		c.Add("10000ms+", 1)
	}
}

type killEvent struct {
	done chan error
}

type Conf struct {
	proxyId     string
	productName string
	zkAddr      string
	f           topology.ZkFactory
	netTimeout  int    //seconds
	proto       string //tcp or tcp4
	provider    string
}

func LoadConf(configFile string) (*Conf, error) {
	srvConf := &Conf{}
	conf, err := utils.InitConfigFromFile(configFile)
	if err != nil {
		log.Fatal(err)
	}

	srvConf.productName, _ = conf.ReadString("product", "test")
	if len(srvConf.productName) == 0 {
		log.Fatalf("invalid config: product entry is missing in %s", configFile)
	}
	srvConf.zkAddr, _ = conf.ReadString("zk", "")
	if len(srvConf.zkAddr) == 0 {
		log.Fatalf("invalid config: need zk entry is missing in %s", configFile)
	}
	srvConf.zkAddr = strings.TrimSpace(srvConf.zkAddr)

	srvConf.proxyId, _ = conf.ReadString("proxy_id", "")
	if len(srvConf.proxyId) == 0 {
		log.Fatalf("invalid config: need proxy_id entry is missing in %s", configFile)
	}

	srvConf.netTimeout, _ = conf.ReadInt("net_timeout", 5)
	srvConf.proto, _ = conf.ReadString("proto", "tcp")
	srvConf.provider, _ = conf.ReadString("coordinator", "zookeeper")
	log.Infof("%+v", srvConf)

	return srvConf, nil
}

type Slot struct {
	slotInfo    *models.Slot
	groupInfo   *models.ServerGroup
	dst         *group.Group
	migrateFrom *group.Group
}

type OnSuicideFun func() error

func needResponse(receivers []string, self models.ProxyInfo) bool {
	var pi models.ProxyInfo
	for _, v := range receivers {
		err := json.Unmarshal([]byte(v), &pi)
		if err != nil {
			//is it old version of dashboard
			if v == self.Id {
				return true
			}
			return false
		}

		if pi.Id == self.Id && pi.Pid == self.Pid && pi.StartAt == self.StartAt {
			return true
		}
	}

	return false
}

func isTheSameSlot(keys [][]byte) bool {
	firstSlot := -1
	for _, k := range keys {
		if firstSlot == -1 {
			firstSlot = mapKey2Slot(k)
		} else {
			if firstSlot != mapKey2Slot(k) {
				return false
			}
		}
	}

	return true
}
<commit_msg>fix compile error<commit_after>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package router

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/reborndb/reborn/pkg/utils"

	"github.com/reborndb/reborn/pkg/models"
	"github.com/reborndb/reborn/pkg/proxy/group"
	"github.com/reborndb/reborn/pkg/proxy/parser"
	"github.com/reborndb/reborn/pkg/proxy/router/topology"

	log "github.com/ngaut/logging"

	"github.com/juju/errors"
	topo "github.com/ngaut/go-zookeeper/zk"
	stats "github.com/ngaut/gostats"

	respcoding "github.com/ngaut/resp"
)

var blackList = []string{
	"KEYS", "MOVE", "OBJECT", "RENAME", "RENAMENX", "SORT", "SCAN", "BITOP" /*"MGET",*/ /* "MSET",*/, "MSETNX", "SCAN",
	"BLPOP", "BRPOP", "BRPOPLPUSH", "PSUBSCRIBE，PUBLISH", "PUNSUBSCRIBE", "SUBSCRIBE", "RANDOMKEY",
	"UNSUBSCRIBE", "DISCARD", "EXEC", "MULTI", "UNWATCH", "WATCH", "SCRIPT EXISTS", "SCRIPT FLUSH", "SCRIPT KILL",
	"SCRIPT LOAD" /*, "AUTH" , "ECHO"*/ /*"QUIT",*/ /*"SELECT",*/, "BGREWRITEAOF", "BGSAVE", "CLIENT KILL", "CLIENT LIST",
	"CONFIG GET", "CONFIG SET", "CONFIG RESETSTAT", "DBSIZE", "DEBUG OBJECT", "DEBUG SEGFAULT", "FLUSHALL", "FLUSHDB",
	"LASTSAVE", "MONITOR", "SAVE", "SHUTDOWN", "SLAVEOF", "SLOWLOG", "SYNC", "TIME", "SLOTSMGRTONE", "SLOTSMGRT",
	"SLOTSDEL",
}

var (
	blackListCommand = make(map[string]struct{})
	OK_BYTES         = []byte("+OK\r\n")
)

func init() {
	for _, k := range blackList {
		blackListCommand[k] = struct{}{}
	}
}

func allowOp(op string) bool {
	_, black := blackListCommand[op]
	return !black
}

func isMulOp(op string) bool {
	if op == "MGET" || op == "DEL" || op == "MSET" {
		return true
	}

	return false
}

func validSlot(i int) bool {
	if i < 0 || i >= models.DEFAULT_SLOT_NUM {
		return false
	}

	return true
}

func WriteMigrateKeyCmd(w io.Writer, addr string, timeoutMs int, keys ...[]byte) error {
	hostPort := strings.Split(addr, ":")
	if len(hostPort) != 2 {
		return errors.Errorf("invalid address " + addr)
	}
	respW := respcoding.NewRESPWriter(w)
	args := make([]string, 0, 4+len(keys))
	args = append(args, "slotsmgrttagone", hostPort[0], hostPort[1], strconv.Itoa(int(timeoutMs)))
	for _, key := range keys {
		args = append(args, string(key))
	}
	err := respW.WriteCommand(args...)
	return errors.Trace(err)
}

type DeadlineReadWriter interface {
	io.ReadWriter
	SetWriteDeadline(t time.Time) error
	SetReadDeadline(t time.Time) error
}

func handleSpecCommand(cmd string, keys [][]byte, timeout int) ([]byte, bool, bool, error) {
	var b []byte
	shouldClose := false
	switch cmd {
	case "PING":
		b = []byte("+PONG\r\n")
	case "QUIT":
		b = OK_BYTES
		shouldClose = true
	case "SELECT":
		b = OK_BYTES
	case "AUTH":
		b = OK_BYTES
	case "ECHO":
		if len(keys) > 0 {
			var err error
			b, err = respcoding.Marshal(string(keys[0]))
			if err != nil {
				return nil, true, false, errors.Trace(err)
			}
		} else {
			return nil, true, false, nil
		}
	}

	if len(b) > 0 {
		return b, shouldClose, true, nil
	}

	return b, shouldClose, false, nil
}

func write2Client(redisReader *bufio.Reader, clientWriter io.Writer) (redisErr error, clientErr error) {
	resp, err := parser.Parse(redisReader)
	if err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	err = resp.WriteTo(clientWriter)
	return nil, errors.Trace(err)
}

type BufioDeadlineReadWriter interface {
	DeadlineReadWriter
	BufioReader() *bufio.Reader
}

func forward(c DeadlineReadWriter, redisConn BufioDeadlineReadWriter, resp *parser.Resp, timeout int) (redisErr error, clientErr error) {
	redisReader := redisConn.BufioReader()
	if err := redisConn.SetWriteDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}
	if err := resp.WriteTo(redisConn); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	if err := redisConn.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return errors.Trace(err), errors.Trace(err)
	}

	if err := c.SetWriteDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil {
		return nil, errors.Trace(err)
	}

	// read and parse redis response
	return write2Client(redisReader, c)
}

func StringsContain(s []string, key string) bool {
	for _, val := range s {
		if val == key { //need our resopnse
			return true
		}
	}

	return false
}

func getRespOpKeys(c *session) (*parser.Resp, []byte, [][]byte, error) {
	resp, err := parser.Parse(c.r) // read client request
	if err != nil {
		return nil, nil, nil, errors.Trace(err)
	}

	op, keys, err := resp.GetOpKeys()
	if err != nil {
		return nil, nil, nil, errors.Trace(err)
	}

	if len(keys) == 0 {
		keys = [][]byte{[]byte("fakeKey")}
	}

	return resp, op, keys, nil
}

func filter(opstr string, keys [][]byte, c *session, timeoutSec int) (rawresp []byte, next bool, err error) {
	if !allowOp(opstr) {
		errmsg, err := respcoding.Marshal(fmt.Errorf("%s not allowed", opstr))
		if err != nil {
			log.Fatal("should never happend", opstr)
		}
		return errmsg, false, errors.New(string(errmsg))
	}

	buf, shouldClose, handled, err := handleSpecCommand(opstr, keys, timeoutSec)
	if shouldClose { //quit command
		return buf, false, errors.Trace(io.EOF)
	}
	if err != nil {
		return nil, false, errors.Trace(err)
	}

	if handled {
		return buf, false, nil
	}

	return nil, true, nil
}

func GetEventPath(evt interface{}) string {
	return evt.(topo.Event).Path
}

func CheckUlimit(min int) {
	ulimitN, err := exec.Command("/bin/sh", "-c", "ulimit -n").Output()
	if err != nil {
		log.Warning("get ulimit failed", err)
	}

	n, err := strconv.Atoi(strings.TrimSpace(string(ulimitN)))
	if err != nil || n < min {
		log.Fatalf("ulimit too small: %d, should be at least %d", n, min)
	}
}

func GetOriginError(err *errors.Err) error {
	if err != nil {
		if err.Cause() == nil && err.Underlying() == nil {
			return err
		} else {
			return err.Underlying()
		}
	}

	return err
}

func recordResponseTime(c *stats.Counters, d time.Duration) {
	switch {
	case d < 5:
		c.Add("0-5ms", 1)
	case d >= 5 && d < 10:
		c.Add("5-10ms", 1)
	case d >= 10 && d < 50:
		c.Add("10-50ms", 1)
	case d >= 50 && d < 200:
		c.Add("50-200ms", 1)
	case d >= 200 && d < 1000:
		c.Add("200-1000ms", 1)
	case d >= 1000 && d < 5000:
		c.Add("1000-5000ms", 1)
	case d >= 5000 && d < 10000:
		c.Add("5000-10000ms", 1)
	default:
		c.Add("10000ms+", 1)
	}
}

type killEvent struct {
	done chan error
}

type Conf struct {
	proxyId     string
	productName string
	zkAddr      string
	f           topology.ZkFactory
	netTimeout  int    //seconds
	proto       string //tcp or tcp4
	provider    string
}

func LoadConf(configFile string) (*Conf, error) {
	srvConf := &Conf{}
	conf, err := utils.InitConfigFromFile(configFile)
	if err != nil {
		log.Fatal(err)
	}

	srvConf.productName, _ = conf.ReadString("product", "test")
	if len(srvConf.productName) == 0 {
		log.Fatalf("invalid config: product entry is missing in %s", configFile)
	}
	srvConf.zkAddr, _ = conf.ReadString("zk", "")
	if len(srvConf.zkAddr) == 0 {
		log.Fatalf("invalid config: need zk entry is missing in %s", configFile)
	}
	srvConf.zkAddr = strings.TrimSpace(srvConf.zkAddr)

	srvConf.proxyId, _ = conf.ReadString("proxy_id", "")
	if len(srvConf.proxyId) == 0 {
		log.Fatalf("invalid config: need proxy_id entry is missing in %s", configFile)
	}

	srvConf.netTimeout, _ = conf.ReadInt("net_timeout", 5)
	srvConf.proto, _ = conf.ReadString("proto", "tcp")
	srvConf.provider, _ = conf.ReadString("coordinator", "zookeeper")
	log.Infof("%+v", srvConf)

	return srvConf, nil
}

type Slot struct {
	slotInfo    *models.Slot
	groupInfo   *models.ServerGroup
	dst         *group.Group
	migrateFrom *group.Group
}

type OnSuicideFun func() error

func needResponse(receivers []string, self models.ProxyInfo) bool {
	var pi models.ProxyInfo
	for _, v := range receivers {
		err := json.Unmarshal([]byte(v), &pi)
		if err != nil {
			//is it old version of dashboard
			if v == self.Id {
				return true
			}
			return false
		}

		if pi.Id == self.Id && pi.Pid == self.Pid && pi.StartAt == self.StartAt {
			return true
		}
	}

	return false
}

func isTheSameSlot(keys [][]byte) bool {
	firstSlot := -1
	for _, k := range keys {
		if firstSlot == -1 {
			firstSlot = mapKey2Slot(k)
		} else {
			if firstSlot != mapKey2Slot(k) {
				return false
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import syscall "syscall"

// Errors are singleton structures. Use the String() method to get their contents --
// it handles the nil (no error) case.
export type Error struct {
	s string
}

// Indexed by errno.
// If we worry about syscall speed (only relevant on failure), we could
// make it an array, but it's probably not important.
var ErrorTab = make(map[int64] *Error);

// Table of all known errors in system.  Use the same error string twice,
// get the same *os.Error.
var ErrorStringTab = make(map[string] *Error);

// These functions contain a race if two goroutines add identical
// errors simultaneously but the consequences are unimportant.

// Allocate an Error objecct, but if it's been seen before, share that one.
export func NewError(s string) *Error {
	if s == "" {
		return nil
	}
	err, ok := ErrorStringTab[s];
	if ok {
		return err
	}
	err = &Error{s};
	ErrorStringTab[s] = err;
	return err;
}

// Allocate an Error objecct, but if it's been seen before, share that one.
export func ErrnoToError(errno int64) *Error {
	if errno == 0 {
		return nil
	}
	// Quick lookup by errno.
	err, ok := ErrorTab[errno];
	if ok {
		return err
	}
	err = NewError(syscall.errstr(errno));
	ErrorTab[errno] = err;
	return err;
}

export var (
	ENONE = ErrnoToError(syscall.ENONE);
	EPERM = ErrnoToError(syscall.EPERM);
	ENOENT = ErrnoToError(syscall.ENOENT);
	ESRCH = ErrnoToError(syscall.ESRCH);
	EINTR = ErrnoToError(syscall.EINTR);
	EIO = ErrnoToError(syscall.EIO);
	ENXIO = ErrnoToError(syscall.ENXIO);
	E2BIG = ErrnoToError(syscall.E2BIG);
	ENOEXEC = ErrnoToError(syscall.ENOEXEC);
	EBADF = ErrnoToError(syscall.EBADF);
	ECHILD = ErrnoToError(syscall.ECHILD);
	EDEADLK = ErrnoToError(syscall.EDEADLK);
	ENOMEM = ErrnoToError(syscall.ENOMEM);
	EACCES = ErrnoToError(syscall.EACCES);
	EFAULT = ErrnoToError(syscall.EFAULT);
	ENOTBLK = ErrnoToError(syscall.ENOTBLK);
	EBUSY = ErrnoToError(syscall.EBUSY);
	EEXIST = ErrnoToError(syscall.EEXIST);
	EXDEV = ErrnoToError(syscall.EXDEV);
	ENODEV = ErrnoToError(syscall.ENODEV);
	ENOTDIR = ErrnoToError(syscall.ENOTDIR);
	EISDIR = ErrnoToError(syscall.EISDIR);
	EINVAL = ErrnoToError(syscall.EINVAL);
	ENFILE = ErrnoToError(syscall.ENFILE);
	EMFILE = ErrnoToError(syscall.EMFILE);
	ENOTTY = ErrnoToError(syscall.ENOTTY);
	ETXTBSY = ErrnoToError(syscall.ETXTBSY);
	EFBIG = ErrnoToError(syscall.EFBIG);
	ENOSPC = ErrnoToError(syscall.ENOSPC);
	ESPIPE = ErrnoToError(syscall.ESPIPE);
	EROFS = ErrnoToError(syscall.EROFS);
	EMLINK = ErrnoToError(syscall.EMLINK);
	EPIPE = ErrnoToError(syscall.EPIPE);
	EDOM = ErrnoToError(syscall.EDOM);
	ERANGE = ErrnoToError(syscall.ERANGE);
	EAGAIN = ErrnoToError(syscall.EAGAIN);
)
const NoError = "No Error"

func (e *Error) String() string {
	if e == nil {
		return NoError
	}
	return e.s
}
<commit_msg>typo<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import syscall "syscall"

// Errors are singleton structures. Use the String() method to get their contents --
// it handles the nil (no error) case.
export type Error struct {
	s string
}

// Indexed by errno.
// If we worry about syscall speed (only relevant on failure), we could
// make it an array, but it's probably not important.
var ErrorTab = make(map[int64] *Error);

// Table of all known errors in system.  Use the same error string twice,
// get the same *os.Error.
var ErrorStringTab = make(map[string] *Error);

// These functions contain a race if two goroutines add identical
// errors simultaneously but the consequences are unimportant.

// Allocate an Error object, but if it's been seen before, share that one.
export func NewError(s string) *Error {
	if s == "" {
		return nil
	}
	err, ok := ErrorStringTab[s];
	if ok {
		return err
	}
	err = &Error{s};
	ErrorStringTab[s] = err;
	return err;
}

// Allocate an Error objecct, but if it's been seen before, share that one.
export func ErrnoToError(errno int64) *Error {
	if errno == 0 {
		return nil
	}
	// Quick lookup by errno.
	err, ok := ErrorTab[errno];
	if ok {
		return err
	}
	err = NewError(syscall.errstr(errno));
	ErrorTab[errno] = err;
	return err;
}

export var (
	ENONE = ErrnoToError(syscall.ENONE);
	EPERM = ErrnoToError(syscall.EPERM);
	ENOENT = ErrnoToError(syscall.ENOENT);
	ESRCH = ErrnoToError(syscall.ESRCH);
	EINTR = ErrnoToError(syscall.EINTR);
	EIO = ErrnoToError(syscall.EIO);
	ENXIO = ErrnoToError(syscall.ENXIO);
	E2BIG = ErrnoToError(syscall.E2BIG);
	ENOEXEC = ErrnoToError(syscall.ENOEXEC);
	EBADF = ErrnoToError(syscall.EBADF);
	ECHILD = ErrnoToError(syscall.ECHILD);
	EDEADLK = ErrnoToError(syscall.EDEADLK);
	ENOMEM = ErrnoToError(syscall.ENOMEM);
	EACCES = ErrnoToError(syscall.EACCES);
	EFAULT = ErrnoToError(syscall.EFAULT);
	ENOTBLK = ErrnoToError(syscall.ENOTBLK);
	EBUSY = ErrnoToError(syscall.EBUSY);
	EEXIST = ErrnoToError(syscall.EEXIST);
	EXDEV = ErrnoToError(syscall.EXDEV);
	ENODEV = ErrnoToError(syscall.ENODEV);
	ENOTDIR = ErrnoToError(syscall.ENOTDIR);
	EISDIR = ErrnoToError(syscall.EISDIR);
	EINVAL = ErrnoToError(syscall.EINVAL);
	ENFILE = ErrnoToError(syscall.ENFILE);
	EMFILE = ErrnoToError(syscall.EMFILE);
	ENOTTY = ErrnoToError(syscall.ENOTTY);
	ETXTBSY = ErrnoToError(syscall.ETXTBSY);
	EFBIG = ErrnoToError(syscall.EFBIG);
	ENOSPC = ErrnoToError(syscall.ENOSPC);
	ESPIPE = ErrnoToError(syscall.ESPIPE);
	EROFS = ErrnoToError(syscall.EROFS);
	EMLINK = ErrnoToError(syscall.EMLINK);
	EPIPE = ErrnoToError(syscall.EPIPE);
	EDOM = ErrnoToError(syscall.EDOM);
	ERANGE = ErrnoToError(syscall.ERANGE);
	EAGAIN = ErrnoToError(syscall.EAGAIN);
)
const NoError = "No Error"

func (e *Error) String() string {
	if e == nil {
		return NoError
	}
	return e.s
}
<|endoftext|>


<commit_before>package parse

import (
	"../types"
	"./comb"
)

const SPACE_CHARS = " ,\t\n\r"

func Parse(source string) types.Object {
	o, err := newState(source).module()()

	if err != nil {
		panic(err.Error())
	}

	return o
}

func (s *state) module() comb.Parser {
	return s.Exhaust(s.elems())
}

func (s *state) elems() comb.Parser {
	return s.Lazy(s.strictElems)
}

func (s *state) strictElems() comb.Parser {
	return s.Many(s.elem())
}

func (s *state) elem() comb.Parser {
	return s.strip(s.Or(s.atom(), s.list()))
}

func (s *state) atom() comb.Parser {
	return s.Or(s.stringLiteral(), s.identifier())
}

func (s *state) identifier() comb.Parser {
	return s.stringify(s.Many1(s.NotInString("()[]'\x00" + SPACE_CHARS)))
}

func (s *state) stringLiteral() comb.Parser {
	b := s.blank()
	c := s.Char('"')

	return s.stringify(s.Wrap(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.And(c, b)))
}

func (s *state) list() comb.Parser {
	return s.wrapChars('(', s.elems(), ')')
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(s.Char(';'), s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) wrapChars(l rune, p comb.Parser, r rune) comb.Parser {
	return s.Wrap(s.And(s.Char(l), s.blank()), p, s.strippedChar(r))
}

func (s *state) strippedChar(r rune) comb.Parser {
	return s.strip(s.Char(r))
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(b, p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.space(), s.comment())))
}

func (s *state) space() comb.Parser {
	return s.Void(s.Many1(s.InString(SPACE_CHARS)))
}

func (s *state) stringify(p comb.Parser) comb.Parser {
	f := func(any interface{}) interface{} {
		xs := any.([]interface{})
		rs := make([]rune, len(xs))

		for i, x := range xs {
			rs[i] = x.(rune)
		}

		return types.NewString(string(rs))
	}

	return s.App(f, p)
}
<commit_msg>Add quote combinator<commit_after>package parse

import (
	"../types"
	"./comb"
)

const SPACE_CHARS = " ,\t\n\r"

func Parse(source string) types.Object {
	o, err := newState(source).module()()

	if err != nil {
		panic(err.Error())
	}

	return o
}

func (s *state) module() comb.Parser {
	return s.Exhaust(s.elems())
}

func (s *state) elems() comb.Parser {
	return s.Lazy(s.strictElems)
}

func (s *state) strictElems() comb.Parser {
	return s.Many(s.elem())
}

func (s *state) elem() comb.Parser {
	return s.strip(s.Or(s.atom(), s.list()))
}

func (s *state) atom() comb.Parser {
	return s.Or(s.stringLiteral(), s.identifier())
}

func (s *state) identifier() comb.Parser {
	return s.stringify(s.Many1(s.NotInString("()[]'\x00" + SPACE_CHARS)))
}

func (s *state) stringLiteral() comb.Parser {
	b := s.blank()
	c := s.Char('"')

	return s.stringify(s.Wrap(
		c,
		s.Many(s.Or(s.NotInString("\"\\"), s.String("\\\""), s.String("\\\\"))),
		s.And(c, b)))
}

func (s *state) list() comb.Parser {
	return s.wrapChars('(', s.elems(), ')')
}

func (s *state) comment() comb.Parser {
	return s.Void(s.And(s.Char(';'), s.Many(s.NotChar('\n')), s.Char('\n')))
}

func (s *state) wrapChars(l rune, p comb.Parser, r rune) comb.Parser {
	return s.Wrap(s.And(s.Char(l), s.blank()), p, s.strippedChar(r))
}

func (s *state) strippedChar(r rune) comb.Parser {
	return s.strip(s.Char(r))
}

func (s *state) strip(p comb.Parser) comb.Parser {
	b := s.blank()
	return s.Wrap(b, p, b)
}

func (s *state) blank() comb.Parser {
	return s.Void(s.Many(s.Or(s.space(), s.comment())))
}

func (s *state) space() comb.Parser {
	return s.Void(s.Many1(s.InString(SPACE_CHARS)))
}

func (s *state) quote(p comb.Parser) comb.Parser {
	return s.And(s.Char('\''), p)
}

func (s *state) stringify(p comb.Parser) comb.Parser {
	f := func(any interface{}) interface{} {
		xs := any.([]interface{})
		rs := make([]rune, len(xs))

		for i, x := range xs {
			rs[i] = x.(rune)
		}

		return types.NewString(string(rs))
	}

	return s.App(f, p)
}
<|endoftext|>


<commit_before><commit_msg>Removing commented out function<commit_after><|endoftext|>


<commit_before>package lib

// Version is the github-nippou version
const Version = "4.1.4"
<commit_msg>:up: Bump version to 4.1.5<commit_after>package lib

// Version is the github-nippou version
const Version = "4.1.5"
<|endoftext|>