<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// How long a node is allowed to go from "Ready" to "NotReady" after a
	// reboot is issued before the test is considered failed.
	rebootNodeNotReadyTimeout = 2 * time.Minute

	// How long a node is allowed to go from "NotReady" to "Ready" after a
	// reboot is issued and it is found to be "NotReady" before the test is
	// considered failed.
	rebootNodeReadyAgainTimeout = 5 * time.Minute

	// How long pods have to be "ready" after the reboot.
	rebootPodReadyAgainTimeout = 5 * time.Minute
)

var _ = framework.KubeDescribe("Reboot [Disruptive] [Feature:Reboot]", func() {
	var f *framework.Framework

	BeforeEach(func() {
		// These tests requires SSH to nodes, so the provider check should be identical to there
		// (the limiting factor is the implementation of util.go's framework.GetSigner(...)).

		// Cluster must support node reboot
		framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			// Most of the reboot tests just make sure that addon/system pods are running, so dump
			// events for the kube-system namespace on failures
			namespaceName := api.NamespaceSystem
			By(fmt.Sprintf("Collecting events from namespace %q.", namespaceName))
			events, err := f.Client.Events(namespaceName).List(api.ListOptions{})
			Expect(err).NotTo(HaveOccurred())

			for _, e := range events.Items {
				framework.Logf("event for %v: %v %v: %v", e.InvolvedObject.Name, e.Source, e.Reason, e.Message)
			}
		}
		// In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a
		// rebooted/deleted node) for up to 5 minutes before all tunnels are dropped and recreated.  Most tests
		// make use of some proxy feature to verify functionality. So, if a reboot test runs right before a test
		// that tries to get logs, for example, we may get unlucky and try to use a closed tunnel to a node that
		// was recently rebooted. There's no good way to framework.Poll for proxies being closed, so we sleep.
		//
		// TODO(cjcullen) reduce this sleep (#19314)
		if framework.ProviderIs("gke") {
			By("waiting 5 minutes for all dead tunnels to be dropped")
			time.Sleep(5 * time.Minute)
		}
	})

	f = framework.NewDefaultFramework("reboot")

	It("each node by ordering clean reboot and ensure they function upon restart", func() {
		// clean shutdown and restart
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
	})

	It("each node by ordering unclean reboot and ensure they function upon restart", func() {
		// unclean shutdown and restart
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
	})

	It("each node by triggering kernel panic and ensure they function upon restart", func() {
		// kernel panic
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
	})

	It("each node by switching off the network interface and ensure they function upon switch on", func() {
		// switch the network interface off for a while to simulate a network outage
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo ip link set eth0 down && sleep 120 && sudo ip link set eth0 up' >/dev/null 2>&1 &")
	})

	It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {
		// tell the firewall to drop all inbound packets for a while
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
		// We still accept packages send from localhost to prevent monit from restarting kubelet.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
			" sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
	})

	It("each node by dropping all outbound packets for a while and ensure they function afterwards", func() {
		// tell the firewall to drop all outbound packets for a while
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
		// We still accept packages send to localhost to prevent monit from restarting kubelet.
		testReboot(f.Client, "nohup sh -c 'sleep 10 &&  sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
			" sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
	})
})

func testReboot(c *client.Client, rebootCmd string) {
	// Get all nodes, and kick off the test on each.
	nodelist := framework.GetReadySchedulableNodesOrDie(c)
	result := make([]bool, len(nodelist.Items))
	wg := sync.WaitGroup{}
	wg.Add(len(nodelist.Items))

	failed := false
	for ix := range nodelist.Items {
		go func(ix int) {
			defer wg.Done()
			n := nodelist.Items[ix]
			result[ix] = rebootNode(c, framework.TestContext.Provider, n.ObjectMeta.Name, rebootCmd)
			if !result[ix] {
				failed = true
			}
		}(ix)
	}

	// Wait for all to finish and check the final result.
	wg.Wait()

	if failed {
		for ix := range nodelist.Items {
			n := nodelist.Items[ix]
			if !result[ix] {
				framework.Logf("Node %s failed reboot test.", n.ObjectMeta.Name)
			}
		}
		framework.Failf("Test failed; at least one node failed to reboot in the time given.")
	}
}

func printStatusAndLogsForNotReadyPods(c *client.Client, ns string, podNames []string, pods []*api.Pod) {
	printFn := func(id, log string, err error, previous bool) {
		prefix := "Retrieving log for container"
		if previous {
			prefix = "Retrieving log for the last terminated container"
		}
		if err != nil {
			framework.Logf("%s %s, err: %v:\n%s\n", prefix, id, err, log)
		} else {
			framework.Logf("%s %s:\n%s\n", prefix, id, log)
		}
	}
	podNameSet := sets.NewString(podNames...)
	for _, p := range pods {
		if p.Namespace != ns {
			continue
		}
		if !podNameSet.Has(p.Name) {
			continue
		}
		if ok, _ := framework.PodRunningReady(p); ok {
			continue
		}
		framework.Logf("Status for not ready pod %s/%s: %+v", p.Namespace, p.Name, p.Status)
		// Print the log of the containers if pod is not running and ready.
		for _, container := range p.Status.ContainerStatuses {
			cIdentifer := fmt.Sprintf("%s/%s/%s", p.Namespace, p.Name, container.Name)
			log, err := framework.GetPodLogs(c, p.Namespace, p.Name, container.Name)
			printFn(cIdentifer, log, err, false)
			// Get log from the previous container.
			if container.RestartCount > 0 {
				printFn(cIdentifer, log, err, true)
			}
		}
	}
}

// rebootNode takes node name on provider through the following steps using c:
//  - ensures the node is ready
//  - ensures all pods on the node are running and ready
//  - reboots the node (by executing rebootCmd over ssh)
//  - ensures the node reaches some non-ready state
//  - ensures the node becomes ready again
//  - ensures all pods on the node become running and ready again
//
// It returns true through result only if all of the steps pass; at the first
// failed step, it will return false through result and not run the rest.
func rebootNode(c *client.Client, provider, name, rebootCmd string) bool {
	// Setup
	ns := api.NamespaceSystem
	ps := framework.NewPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(api.PodHostField, name))
	defer ps.Stop()

	// Get the node initially.
	framework.Logf("Getting %s", name)
	node, err := c.Nodes().Get(name)
	if err != nil {
		framework.Logf("Couldn't get node %s", name)
		return false
	}

	// Node sanity check: ensure it is "ready".
	if !framework.WaitForNodeToBeReady(c, name, framework.NodeReadyInitialTimeout) {
		return false
	}

	// Get all the pods on the node that don't have liveness probe set.
	// Liveness probe may cause restart of a pod during node reboot, and the pod may not be running.
	pods := ps.List()
	podNames := []string{}
	for _, p := range pods {
		probe := false
		for _, c := range p.Spec.Containers {
			if c.LivenessProbe != nil {
				probe = true
				break
			}
		}
		if !probe {
			podNames = append(podNames, p.ObjectMeta.Name)
		}
	}
	framework.Logf("Node %s has %d assigned pods with no liveness probes: %v", name, len(podNames), podNames)

	// For each pod, we do a sanity check to ensure it's running / healthy
	// or succeeded now, as that's what we'll be checking later.
	if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, framework.PodReadyBeforeTimeout) {
		printStatusAndLogsForNotReadyPods(c, ns, podNames, pods)
		return false
	}

	// Reboot the node.
	if err = framework.IssueSSHCommand(rebootCmd, provider, node); err != nil {
		framework.Logf("Error while issuing ssh command: %v", err)
		return false
	}

	// Wait for some kind of "not ready" status.
	if !framework.WaitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) {
		return false
	}

	// Wait for some kind of "ready" status.
	if !framework.WaitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) {
		return false
	}

	// Ensure all of the pods that we found on this node before the reboot are
	// running / healthy, or succeeded.
	if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, rebootPodReadyAgainTimeout) {
		newPods := ps.List()
		printStatusAndLogsForNotReadyPods(c, ns, podNames, newPods)
		return false
	}

	framework.Logf("Reboot successful on node %s", name)
	return true
}
<commit_msg>try to use ifdown/ifup if available<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// How long a node is allowed to go from "Ready" to "NotReady" after a
	// reboot is issued before the test is considered failed.
	rebootNodeNotReadyTimeout = 2 * time.Minute

	// How long a node is allowed to go from "NotReady" to "Ready" after a
	// reboot is issued and it is found to be "NotReady" before the test is
	// considered failed.
	rebootNodeReadyAgainTimeout = 5 * time.Minute

	// How long pods have to be "ready" after the reboot.
	rebootPodReadyAgainTimeout = 5 * time.Minute
)

var _ = framework.KubeDescribe("Reboot [Disruptive] [Feature:Reboot]", func() {
	var f *framework.Framework

	BeforeEach(func() {
		// These tests requires SSH to nodes, so the provider check should be identical to there
		// (the limiting factor is the implementation of util.go's framework.GetSigner(...)).

		// Cluster must support node reboot
		framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			// Most of the reboot tests just make sure that addon/system pods are running, so dump
			// events for the kube-system namespace on failures
			namespaceName := api.NamespaceSystem
			By(fmt.Sprintf("Collecting events from namespace %q.", namespaceName))
			events, err := f.Client.Events(namespaceName).List(api.ListOptions{})
			Expect(err).NotTo(HaveOccurred())

			for _, e := range events.Items {
				framework.Logf("event for %v: %v %v: %v", e.InvolvedObject.Name, e.Source, e.Reason, e.Message)
			}
		}
		// In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a
		// rebooted/deleted node) for up to 5 minutes before all tunnels are dropped and recreated.  Most tests
		// make use of some proxy feature to verify functionality. So, if a reboot test runs right before a test
		// that tries to get logs, for example, we may get unlucky and try to use a closed tunnel to a node that
		// was recently rebooted. There's no good way to framework.Poll for proxies being closed, so we sleep.
		//
		// TODO(cjcullen) reduce this sleep (#19314)
		if framework.ProviderIs("gke") {
			By("waiting 5 minutes for all dead tunnels to be dropped")
			time.Sleep(5 * time.Minute)
		}
	})

	f = framework.NewDefaultFramework("reboot")

	It("each node by ordering clean reboot and ensure they function upon restart", func() {
		// clean shutdown and restart
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
	})

	It("each node by ordering unclean reboot and ensure they function upon restart", func() {
		// unclean shutdown and restart
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
	})

	It("each node by triggering kernel panic and ensure they function upon restart", func() {
		// kernel panic
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
	})

	It("each node by switching off the network interface and ensure they function upon switch on", func() {
		// switch the network interface off for a while to simulate a network outage
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && (sudo ifdown eth0 || sudo ip link set eth0 down) && sleep 120 && (sudo ifup eth0 || sudo ip link set eth0 up)' >/dev/null 2>&1 &")
	})

	It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {
		// tell the firewall to drop all inbound packets for a while
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
		// We still accept packages send from localhost to prevent monit from restarting kubelet.
		testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
			" sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
	})

	It("each node by dropping all outbound packets for a while and ensure they function afterwards", func() {
		// tell the firewall to drop all outbound packets for a while
		// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
		// We still accept packages send to localhost to prevent monit from restarting kubelet.
		testReboot(f.Client, "nohup sh -c 'sleep 10 &&  sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
			" sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
	})
})

func testReboot(c *client.Client, rebootCmd string) {
	// Get all nodes, and kick off the test on each.
	nodelist := framework.GetReadySchedulableNodesOrDie(c)
	result := make([]bool, len(nodelist.Items))
	wg := sync.WaitGroup{}
	wg.Add(len(nodelist.Items))

	failed := false
	for ix := range nodelist.Items {
		go func(ix int) {
			defer wg.Done()
			n := nodelist.Items[ix]
			result[ix] = rebootNode(c, framework.TestContext.Provider, n.ObjectMeta.Name, rebootCmd)
			if !result[ix] {
				failed = true
			}
		}(ix)
	}

	// Wait for all to finish and check the final result.
	wg.Wait()

	if failed {
		for ix := range nodelist.Items {
			n := nodelist.Items[ix]
			if !result[ix] {
				framework.Logf("Node %s failed reboot test.", n.ObjectMeta.Name)
			}
		}
		framework.Failf("Test failed; at least one node failed to reboot in the time given.")
	}
}

func printStatusAndLogsForNotReadyPods(c *client.Client, ns string, podNames []string, pods []*api.Pod) {
	printFn := func(id, log string, err error, previous bool) {
		prefix := "Retrieving log for container"
		if previous {
			prefix = "Retrieving log for the last terminated container"
		}
		if err != nil {
			framework.Logf("%s %s, err: %v:\n%s\n", prefix, id, err, log)
		} else {
			framework.Logf("%s %s:\n%s\n", prefix, id, log)
		}
	}
	podNameSet := sets.NewString(podNames...)
	for _, p := range pods {
		if p.Namespace != ns {
			continue
		}
		if !podNameSet.Has(p.Name) {
			continue
		}
		if ok, _ := framework.PodRunningReady(p); ok {
			continue
		}
		framework.Logf("Status for not ready pod %s/%s: %+v", p.Namespace, p.Name, p.Status)
		// Print the log of the containers if pod is not running and ready.
		for _, container := range p.Status.ContainerStatuses {
			cIdentifer := fmt.Sprintf("%s/%s/%s", p.Namespace, p.Name, container.Name)
			log, err := framework.GetPodLogs(c, p.Namespace, p.Name, container.Name)
			printFn(cIdentifer, log, err, false)
			// Get log from the previous container.
			if container.RestartCount > 0 {
				printFn(cIdentifer, log, err, true)
			}
		}
	}
}

// rebootNode takes node name on provider through the following steps using c:
//  - ensures the node is ready
//  - ensures all pods on the node are running and ready
//  - reboots the node (by executing rebootCmd over ssh)
//  - ensures the node reaches some non-ready state
//  - ensures the node becomes ready again
//  - ensures all pods on the node become running and ready again
//
// It returns true through result only if all of the steps pass; at the first
// failed step, it will return false through result and not run the rest.
func rebootNode(c *client.Client, provider, name, rebootCmd string) bool {
	// Setup
	ns := api.NamespaceSystem
	ps := framework.NewPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(api.PodHostField, name))
	defer ps.Stop()

	// Get the node initially.
	framework.Logf("Getting %s", name)
	node, err := c.Nodes().Get(name)
	if err != nil {
		framework.Logf("Couldn't get node %s", name)
		return false
	}

	// Node sanity check: ensure it is "ready".
	if !framework.WaitForNodeToBeReady(c, name, framework.NodeReadyInitialTimeout) {
		return false
	}

	// Get all the pods on the node that don't have liveness probe set.
	// Liveness probe may cause restart of a pod during node reboot, and the pod may not be running.
	pods := ps.List()
	podNames := []string{}
	for _, p := range pods {
		probe := false
		for _, c := range p.Spec.Containers {
			if c.LivenessProbe != nil {
				probe = true
				break
			}
		}
		if !probe {
			podNames = append(podNames, p.ObjectMeta.Name)
		}
	}
	framework.Logf("Node %s has %d assigned pods with no liveness probes: %v", name, len(podNames), podNames)

	// For each pod, we do a sanity check to ensure it's running / healthy
	// or succeeded now, as that's what we'll be checking later.
	if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, framework.PodReadyBeforeTimeout) {
		printStatusAndLogsForNotReadyPods(c, ns, podNames, pods)
		return false
	}

	// Reboot the node.
	if err = framework.IssueSSHCommand(rebootCmd, provider, node); err != nil {
		framework.Logf("Error while issuing ssh command: %v", err)
		return false
	}

	// Wait for some kind of "not ready" status.
	if !framework.WaitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) {
		return false
	}

	// Wait for some kind of "ready" status.
	if !framework.WaitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) {
		return false
	}

	// Ensure all of the pods that we found on this node before the reboot are
	// running / healthy, or succeeded.
	if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, rebootPodReadyAgainTimeout) {
		newPods := ps.List()
		printStatusAndLogsForNotReadyPods(c, ns, podNames, newPods)
		return false
	}

	framework.Logf("Reboot successful on node %s", name)
	return true
}
<|endoftext|>


<commit_before>package v8worker

/*
#cgo CXXFLAGS: -std=c++11
#cgo pkg-config: v8.pc
#include <stdlib.h>
#include "binding.h"
*/
import "C"
import "errors"

import "unsafe"
import "sync"
import "runtime"

type workerTableIndex int

var workerTableLock sync.Mutex

// This table will store all pointers to all active workers. Because we can't safely
// pass pointers to Go objects to C, we instead pass a key to this table.
var workerTable = make(map[workerTableIndex]*Worker)

// Keeps track of the last used table index. Incremeneted when a worker is created.
var workerTableNextAvailable workerTableIndex = 0

// To receive messages from javascript...
type ReceiveMessageCallback func(msg string)

// To send a message from javascript and synchronously return a string.
type ReceiveSyncMessageCallback func(msg string) string

// Don't init V8 more than once.
var initV8Once sync.Once

// This is a golang wrapper around a single V8 Isolate.
type Worker struct {
	cWorker    *C.worker
	cb         ReceiveMessageCallback
	sync_cb    ReceiveSyncMessageCallback
	tableIndex workerTableIndex
}

// Return the V8 version E.G. "4.3.59"
func Version() string {
	return C.GoString(C.worker_version())
}

func workerTableLookup(index workerTableIndex) *Worker {
	workerTableLock.Lock()
	defer workerTableLock.Unlock()
	return workerTable[index]
}

//export recvCb
func recvCb(msg_s *C.char, index workerTableIndex) {
	msg := C.GoString(msg_s)
	worker := workerTableLookup(index)
	worker.cb(msg)
}

//export recvSyncCb
func recvSyncCb(msg_s *C.char, index workerTableIndex) *C.char {
	msg := C.GoString(msg_s)
	worker := workerTableLookup(index)
	return_s := C.CString(worker.sync_cb(msg))
	return return_s
}

// Creates a new worker, which corresponds to a V8 isolate. A single threaded
// standalone execution context.
func New(cb ReceiveMessageCallback, sync_cb ReceiveSyncMessageCallback) *Worker {
	workerTableLock.Lock()
	worker := &Worker{
		cb:         cb,
		sync_cb:    sync_cb,
		tableIndex: workerTableNextAvailable,
	}

	workerTableNextAvailable++
	workerTable[worker.tableIndex] = worker
	workerTableLock.Unlock()

	initV8Once.Do(func() {
		C.v8_init()
	})

	callback := C.worker_recv_cb(C.go_recv_cb)
	receiveSync_callback := C.worker_recv_sync_cb(C.go_recv_sync_cb)

	worker.cWorker = C.worker_new(callback, receiveSync_callback, C.int(worker.tableIndex))
	runtime.SetFinalizer(worker, func(final_worker *Worker) {
		workerTableLock.Lock()
		delete(workerTable, final_worker.tableIndex)
		workerTableLock.Unlock()
		C.worker_dispose(final_worker.cWorker)
	})
	return worker
}

// Load and executes a javascript file with the filename specified by
// scriptName and the contents of the file specified by the param code.
func (w *Worker) Load(scriptName string, code string) error {
	scriptName_s := C.CString(scriptName)
	code_s := C.CString(code)
	defer C.free(unsafe.Pointer(scriptName_s))
	defer C.free(unsafe.Pointer(code_s))

	r := C.worker_load(w.cWorker, scriptName_s, code_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}
	return nil
}

// Sends a message to a worker. The $recv callback in js will be called.
func (w *Worker) Send(msg string) error {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	r := C.worker_send(w.cWorker, msg_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}

	return nil
}

// SendSync sends a message to a worker. The $recvSync callback in js will be called.
// That callback will return a string which is passed to golang and used as the return value of SendSync.
func (w *Worker) SendSync(msg string) string {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	svalue := C.worker_send_sync(w.cWorker, msg_s)
	return C.GoString(svalue)
}

// Terminates execution of javascript
func (w *Worker) TerminateExecution() {
	C.worker_terminate_execution(w.cWorker)
}
<commit_msg>Fix memory leak with workerTable<commit_after>package v8worker

/*
#cgo CXXFLAGS: -std=c++11
#cgo pkg-config: v8.pc
#include <stdlib.h>
#include "binding.h"
*/
import "C"
import "errors"

import "unsafe"
import "sync"
import "runtime"

type workerTableIndex int

var workerTableLock sync.Mutex

// This table will store all pointers to all active workers. Because we can't safely
// pass pointers to Go objects to C, we instead pass a key to this table.
var workerTable = make(map[workerTableIndex]*worker)

// Keeps track of the last used table index. Incremeneted when a worker is created.
var workerTableNextAvailable workerTableIndex = 0

// To receive messages from javascript...
type ReceiveMessageCallback func(msg string)

// To send a message from javascript and synchronously return a string.
type ReceiveSyncMessageCallback func(msg string) string

// Don't init V8 more than once.
var initV8Once sync.Once

// Internal worker struct which is stored in the workerTable.
// Weak-ref pattern https://groups.google.com/forum/#!topic/golang-nuts/1ItNOOj8yW8/discussion
type worker struct {
	cWorker    *C.worker
	cb         ReceiveMessageCallback
	sync_cb    ReceiveSyncMessageCallback
	tableIndex workerTableIndex
}

// This is a golang wrapper around a single V8 Isolate.
type Worker struct {
	*worker
}

// Return the V8 version E.G. "4.3.59"
func Version() string {
	return C.GoString(C.worker_version())
}

func workerTableLookup(index workerTableIndex) *worker {
	workerTableLock.Lock()
	defer workerTableLock.Unlock()
	return workerTable[index]
}

//export recvCb
func recvCb(msg_s *C.char, index workerTableIndex) {
	msg := C.GoString(msg_s)
	w := workerTableLookup(index)
	w.cb(msg)
}

//export recvSyncCb
func recvSyncCb(msg_s *C.char, index workerTableIndex) *C.char {
	msg := C.GoString(msg_s)
	w := workerTableLookup(index)
	return_s := C.CString(w.sync_cb(msg))
	return return_s
}

// Creates a new worker, which corresponds to a V8 isolate. A single threaded
// standalone execution context.
func New(cb ReceiveMessageCallback, sync_cb ReceiveSyncMessageCallback) *Worker {
	workerTableLock.Lock()
	w := &worker{
		cb:         cb,
		sync_cb:    sync_cb,
		tableIndex: workerTableNextAvailable,
	}

	workerTableNextAvailable++
	workerTable[w.tableIndex] = w
	workerTableLock.Unlock()

	initV8Once.Do(func() {
		C.v8_init()
	})

	callback := C.worker_recv_cb(C.go_recv_cb)
	receiveSync_callback := C.worker_recv_sync_cb(C.go_recv_sync_cb)

	w.cWorker = C.worker_new(callback, receiveSync_callback, C.int(w.tableIndex))

	externalWorker := &Worker{worker: w}

	runtime.SetFinalizer(externalWorker, func(final_worker *Worker) {
		workerTableLock.Lock()
		w := final_worker.worker
		delete(workerTable, w.tableIndex)
		workerTableLock.Unlock()
		C.worker_dispose(w.cWorker)
	})
	return externalWorker
}

// Load and executes a javascript file with the filename specified by
// scriptName and the contents of the file specified by the param code.
func (w *Worker) Load(scriptName string, code string) error {
	scriptName_s := C.CString(scriptName)
	code_s := C.CString(code)
	defer C.free(unsafe.Pointer(scriptName_s))
	defer C.free(unsafe.Pointer(code_s))

	r := C.worker_load(w.worker.cWorker, scriptName_s, code_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.worker.cWorker))
		return errors.New(errStr)
	}
	return nil
}

// Sends a message to a worker. The $recv callback in js will be called.
func (w *Worker) Send(msg string) error {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	r := C.worker_send(w.worker.cWorker, msg_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.worker.cWorker))
		return errors.New(errStr)
	}

	return nil
}

// SendSync sends a message to a worker. The $recvSync callback in js will be called.
// That callback will return a string which is passed to golang and used as the return value of SendSync.
func (w *Worker) SendSync(msg string) string {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	svalue := C.worker_send_sync(w.worker.cWorker, msg_s)
	return C.GoString(svalue)
}

// Terminates execution of javascript
func (w *Worker) TerminateExecution() {
	C.worker_terminate_execution(w.worker.cWorker)
}
<|endoftext|>


<commit_before>package logging

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
)

const (
	// logBufferSize is the size of the buffer.
	logBufferSize = 64 * 1024

	// bufferFlushDuration is the duration at which we flush the buffer.
	bufferFlushDuration = 100 * time.Millisecond

	// lineScanLimit is the number of bytes we will attempt to scan for new
	// lines when approaching the end of the file to avoid a log line being
	// split between two files. Any single line that is greater than this limit
	// may be split.
	lineScanLimit = 32 * 1024

	// newLineDelimiter is the delimiter used for new lines.
	newLineDelimiter = '\n'
)

// FileRotator writes bytes to a rotated set of files
type FileRotator struct {
	MaxFiles int   // MaxFiles is the maximum number of rotated files allowed in a path
	FileSize int64 // FileSize is the size a rotated file is allowed to grow

	path             string // path is the path on the file system where the rotated set of files are opened
	baseFileName     string // baseFileName is the base file name of the rotated files
	logFileIdx       int    // logFileIdx is the current index of the rotated files
	oldestLogFileIdx int    // oldestLogFileIdx is the index of the oldest log file in a path

	currentFile *os.File // currentFile is the file that is currently getting written
	currentWr   int64    // currentWr is the number of bytes written to the current file
	bufw        *bufio.Writer
	bufLock     sync.Mutex

	flushTicker *time.Ticker
	logger      hclog.Logger
	purgeCh     chan struct{}
	doneCh      chan struct{}

	closed     bool
	closedLock sync.Mutex
}

// NewFileRotator returns a new file rotator
func NewFileRotator(path string, baseFile string, maxFiles int,
	fileSize int64, logger hclog.Logger) (*FileRotator, error) {
	logger = logger.Named("rotator")
	rotator := &FileRotator{
		MaxFiles: maxFiles,
		FileSize: fileSize,

		path:         path,
		baseFileName: baseFile,

		flushTicker: time.NewTicker(bufferFlushDuration),
		logger:      logger,
		purgeCh:     make(chan struct{}, 1),
		doneCh:      make(chan struct{}, 1),
	}

	if err := rotator.lastFile(); err != nil {
		return nil, err
	}
	go rotator.purgeOldFiles()
	go rotator.flushPeriodically()
	return rotator, nil
}

// Write writes a byte array to a file and rotates the file if it's size becomes
// equal to the maximum size the user has defined.
func (f *FileRotator) Write(p []byte) (n int, err error) {
	n = 0
	var forceRotate bool

	for n < len(p) {
		// Check if we still have space in the current file, otherwise close and
		// open the next file
		if forceRotate || f.currentWr >= f.FileSize {
			forceRotate = false
			f.flushBuffer()
			f.currentFile.Close()
			if err := f.nextFile(); err != nil {
				f.logger.Error("error creating next file", "err", err)
				return 0, err
			}
		}
		// Calculate the remaining size on this file and how much we have left
		// to write
		remainingSpace := f.FileSize - f.currentWr
		remainingToWrite := int64(len(p[n:]))

		// Check if we are near the end of the file. If we are we attempt to
		// avoid a log line being split between two files.
		var nw int
		if (remainingSpace - lineScanLimit) < remainingToWrite {
			// Scan for new line and if the data up to new line fits in current
			// file, write to buffer
			idx := bytes.IndexByte(p[n:], newLineDelimiter)
			if idx >= 0 && (remainingSpace-int64(idx)-1) >= 0 {
				// We have space so write it to buffer
				nw, err = f.writeToBuffer(p[n : n+idx+1])
			} else if idx >= 0 {
				// We found a new line but don't have space so just force rotate
				forceRotate = true
			} else if remainingToWrite > f.FileSize || f.FileSize-lineScanLimit < 0 {
				// There is no new line remaining but there is no point in
				// rotating since the remaining data will not even fit in the
				// next file either so just fill this one up.
				li := int64(n) + remainingSpace
				if remainingSpace > remainingToWrite {
					li = int64(n) + remainingToWrite
				}
				nw, err = f.writeToBuffer(p[n:li])
			} else {
				// There is no new line in the data remaining for us to write
				// and it will fit in the next file so rotate.
				forceRotate = true
			}
		} else {
			// Write all the bytes in the current file
			nw, err = f.writeToBuffer(p[n:])
		}

		// Increment the number of bytes written so far in this method
		// invocation
		n += nw

		// Increment the total number of bytes in the file
		f.currentWr += int64(n)
		if err != nil {
			f.logger.Error("error writing to file", "err", err)
			return
		}
	}
	return
}

// nextFile opens the next file and purges older files if the number of rotated
// files is larger than the maximum files configured by the user
func (f *FileRotator) nextFile() error {
	nextFileIdx := f.logFileIdx
	for {
		nextFileIdx += 1
		logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, nextFileIdx))
		if fi, err := os.Stat(logFileName); err == nil {
			if fi.IsDir() || fi.Size() >= f.FileSize {
				continue
			}
		}
		f.logFileIdx = nextFileIdx
		if err := f.createFile(); err != nil {
			return err
		}
		break
	}
	// Purge old files if we have more files than MaxFiles
	f.closedLock.Lock()
	defer f.closedLock.Unlock()
	if f.logFileIdx-f.oldestLogFileIdx >= f.MaxFiles && !f.closed {
		select {
		case f.purgeCh <- struct{}{}:
		default:
		}
	}
	return nil
}

// lastFile finds out the rotated file with the largest index in a path.
func (f *FileRotator) lastFile() error {
	finfos, err := ioutil.ReadDir(f.path)
	if err != nil {
		return err
	}

	prefix := fmt.Sprintf("%s.", f.baseFileName)
	for _, fi := range finfos {
		if fi.IsDir() {
			continue
		}
		if strings.HasPrefix(fi.Name(), prefix) {
			fileIdx := strings.TrimPrefix(fi.Name(), prefix)
			n, err := strconv.Atoi(fileIdx)
			if err != nil {
				continue
			}
			if n > f.logFileIdx {
				f.logFileIdx = n
			}
		}
	}
	if err := f.createFile(); err != nil {
		return err
	}
	return nil
}

// createFile opens a new or existing file for writing
func (f *FileRotator) createFile() error {
	logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, f.logFileIdx))
	cFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}

	f.currentFile = cFile
	fi, err := f.currentFile.Stat()
	if err != nil {
		return err
	}
	f.currentWr = fi.Size()
	f.createOrResetBuffer()
	return nil
}

// flushPeriodically flushes the buffered writer every 100ms to the underlying
// file
func (f *FileRotator) flushPeriodically() {
	for range f.flushTicker.C {
		f.flushBuffer()
	}
}

func (f *FileRotator) Close() {
	f.closedLock.Lock()
	defer f.closedLock.Unlock()

	// Stop the ticker and flush for one last time
	f.flushTicker.Stop()
	f.flushBuffer()

	// Stop the purge go routine
	if !f.closed {
		f.doneCh <- struct{}{}
		close(f.purgeCh)
		f.closed = true
	}
}

// purgeOldFiles removes older files and keeps only the last N files rotated for
// a file
func (f *FileRotator) purgeOldFiles() {
	for {
		select {
		case <-f.purgeCh:
			var fIndexes []int
			files, err := ioutil.ReadDir(f.path)
			if err != nil {
				f.logger.Error("error getting directory listing", "err", err)
				return
			}
			// Inserting all the rotated files in a slice
			for _, fi := range files {
				if strings.HasPrefix(fi.Name(), f.baseFileName) {
					fileIdx := strings.TrimPrefix(fi.Name(), fmt.Sprintf("%s.", f.baseFileName))
					n, err := strconv.Atoi(fileIdx)
					if err != nil {
						f.logger.Error("error extracting file index", "err", err)
						continue
					}
					fIndexes = append(fIndexes, n)
				}
			}

			// Not continuing to delete files if the number of files is not more
			// than MaxFiles
			if len(fIndexes) <= f.MaxFiles {
				continue
			}

			// Sorting the file indexes so that we can purge the older files and keep
			// only the number of files as configured by the user
			sort.Sort(sort.IntSlice(fIndexes))
			toDelete := fIndexes[0 : len(fIndexes)-f.MaxFiles]
			for _, fIndex := range toDelete {
				fname := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, fIndex))
				err := os.RemoveAll(fname)
				if err != nil {
					f.logger.Error("error removing file", "filename", fname, "err", err)
				}
			}
			f.oldestLogFileIdx = fIndexes[0]
		case <-f.doneCh:
			return
		}
	}
}

// flushBuffer flushes the buffer
func (f *FileRotator) flushBuffer() error {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw != nil {
		return f.bufw.Flush()
	}
	return nil
}

// writeToBuffer writes the byte array to buffer
func (f *FileRotator) writeToBuffer(p []byte) (int, error) {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	return f.bufw.Write(p)
}

// createOrResetBuffer creates a new buffer if we don't have one otherwise
// resets the buffer
func (f *FileRotator) createOrResetBuffer() {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw == nil {
		f.bufw = bufio.NewWriterSize(f.currentFile, logBufferSize)
	} else {
		f.bufw.Reset(f.currentFile)
	}
}
<commit_msg>Recover from any possible io error when invoking Write on FileRotator<commit_after>package logging

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
)

const (
	// logBufferSize is the size of the buffer.
	logBufferSize = 64 * 1024

	// bufferFlushDuration is the duration at which we flush the buffer.
	bufferFlushDuration = 100 * time.Millisecond

	// lineScanLimit is the number of bytes we will attempt to scan for new
	// lines when approaching the end of the file to avoid a log line being
	// split between two files. Any single line that is greater than this limit
	// may be split.
	lineScanLimit = 32 * 1024

	// newLineDelimiter is the delimiter used for new lines.
	newLineDelimiter = '\n'
)

// FileRotator writes bytes to a rotated set of files
type FileRotator struct {
	MaxFiles int   // MaxFiles is the maximum number of rotated files allowed in a path
	FileSize int64 // FileSize is the size a rotated file is allowed to grow

	path             string // path is the path on the file system where the rotated set of files are opened
	baseFileName     string // baseFileName is the base file name of the rotated files
	logFileIdx       int    // logFileIdx is the current index of the rotated files
	oldestLogFileIdx int    // oldestLogFileIdx is the index of the oldest log file in a path

	currentFile *os.File // currentFile is the file that is currently getting written
	currentWr   int64    // currentWr is the number of bytes written to the current file
	bufw        *bufio.Writer
	bufLock     sync.Mutex

	flushTicker *time.Ticker
	logger      hclog.Logger
	purgeCh     chan struct{}
	doneCh      chan struct{}

	closed     bool
	closedLock sync.Mutex
}

// NewFileRotator returns a new file rotator
func NewFileRotator(path string, baseFile string, maxFiles int,
	fileSize int64, logger hclog.Logger) (*FileRotator, error) {
	logger = logger.Named("rotator")
	rotator := &FileRotator{
		MaxFiles: maxFiles,
		FileSize: fileSize,

		path:         path,
		baseFileName: baseFile,

		flushTicker: time.NewTicker(bufferFlushDuration),
		logger:      logger,
		purgeCh:     make(chan struct{}, 1),
		doneCh:      make(chan struct{}, 1),
	}

	if err := rotator.lastFile(); err != nil {
		return nil, err
	}
	go rotator.purgeOldFiles()
	go rotator.flushPeriodically()
	return rotator, nil
}

// Write writes a byte array to a file and rotates the file if it's size becomes
// equal to the maximum size the user has defined.
func (f *FileRotator) Write(p []byte) (n int, err error) {
	n = 0
	var forceRotate bool

	for n < len(p) {
		// Check if we still have space in the current file, otherwise close and
		// open the next file
		if forceRotate || f.currentWr >= f.FileSize {
			forceRotate = false
			f.flushBuffer()
			f.currentFile.Close()
			if err := f.nextFile(); err != nil {
				f.logger.Error("error creating next file", "err", err)
				return 0, err
			}
		}
		// Calculate the remaining size on this file and how much we have left
		// to write
		remainingSpace := f.FileSize - f.currentWr
		remainingToWrite := int64(len(p[n:]))

		// Check if we are near the end of the file. If we are we attempt to
		// avoid a log line being split between two files.
		var nw int
		if (remainingSpace - lineScanLimit) < remainingToWrite {
			// Scan for new line and if the data up to new line fits in current
			// file, write to buffer
			idx := bytes.IndexByte(p[n:], newLineDelimiter)
			if idx >= 0 && (remainingSpace-int64(idx)-1) >= 0 {
				// We have space so write it to buffer
				nw, err = f.writeToBuffer(p[n : n+idx+1])
			} else if idx >= 0 {
				// We found a new line but don't have space so just force rotate
				forceRotate = true
			} else if remainingToWrite > f.FileSize || f.FileSize-lineScanLimit < 0 {
				// There is no new line remaining but there is no point in
				// rotating since the remaining data will not even fit in the
				// next file either so just fill this one up.
				li := int64(n) + remainingSpace
				if remainingSpace > remainingToWrite {
					li = int64(n) + remainingToWrite
				}
				nw, err = f.writeToBuffer(p[n:li])
			} else {
				// There is no new line in the data remaining for us to write
				// and it will fit in the next file so rotate.
				forceRotate = true
			}
		} else {
			// Write all the bytes in the current file
			nw, err = f.writeToBuffer(p[n:])
		}

		// Increment the number of bytes written so far in this method
		// invocation
		n += nw

		// Increment the total number of bytes in the file
		f.currentWr += int64(n)
		if err != nil {
			f.logger.Error("error writing to file", "err", err)

			// As bufio writer does not automatically recover in case of any
			// io error, we need to recover from it manually resetting the
			// writter.
			f.createOrResetBuffer()

			return
		}
	}
	return
}

// nextFile opens the next file and purges older files if the number of rotated
// files is larger than the maximum files configured by the user
func (f *FileRotator) nextFile() error {
	nextFileIdx := f.logFileIdx
	for {
		nextFileIdx += 1
		logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, nextFileIdx))
		if fi, err := os.Stat(logFileName); err == nil {
			if fi.IsDir() || fi.Size() >= f.FileSize {
				continue
			}
		}
		f.logFileIdx = nextFileIdx
		if err := f.createFile(); err != nil {
			return err
		}
		break
	}
	// Purge old files if we have more files than MaxFiles
	f.closedLock.Lock()
	defer f.closedLock.Unlock()
	if f.logFileIdx-f.oldestLogFileIdx >= f.MaxFiles && !f.closed {
		select {
		case f.purgeCh <- struct{}{}:
		default:
		}
	}
	return nil
}

// lastFile finds out the rotated file with the largest index in a path.
func (f *FileRotator) lastFile() error {
	finfos, err := ioutil.ReadDir(f.path)
	if err != nil {
		return err
	}

	prefix := fmt.Sprintf("%s.", f.baseFileName)
	for _, fi := range finfos {
		if fi.IsDir() {
			continue
		}
		if strings.HasPrefix(fi.Name(), prefix) {
			fileIdx := strings.TrimPrefix(fi.Name(), prefix)
			n, err := strconv.Atoi(fileIdx)
			if err != nil {
				continue
			}
			if n > f.logFileIdx {
				f.logFileIdx = n
			}
		}
	}
	if err := f.createFile(); err != nil {
		return err
	}
	return nil
}

// createFile opens a new or existing file for writing
func (f *FileRotator) createFile() error {
	logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, f.logFileIdx))
	cFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}

	f.currentFile = cFile
	fi, err := f.currentFile.Stat()
	if err != nil {
		return err
	}
	f.currentWr = fi.Size()
	f.createOrResetBuffer()
	return nil
}

// flushPeriodically flushes the buffered writer every 100ms to the underlying
// file
func (f *FileRotator) flushPeriodically() {
	for range f.flushTicker.C {
		f.flushBuffer()
	}
}

func (f *FileRotator) Close() {
	f.closedLock.Lock()
	defer f.closedLock.Unlock()

	// Stop the ticker and flush for one last time
	f.flushTicker.Stop()
	f.flushBuffer()

	// Stop the purge go routine
	if !f.closed {
		f.doneCh <- struct{}{}
		close(f.purgeCh)
		f.closed = true
	}
}

// purgeOldFiles removes older files and keeps only the last N files rotated for
// a file
func (f *FileRotator) purgeOldFiles() {
	for {
		select {
		case <-f.purgeCh:
			var fIndexes []int
			files, err := ioutil.ReadDir(f.path)
			if err != nil {
				f.logger.Error("error getting directory listing", "err", err)
				return
			}
			// Inserting all the rotated files in a slice
			for _, fi := range files {
				if strings.HasPrefix(fi.Name(), f.baseFileName) {
					fileIdx := strings.TrimPrefix(fi.Name(), fmt.Sprintf("%s.", f.baseFileName))
					n, err := strconv.Atoi(fileIdx)
					if err != nil {
						f.logger.Error("error extracting file index", "err", err)
						continue
					}
					fIndexes = append(fIndexes, n)
				}
			}

			// Not continuing to delete files if the number of files is not more
			// than MaxFiles
			if len(fIndexes) <= f.MaxFiles {
				continue
			}

			// Sorting the file indexes so that we can purge the older files and keep
			// only the number of files as configured by the user
			sort.Sort(sort.IntSlice(fIndexes))
			toDelete := fIndexes[0 : len(fIndexes)-f.MaxFiles]
			for _, fIndex := range toDelete {
				fname := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, fIndex))
				err := os.RemoveAll(fname)
				if err != nil {
					f.logger.Error("error removing file", "filename", fname, "err", err)
				}
			}
			f.oldestLogFileIdx = fIndexes[0]
		case <-f.doneCh:
			return
		}
	}
}

// flushBuffer flushes the buffer
func (f *FileRotator) flushBuffer() error {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw != nil {
		return f.bufw.Flush()
	}
	return nil
}

// writeToBuffer writes the byte array to buffer
func (f *FileRotator) writeToBuffer(p []byte) (int, error) {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	return f.bufw.Write(p)
}

// createOrResetBuffer creates a new buffer if we don't have one otherwise
// resets the buffer
func (f *FileRotator) createOrResetBuffer() {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw == nil {
		f.bufw = bufio.NewWriterSize(f.currentFile, logBufferSize)
	} else {
		f.bufw.Reset(f.currentFile)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net"
	"strconv"
	"time"

	"github.com/ugorji/go/codec"
	"github.com/yosisa/fluxion/message"
	"github.com/yosisa/fluxion/plugin"
)

var mh = &codec.MsgpackHandle{RawToString: true}

type Config struct {
	Bind string `toml:"bind"`
}

type ForwardInput struct {
	env     *plugin.Env
	conf    *Config
	ln      net.Listener
	udpConn *net.UDPConn
	closed  bool
}

func (i *ForwardInput) Init(env *plugin.Env) error {
	i.env = env
	i.conf = &Config{}
	return env.ReadConfig(i.conf)
}

func (i *ForwardInput) Start() error {
	ln, err := net.Listen("tcp", i.conf.Bind)
	if err != nil {
		return err
	}
	addr, err := net.ResolveUDPAddr("udp", i.conf.Bind)
	if err != nil {
		ln.Close()
		return err
	}
	sock, err := net.ListenUDP("udp", addr)
	if err != nil {
		ln.Close()
		return err
	}

	i.ln = ln
	i.udpConn = sock
	go i.accept()
	go i.heartbeatHandler()
	return nil
}

func (i *ForwardInput) Close() error {
	i.closed = true
	err1 := i.ln.Close()
	err2 := i.udpConn.Close()
	if err1 != nil {
		return err1
	}
	return err2
}

func (i *ForwardInput) accept() {
	for !i.closed {
		conn, err := i.ln.Accept()
		if err != nil {
			continue
		}
		go i.handleConnection(conn)
	}
}

func (i *ForwardInput) handleConnection(conn net.Conn) {
	dec := codec.NewDecoder(conn, mh)
	for {
		var v []interface{}
		err := dec.Decode(&v)
		if err != nil {
			if err != io.EOF {
				i.env.Log.Warning(err)
			}
			return
		}

		tag := v[0].(string)
		switch len(v) {
		case 2:
			dec2 := codec.NewDecoderBytes([]byte(v[1].(string)), mh)
			for {
				var v2 []interface{}
				if err := dec2.Decode(&v2); err != nil {
					if err != io.EOF {
						i.env.Log.Warning(err)
					}
					break
				}

				t, err := parseTime(v2[0])
				if err != nil {
					i.env.Log.Errorf("Time decode error: %v, skipping", err)
					continue
				}
				r := message.NewEventWithTime(tag, t, parseValue(v2[1]))
				i.env.Emit(r)
			}
		case 3:
			t, err := parseTime(v[1])
			if err != nil {
				i.env.Log.Errorf("Time decode error: %v, skipping", err)
				continue
			}
			r := message.NewEventWithTime(tag, t, parseValue(v[2]))
			i.env.Emit(r)
		}
	}
}

func (i *ForwardInput) heartbeatHandler() {
	buf := make([]byte, 1024)
	res := []byte{0}
	for {
		_, remote, err := i.udpConn.ReadFromUDP(buf)
		if i.closed {
			return
		}
		if err != nil {
			i.env.Log.Warning(err)
			continue
		}
		i.udpConn.WriteToUDP(res, remote)
	}
}

func parseTime(v interface{}) (t time.Time, err error) {
	var n int64
	switch typed := v.(type) {
	case []byte:
		err = t.UnmarshalBinary(typed)
		return
	case string:
		err = t.UnmarshalBinary([]byte(typed))
		return
	case int64:
		n = typed
	case uint64:
		n = int64(typed)
	case float64:
		nsec := int64((typed - float64(int64(typed))) * 1e9)
		t = time.Unix(int64(typed), nsec)
		return
	default:
		if n, err = strconv.ParseInt(fmt.Sprintf("%v", typed), 10, 64); err != nil {
			return
		}
	}
	t = time.Unix(n, 0)
	return
}

func parseValue(v interface{}) map[string]interface{} {
	r := make(map[string]interface{})
	for key, val := range v.(map[interface{}]interface{}) {
		r[key.(string)] = val
	}
	return r
}

func main() {
	plugin.New("in-forward", func() plugin.Plugin {
		return &ForwardInput{}
	}).Run()
}
<commit_msg>Enforce map type<commit_after>package main

import (
	"fmt"
	"io"
	"net"
	"reflect"
	"strconv"
	"time"

	"github.com/ugorji/go/codec"
	"github.com/yosisa/fluxion/message"
	"github.com/yosisa/fluxion/plugin"
)

var mh = &codec.MsgpackHandle{RawToString: true}

type Config struct {
	Bind string `toml:"bind"`
}

type ForwardInput struct {
	env     *plugin.Env
	conf    *Config
	ln      net.Listener
	udpConn *net.UDPConn
	closed  bool
}

func (i *ForwardInput) Init(env *plugin.Env) error {
	i.env = env
	i.conf = &Config{}
	return env.ReadConfig(i.conf)
}

func (i *ForwardInput) Start() error {
	ln, err := net.Listen("tcp", i.conf.Bind)
	if err != nil {
		return err
	}
	addr, err := net.ResolveUDPAddr("udp", i.conf.Bind)
	if err != nil {
		ln.Close()
		return err
	}
	sock, err := net.ListenUDP("udp", addr)
	if err != nil {
		ln.Close()
		return err
	}

	i.ln = ln
	i.udpConn = sock
	go i.accept()
	go i.heartbeatHandler()
	return nil
}

func (i *ForwardInput) Close() error {
	i.closed = true
	err1 := i.ln.Close()
	err2 := i.udpConn.Close()
	if err1 != nil {
		return err1
	}
	return err2
}

func (i *ForwardInput) accept() {
	for !i.closed {
		conn, err := i.ln.Accept()
		if err != nil {
			continue
		}
		go i.handleConnection(conn)
	}
}

func (i *ForwardInput) handleConnection(conn net.Conn) {
	dec := codec.NewDecoder(conn, mh)
	for {
		var v []interface{}
		err := dec.Decode(&v)
		if err != nil {
			if err != io.EOF {
				i.env.Log.Warning(err)
			}
			return
		}

		tag := v[0].(string)
		switch len(v) {
		case 2:
			dec2 := codec.NewDecoderBytes([]byte(v[1].(string)), mh)
			for {
				var v2 []interface{}
				if err := dec2.Decode(&v2); err != nil {
					if err != io.EOF {
						i.env.Log.Warning(err)
					}
					break
				}

				t, err := parseTime(v2[0])
				if err != nil {
					i.env.Log.Errorf("Time decode error: %v, skipping", err)
					continue
				}
				r := message.NewEventWithTime(tag, t, v2[1].(map[string]interface{}))
				i.env.Emit(r)
			}
		case 3:
			t, err := parseTime(v[1])
			if err != nil {
				i.env.Log.Errorf("Time decode error: %v, skipping", err)
				continue
			}
			r := message.NewEventWithTime(tag, t, v[2].(map[string]interface{}))
			i.env.Emit(r)
		}
	}
}

func (i *ForwardInput) heartbeatHandler() {
	buf := make([]byte, 1024)
	res := []byte{0}
	for {
		_, remote, err := i.udpConn.ReadFromUDP(buf)
		if i.closed {
			return
		}
		if err != nil {
			i.env.Log.Warning(err)
			continue
		}
		i.udpConn.WriteToUDP(res, remote)
	}
}

func parseTime(v interface{}) (t time.Time, err error) {
	var n int64
	switch typed := v.(type) {
	case []byte:
		err = t.UnmarshalBinary(typed)
		return
	case string:
		err = t.UnmarshalBinary([]byte(typed))
		return
	case int64:
		n = typed
	case uint64:
		n = int64(typed)
	case float64:
		nsec := int64((typed - float64(int64(typed))) * 1e9)
		t = time.Unix(int64(typed), nsec)
		return
	default:
		if n, err = strconv.ParseInt(fmt.Sprintf("%v", typed), 10, 64); err != nil {
			return
		}
	}
	t = time.Unix(n, 0)
	return
}

func main() {
	plugin.New("in-forward", func() plugin.Plugin {
		return &ForwardInput{}
	}).Run()
}

func init() {
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/influxdb/telegraf/plugins"
)

var Debug bool

type Metric struct {
	Region     string
	MetricName string
	Namespace  string
	Statistics []string
	Period     int64
	Duration   int64
	Unit       string
	Dimensions map[string]string
}

type CloudWatch struct {
	Debug   bool
	Metrics []Metric
}

func (cw *CloudWatch) Description() string {
	return "Pull metrics from AWS CloudWatch."
}

func (cw *CloudWatch) SampleConfig() string {
	return "ok = true # indicate if everything is fine"
}

func (cw *CloudWatch) Gather(acc plugins.Accumulator) error {

	Debug = cw.Debug

	for _, m := range cw.Metrics {
		m.PushMetrics(acc)
	}

	return nil
}

func printDebug(m ...interface{}) {
	if Debug {
		fmt.Println(m...)
	}
}

func convertDimensions(dims map[string]string) []*cloudwatch.Dimension {
	awsDims := make([]*cloudwatch.Dimension, len(dims))
	var i int
	for key, value := range dims {
		awsDims[i] = &cloudwatch.Dimension{
			Name:  aws.String(key),
			Value: aws.String(value),
		}
		i++
	}
	return awsDims
}

func (m *Metric) PushMetrics(acc plugins.Accumulator) error {

	sess := session.New(&aws.Config{Region: aws.String(m.Region)})
	svc := cloudwatch.New(sess)

	params := &cloudwatch.GetMetricStatisticsInput{
		EndTime:    aws.Time(time.Now()),
		MetricName: aws.String(m.MetricName),
		Namespace:  aws.String(m.Namespace),
		Period:     aws.Int64(m.Period),
		StartTime:  aws.Time(time.Now().Add(-time.Duration(m.Duration) * time.Second)),
		Statistics: aws.StringSlice(m.Statistics),
		Dimensions: convertDimensions(m.Dimensions),
		Unit:       aws.String(m.Unit),
	}

	printDebug(params)

	resp, err := svc.GetMetricStatistics(params)

	if err != nil {
		fmt.Println(err.Error())
		return err
	}

	for _, d := range resp.Datapoints {
		acc.Add(*resp.Label, *d.Average, m.Dimensions, *d.Timestamp)
	}

	printDebug(resp)

	return nil
}

func init() {
	plugins.Add("cloudwatch", func() plugins.Plugin { return &CloudWatch{} })
}
<commit_msg>fix bug where Dimensions was being modified in accumulator.Add function<commit_after>package aws

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/influxdb/telegraf/plugins"
)

var Debug bool

type Metric struct {
	Region     string
	MetricName string
	Namespace  string
	Statistics []string
	Period     int64
	Duration   int64
	Unit       string
	Dimensions map[string]string
}

type CloudWatch struct {
	Debug   bool
	Metrics []Metric
}

func (cw *CloudWatch) Description() string {
	return "Pull metrics from AWS CloudWatch."
}

func (cw *CloudWatch) SampleConfig() string {
	return "ok = true # indicate if everything is fine"
}

func (cw *CloudWatch) Gather(acc plugins.Accumulator) error {

	Debug = cw.Debug

	for _, m := range cw.Metrics {
		m.PushMetrics(acc)
	}

	return nil
}

func printDebug(m ...interface{}) {
	if Debug {
		fmt.Println(m...)
	}
}

func convertDimensions(dims map[string]string) []*cloudwatch.Dimension {
	awsDims := make([]*cloudwatch.Dimension, len(dims))
	var i int
	for key, value := range dims {
		awsDims[i] = &cloudwatch.Dimension{
			Name:  aws.String(key),
			Value: aws.String(value),
		}
		i++
	}
	return awsDims
}

func copyDims(dims map[string]string) map[string]string {
	dimsCopy := make(map[string]string)
	for k, v := range dims {
		dimsCopy[k] = v
	}
	return dimsCopy
}

func (m *Metric) PushMetrics(acc plugins.Accumulator) error {

	sess := session.New(&aws.Config{Region: aws.String(m.Region)})
	svc := cloudwatch.New(sess)

	params := &cloudwatch.GetMetricStatisticsInput{
		EndTime:    aws.Time(time.Now()),
		MetricName: aws.String(m.MetricName),
		Namespace:  aws.String(m.Namespace),
		Period:     aws.Int64(m.Period),
		StartTime:  aws.Time(time.Now().Add(-time.Duration(m.Duration) * time.Second)),
		Statistics: aws.StringSlice(m.Statistics),
		Dimensions: convertDimensions(m.Dimensions),
		Unit:       aws.String(m.Unit),
	}

	printDebug(params)

	resp, err := svc.GetMetricStatistics(params)

	if err != nil {
		fmt.Println(err.Error())
		return err
	}

	printDebug(resp)

	for _, d := range resp.Datapoints {
		acc.Add(*resp.Label, *d.Average, copyDims(m.Dimensions), *d.Timestamp)
	}
	return nil
}

func init() {
	plugins.Add("cloudwatch", func() plugins.Plugin { return &CloudWatch{} })
}
<|endoftext|>


<commit_before>package ecs

import (
	"os"
	"strings"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/config"
	"github.com/influxdata/telegraf/filter"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// Ecs config object
type Ecs struct {
	EndpointURL string `toml:"endpoint_url"`
	Timeout     config.Duration

	ContainerNameInclude []string `toml:"container_name_include"`
	ContainerNameExclude []string `toml:"container_name_exclude"`

	ContainerStatusInclude []string `toml:"container_status_include"`
	ContainerStatusExclude []string `toml:"container_status_exclude"`

	LabelInclude []string `toml:"ecs_label_include"`
	LabelExclude []string `toml:"ecs_label_exclude"`

	newClient func(timeout time.Duration, endpoint string, version int) (*EcsClient, error)

	client              Client
	filtersCreated      bool
	labelFilter         filter.Filter
	containerNameFilter filter.Filter
	statusFilter        filter.Filter
	metadataVersion     int
}

const (
	KB = 1000
	MB = 1000 * KB
	GB = 1000 * MB
	TB = 1000 * GB
	PB = 1000 * TB

	v2Endpoint = "http://169.254.170.2"
)

var sampleConfig = `
  ## ECS metadata url.
  ## Metadata v2 API is used if set explicitly. Otherwise,
  ## v3 metadata endpoint API is used if available.
  # endpoint_url = ""

  ## Containers to include and exclude. Globs accepted.
  ## Note that an empty array for both will include all containers
  # container_name_include = []
  # container_name_exclude = []

  ## Container states to include and exclude. Globs accepted.
  ## When empty only containers in the "RUNNING" state will be captured.
  ## Possible values are "NONE", "PULLED", "CREATED", "RUNNING",
  ## "RESOURCES_PROVISIONED", "STOPPED".
  # container_status_include = []
  # container_status_exclude = []

  ## ecs labels to include and exclude as tags.  Globs accepted.
  ## Note that an empty array for both will include all labels as tags
  ecs_label_include = [ "com.amazonaws.ecs.*" ]
  ecs_label_exclude = []

  ## Timeout for queries.
  # timeout = "5s"
`

// Description describes ECS plugin
func (ecs *Ecs) Description() string {
	return "Read metrics about docker containers from Fargate/ECS v2, v3 meta endpoints."
}

// SampleConfig returns the ECS example config
func (ecs *Ecs) SampleConfig() string {
	return sampleConfig
}

// Gather is the entrypoint for telegraf metrics collection
func (ecs *Ecs) Gather(acc telegraf.Accumulator) error {
	err := initSetup(ecs)
	if err != nil {
		return err
	}

	task, stats, err := PollSync(ecs.client)
	if err != nil {
		return err
	}

	mergeTaskStats(task, stats)

	taskTags := map[string]string{
		"cluster":  task.Cluster,
		"task_arn": task.TaskARN,
		"family":   task.Family,
		"revision": task.Revision,
	}

	// accumulate metrics
	ecs.accTask(task, taskTags, acc)
	ecs.accContainers(task, taskTags, acc)

	return nil
}

func initSetup(ecs *Ecs) error {
	if ecs.client == nil {
		resolveEndpoint(ecs)

		c, err := ecs.newClient(time.Duration(ecs.Timeout), ecs.EndpointURL, ecs.metadataVersion)
		if err != nil {
			return err
		}
		ecs.client = c
	}

	// Create filters
	if !ecs.filtersCreated {
		err := ecs.createContainerNameFilters()
		if err != nil {
			return err
		}
		err = ecs.createContainerStatusFilters()
		if err != nil {
			return err
		}
		err = ecs.createLabelFilters()
		if err != nil {
			return err
		}
		ecs.filtersCreated = true
	}

	return nil
}

func resolveEndpoint(ecs *Ecs) {
	if ecs.EndpointURL != "" {
		// Use metadata v2 API since endpoint is set explicitly.
		ecs.metadataVersion = 2
		return
	}

	// Auto-detect metadata endpoint version.

	// Use metadata v3 if available.
	// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v3.html
	v3Endpoint := os.Getenv("ECS_CONTAINER_METADATA_URI")
	if v3Endpoint != "" {
		ecs.EndpointURL = v3Endpoint
		ecs.metadataVersion = 3
		return
	}

	// Use v2 endpoint if nothing else is available.
	ecs.EndpointURL = v2Endpoint
	ecs.metadataVersion = 2
}

func (ecs *Ecs) accTask(task *Task, tags map[string]string, acc telegraf.Accumulator) {
	taskFields := map[string]interface{}{
		"desired_status": task.DesiredStatus,
		"known_status":   task.KnownStatus,
		"limit_cpu":      task.Limits["CPU"],
		"limit_mem":      task.Limits["Memory"],
	}

	acc.AddFields("ecs_task", taskFields, tags, task.PullStoppedAt)
}

func (ecs *Ecs) accContainers(task *Task, taskTags map[string]string, acc telegraf.Accumulator) {
	for _, c := range task.Containers {
		if !ecs.containerNameFilter.Match(c.Name) {
			continue
		}

		if !ecs.statusFilter.Match(strings.ToUpper(c.KnownStatus)) {
			continue
		}

		// add matching ECS container Labels
		containerTags := map[string]string{
			"id":   c.ID,
			"name": c.Name,
		}
		for k, v := range c.Labels {
			if ecs.labelFilter.Match(k) {
				containerTags[k] = v
			}
		}
		tags := mergeTags(taskTags, containerTags)

		parseContainerStats(c, acc, tags)
	}
}

// returns a new map with the same content values as the input map
func copyTags(in map[string]string) map[string]string {
	out := make(map[string]string)
	for k, v := range in {
		out[k] = v
	}
	return out
}

// returns a new map with the merged content values of the two input maps
func mergeTags(a map[string]string, b map[string]string) map[string]string {
	c := copyTags(a)
	for k, v := range b {
		c[k] = v
	}
	return c
}

func (ecs *Ecs) createContainerNameFilters() error {
	containerNameFilter, err := filter.NewIncludeExcludeFilter(ecs.ContainerNameInclude, ecs.ContainerNameExclude)
	if err != nil {
		return err
	}
	ecs.containerNameFilter = containerNameFilter
	return nil
}

func (ecs *Ecs) createLabelFilters() error {
	labelFilter, err := filter.NewIncludeExcludeFilter(ecs.LabelInclude, ecs.LabelExclude)
	if err != nil {
		return err
	}
	ecs.labelFilter = labelFilter
	return nil
}

func (ecs *Ecs) createContainerStatusFilters() error {
	if len(ecs.ContainerStatusInclude) == 0 && len(ecs.ContainerStatusExclude) == 0 {
		ecs.ContainerStatusInclude = []string{"RUNNING"}
	}

	// ECS uses uppercase status names, normalizing for comparison.
	for i, include := range ecs.ContainerStatusInclude {
		ecs.ContainerStatusInclude[i] = strings.ToUpper(include)
	}
	for i, exclude := range ecs.ContainerStatusExclude {
		ecs.ContainerStatusExclude[i] = strings.ToUpper(exclude)
	}

	statusFilter, err := filter.NewIncludeExcludeFilter(ecs.ContainerStatusInclude, ecs.ContainerStatusExclude)
	if err != nil {
		return err
	}
	ecs.statusFilter = statusFilter
	return nil
}

func init() {
	inputs.Add("ecs", func() telegraf.Input {
		return &Ecs{
			EndpointURL:    "",
			Timeout:        config.Duration(5 * time.Second),
			newClient:      NewClient,
			filtersCreated: false,
		}
	})
}
<commit_msg>fix: use current time as ecs timestamp (#10636)<commit_after>package ecs

import (
	"os"
	"strings"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/config"
	"github.com/influxdata/telegraf/filter"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// Ecs config object
type Ecs struct {
	EndpointURL string `toml:"endpoint_url"`
	Timeout     config.Duration

	ContainerNameInclude []string `toml:"container_name_include"`
	ContainerNameExclude []string `toml:"container_name_exclude"`

	ContainerStatusInclude []string `toml:"container_status_include"`
	ContainerStatusExclude []string `toml:"container_status_exclude"`

	LabelInclude []string `toml:"ecs_label_include"`
	LabelExclude []string `toml:"ecs_label_exclude"`

	newClient func(timeout time.Duration, endpoint string, version int) (*EcsClient, error)

	client              Client
	filtersCreated      bool
	labelFilter         filter.Filter
	containerNameFilter filter.Filter
	statusFilter        filter.Filter
	metadataVersion     int
}

const (
	KB = 1000
	MB = 1000 * KB
	GB = 1000 * MB
	TB = 1000 * GB
	PB = 1000 * TB

	v2Endpoint = "http://169.254.170.2"
)

var sampleConfig = `
  ## ECS metadata url.
  ## Metadata v2 API is used if set explicitly. Otherwise,
  ## v3 metadata endpoint API is used if available.
  # endpoint_url = ""

  ## Containers to include and exclude. Globs accepted.
  ## Note that an empty array for both will include all containers
  # container_name_include = []
  # container_name_exclude = []

  ## Container states to include and exclude. Globs accepted.
  ## When empty only containers in the "RUNNING" state will be captured.
  ## Possible values are "NONE", "PULLED", "CREATED", "RUNNING",
  ## "RESOURCES_PROVISIONED", "STOPPED".
  # container_status_include = []
  # container_status_exclude = []

  ## ecs labels to include and exclude as tags.  Globs accepted.
  ## Note that an empty array for both will include all labels as tags
  ecs_label_include = [ "com.amazonaws.ecs.*" ]
  ecs_label_exclude = []

  ## Timeout for queries.
  # timeout = "5s"
`

// Description describes ECS plugin
func (ecs *Ecs) Description() string {
	return "Read metrics about docker containers from Fargate/ECS v2, v3 meta endpoints."
}

// SampleConfig returns the ECS example config
func (ecs *Ecs) SampleConfig() string {
	return sampleConfig
}

// Gather is the entrypoint for telegraf metrics collection
func (ecs *Ecs) Gather(acc telegraf.Accumulator) error {
	err := initSetup(ecs)
	if err != nil {
		return err
	}

	task, stats, err := PollSync(ecs.client)
	if err != nil {
		return err
	}

	mergeTaskStats(task, stats)

	taskTags := map[string]string{
		"cluster":  task.Cluster,
		"task_arn": task.TaskARN,
		"family":   task.Family,
		"revision": task.Revision,
	}

	// accumulate metrics
	ecs.accTask(task, taskTags, acc)
	ecs.accContainers(task, taskTags, acc)

	return nil
}

func initSetup(ecs *Ecs) error {
	if ecs.client == nil {
		resolveEndpoint(ecs)

		c, err := ecs.newClient(time.Duration(ecs.Timeout), ecs.EndpointURL, ecs.metadataVersion)
		if err != nil {
			return err
		}
		ecs.client = c
	}

	// Create filters
	if !ecs.filtersCreated {
		err := ecs.createContainerNameFilters()
		if err != nil {
			return err
		}
		err = ecs.createContainerStatusFilters()
		if err != nil {
			return err
		}
		err = ecs.createLabelFilters()
		if err != nil {
			return err
		}
		ecs.filtersCreated = true
	}

	return nil
}

func resolveEndpoint(ecs *Ecs) {
	if ecs.EndpointURL != "" {
		// Use metadata v2 API since endpoint is set explicitly.
		ecs.metadataVersion = 2
		return
	}

	// Auto-detect metadata endpoint version.

	// Use metadata v3 if available.
	// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v3.html
	v3Endpoint := os.Getenv("ECS_CONTAINER_METADATA_URI")
	if v3Endpoint != "" {
		ecs.EndpointURL = v3Endpoint
		ecs.metadataVersion = 3
		return
	}

	// Use v2 endpoint if nothing else is available.
	ecs.EndpointURL = v2Endpoint
	ecs.metadataVersion = 2
}

func (ecs *Ecs) accTask(task *Task, tags map[string]string, acc telegraf.Accumulator) {
	taskFields := map[string]interface{}{
		"desired_status": task.DesiredStatus,
		"known_status":   task.KnownStatus,
		"limit_cpu":      task.Limits["CPU"],
		"limit_mem":      task.Limits["Memory"],
	}

	acc.AddFields("ecs_task", taskFields, tags)
}

func (ecs *Ecs) accContainers(task *Task, taskTags map[string]string, acc telegraf.Accumulator) {
	for _, c := range task.Containers {
		if !ecs.containerNameFilter.Match(c.Name) {
			continue
		}

		if !ecs.statusFilter.Match(strings.ToUpper(c.KnownStatus)) {
			continue
		}

		// add matching ECS container Labels
		containerTags := map[string]string{
			"id":   c.ID,
			"name": c.Name,
		}
		for k, v := range c.Labels {
			if ecs.labelFilter.Match(k) {
				containerTags[k] = v
			}
		}
		tags := mergeTags(taskTags, containerTags)

		parseContainerStats(c, acc, tags)
	}
}

// returns a new map with the same content values as the input map
func copyTags(in map[string]string) map[string]string {
	out := make(map[string]string)
	for k, v := range in {
		out[k] = v
	}
	return out
}

// returns a new map with the merged content values of the two input maps
func mergeTags(a map[string]string, b map[string]string) map[string]string {
	c := copyTags(a)
	for k, v := range b {
		c[k] = v
	}
	return c
}

func (ecs *Ecs) createContainerNameFilters() error {
	containerNameFilter, err := filter.NewIncludeExcludeFilter(ecs.ContainerNameInclude, ecs.ContainerNameExclude)
	if err != nil {
		return err
	}
	ecs.containerNameFilter = containerNameFilter
	return nil
}

func (ecs *Ecs) createLabelFilters() error {
	labelFilter, err := filter.NewIncludeExcludeFilter(ecs.LabelInclude, ecs.LabelExclude)
	if err != nil {
		return err
	}
	ecs.labelFilter = labelFilter
	return nil
}

func (ecs *Ecs) createContainerStatusFilters() error {
	if len(ecs.ContainerStatusInclude) == 0 && len(ecs.ContainerStatusExclude) == 0 {
		ecs.ContainerStatusInclude = []string{"RUNNING"}
	}

	// ECS uses uppercase status names, normalizing for comparison.
	for i, include := range ecs.ContainerStatusInclude {
		ecs.ContainerStatusInclude[i] = strings.ToUpper(include)
	}
	for i, exclude := range ecs.ContainerStatusExclude {
		ecs.ContainerStatusExclude[i] = strings.ToUpper(exclude)
	}

	statusFilter, err := filter.NewIncludeExcludeFilter(ecs.ContainerStatusInclude, ecs.ContainerStatusExclude)
	if err != nil {
		return err
	}
	ecs.statusFilter = statusFilter
	return nil
}

func init() {
	inputs.Add("ecs", func() telegraf.Input {
		return &Ecs{
			EndpointURL:    "",
			Timeout:        config.Duration(5 * time.Second),
			newClient:      NewClient,
			filtersCreated: false,
		}
	})
}
<|endoftext|>


<commit_before>package lgtm_test

import (
	"testing"

	"github.com/kyokomi/slackbot/plugins"
	"github.com/kyokomi/slackbot/plugins/lgtm"
	"fmt"
)

var testEvent = plugins.NewTestEvent("LGTM")

func TestCheckMessage(t *testing.T) {
	p := lgtm.Plugin{}
	ok, _ := p.CheckMessage(*testEvent, testEvent.BaseText())
	if !ok {
		t.Errorf("ERROR check = NG")
	}
}

func TestDoAction(t *testing.T) {
	p := lgtm.Plugin{}

	next := p.DoAction(*testEvent, testEvent.BaseText())

	if next != false {
		t.Errorf("ERROR next != false")
	}
}

func TestGetLGTMImageURL(t *testing.T) {
	if message, ok := lgtm.GetLGTMImageURL("hoge"); !ok {
		t.Errorf("get lgtm image don't error", message)
	} else {
		fmt.Println(message)
	}

	if message, ok := lgtm.GetLGTMImageURL("https://raw.githubusercontent.com/kyokomi/slackbot/master/README.md"); !ok {
		t.Errorf("get lgtm image don't error", message)
	} else {
		fmt.Println(message)
	}
}
<commit_msg>fix govet<commit_after>package lgtm_test

import (
	"testing"

	"github.com/kyokomi/slackbot/plugins"
	"github.com/kyokomi/slackbot/plugins/lgtm"
	"fmt"
)

var testEvent = plugins.NewTestEvent("LGTM")

func TestCheckMessage(t *testing.T) {
	p := lgtm.Plugin{}
	ok, _ := p.CheckMessage(*testEvent, testEvent.BaseText())
	if !ok {
		t.Errorf("ERROR check = NG")
	}
}

func TestDoAction(t *testing.T) {
	p := lgtm.Plugin{}

	next := p.DoAction(*testEvent, testEvent.BaseText())

	if next != false {
		t.Errorf("ERROR next != false")
	}
}

func TestGetLGTMImageURL(t *testing.T) {
	if message, ok := lgtm.GetLGTMImageURL("hoge"); !ok {
		t.Errorf("get lgtm image don't error %s", message)
	} else {
		fmt.Println(message)
	}

	if message, ok := lgtm.GetLGTMImageURL("https://raw.githubusercontent.com/kyokomi/slackbot/master/README.md"); !ok {
		t.Errorf("get lgtm image don't error %s", message)
	} else {
		fmt.Println(message)
	}
}
<|endoftext|>


<commit_before>// prefmgr exposes hal's preferences as a bot command and over REST
package prefmgr

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/netflix/hal-9001/hal"
)

const NAME = "prefmgr"

const HELP = `Listing keys with no filter will list all keys visible to the active user and room.

!prefs list --key KEY
!prefs list --user USER --room CHANNEL --plugin PLUGIN --key KEY --def DEFAULT
`

var cli *hal.Cmd

var slackLinkRE *regexp.Regexp

func init() {
	cli = hal.NewCmd("!pref", true).SetUsage("Manage hal preferences over chat.")

	keyUsage := "the key name, up to 190 utf8 characters"
	valueUsage := "the value, arbitrary utf8"
	roomUsage := "the chat room id (usually auto-resolved, '*' for 'this room')"
	userUsage := "the user id (usually auto-resolved, '*' for 'executing user')"
	brokerUsage := "the broker name. e.g. 'slack' ('*' for 'this broker')"
	pluginUsage := "the plugin name. e.g. 'archive' ('*' for 'this plugin')"

	cli.AddSubCmd("set").
		SetUsage("set a preference key/value").
		SubCmd().AddKVParam("key", true).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", true).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("list").
		AddAlias("get").
		SetUsage("retreive preferences, optionally filtered by the provided attributes").
		SubCmd().
		AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("find").
		SetUsage("retreive preferences following precedence rules").
		SubCmd().AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("rm").
		SetUsage("delete a preference by id").
		AddIdxParam(0, true).
		SetUsage("the preference id to delete")

	slackLinkRE = regexp.MustCompile("^<(?:http|mailto):.*|.*>$")
}

func Register() {
	plugin := hal.Plugin{
		Name:  NAME,
		Func:  prefmgr,
		Regex: "^!prefs",
	}
	plugin.Register()

	http.HandleFunc("/v1/prefs", httpPrefs)
}

// prefmgr is called when someone executes !pref in the chat system
func prefmgr(evt hal.Evt) {
	req, err := cli.Process(evt.BodyAsArgv())
	if err != nil {
		// eww...
		switch err.(type) {
		case hal.SubCmdNotFound:
			evt.Reply(cli.Usage())
		default:
			evt.Reply(err.Error())
		}
		return
	}

	switch req.SubCmdToken() {
	case "", "help":
		evt.Reply(cli.Usage())
	case "set":
		cliSet(req.SubCmdInst(), &evt)
	case "list":
		cliList(req.SubCmdInst(), &evt)
	case "find":
		cliFind(req.SubCmdInst(), &evt)
	case "rm":
		cliRm(req.SubCmdInst(), &evt)
	default:
		evt.Reply(req.Usage())
	}
}

// cmd2pref copies data from the hal.Cmd and hal.Evt into a hal.Pref, resolving
// *'s on the way.
func cmd2pref(req *hal.SubCmdInst, evt *hal.Evt) (*hal.Pref, error) {
	var out hal.Pref

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			out.Key = stripAutoLinks(key)
		case "value":
			value, err = pi.String()
			out.Value = stripAutoLinks(value)
		case "room":
			out.Room = pi.DefString(evt.RoomId)
		case "user":
			out.User = pi.DefString(evt.UserId)
		case "broker":
			out.Broker = pi.DefString(evt.BrokerName())
		case "plugin":
			out.Plugin, _ = pi.String()
		}

		// return on the first error
		if err != nil {
			return nil, err
		}
	}

	return &out, nil
}

// cliList implements !pref list
func cliList(req *hal.SubCmdInst, evt *hal.Evt) {
	opts := hal.Pref{}
	prefs := opts.Find()

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			prefs = prefs.Key(stripAutoLinks(key))
		case "value":
			value, err = pi.String()
			prefs = prefs.Value(stripAutoLinks(value))
		case "room":
			prefs = prefs.Room(pi.DefString(evt.RoomId))
		case "user":
			prefs = prefs.User(pi.DefString(evt.UserId))
		case "broker":
			prefs = prefs.Broker(pi.DefString(evt.BrokerName()))
		case "plugin":
			prefs = prefs.Plugin(pi.DefString(NAME))
		}

		if err != nil {
			evt.Error(err)
			return
		}
	}

	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliFind implements !pref find
func cliFind(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	prefs := opts.Find()
	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliSet implements !pref set
func cliSet(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	if opts.Room != "" && !evt.Broker.LooksLikeRoomId(opts.Room) {
		opts.Room = evt.Broker.RoomNameToId(opts.Room)
	}

	if opts.User != "" && !evt.Broker.LooksLikeUserId(opts.User) {
		opts.User = evt.Broker.UserNameToId(opts.User)
	}

	// TODO: check plugin name validity
	// TODO: check broker name validity

	fmt.Printf("Setting pref: %q\n", opts.String())
	err = opts.Set()
	if err != nil {
		evt.Replyf("Failed to set pref: %q", err)
	} else {
		data := opts.GetPrefs().Table()
		evt.ReplyTable(data[0], data[1:])
	}
}

// cliRm implements !pref rm <id>
func cliRm(req *hal.SubCmdInst, evt *hal.Evt) {
	id, err := req.GetIdxParamInst(0).Int()
	if err != nil {
		panic(err) // TODO: placeholder
	}

	err = hal.RmPrefId(id)
	if err != nil {
		evt.Replyf("Failed to delete pref with id %d: %s", id, err)
	} else {
		evt.Replyf("Deleted pref id %d.", id)
	}
}

func stripAutoLinks(in string) string {
	if slackLinkRE.MatchString(in) {
		parts := strings.Split(strings.TrimSuffix(in, ">"), "|")
		if len(parts) == 2 {
			return parts[1]
		}
	}

	return in
}

// httpPrefs is the http handler for returning preferences as JSON
func httpPrefs(w http.ResponseWriter, r *http.Request) {
}
<commit_msg>make the plugin RE match the command name<commit_after>// prefmgr exposes hal's preferences as a bot command and over REST
package prefmgr

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/netflix/hal-9001/hal"
)

const NAME = "prefmgr"

const HELP = `Listing keys with no filter will list all keys visible to the active user and room.

!prefs list --key KEY
!prefs list --user USER --room CHANNEL --plugin PLUGIN --key KEY --def DEFAULT
`

var cli *hal.Cmd

var slackLinkRE *regexp.Regexp

func init() {
	cli = hal.NewCmd("!pref", true).SetUsage("Manage hal preferences over chat.")

	keyUsage := "the key name, up to 190 utf8 characters"
	valueUsage := "the value, arbitrary utf8"
	roomUsage := "the chat room id (usually auto-resolved, '*' for 'this room')"
	userUsage := "the user id (usually auto-resolved, '*' for 'executing user')"
	brokerUsage := "the broker name. e.g. 'slack' ('*' for 'this broker')"
	pluginUsage := "the plugin name. e.g. 'archive' ('*' for 'this plugin')"

	cli.AddSubCmd("set").
		SetUsage("set a preference key/value").
		SubCmd().AddKVParam("key", true).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", true).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("list").
		AddAlias("get").
		SetUsage("retreive preferences, optionally filtered by the provided attributes").
		SubCmd().
		AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("find").
		SetUsage("retreive preferences following precedence rules").
		SubCmd().AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("rm").
		SetUsage("delete a preference by id").
		AddIdxParam(0, true).
		SetUsage("the preference id to delete")

	slackLinkRE = regexp.MustCompile("^<(?:http|mailto):.*|.*>$")
}

func Register() {
	plugin := hal.Plugin{
		Name:  NAME,
		Func:  prefmgr,
		Regex: "^!pref",
	}
	plugin.Register()
}

// prefmgr is called when someone executes !pref in the chat system
func prefmgr(evt hal.Evt) {
	req, err := cli.Process(evt.BodyAsArgv())
	if err != nil {
		// eww...
		switch err.(type) {
		case hal.SubCmdNotFound:
			evt.Reply(cli.Usage())
		default:
			evt.Reply(err.Error())
		}
		return
	}

	switch req.SubCmdToken() {
	case "", "help":
		evt.Reply(cli.Usage())
	case "set":
		cliSet(req.SubCmdInst(), &evt)
	case "list":
		cliList(req.SubCmdInst(), &evt)
	case "find":
		cliFind(req.SubCmdInst(), &evt)
	case "rm":
		cliRm(req.SubCmdInst(), &evt)
	default:
		evt.Reply(req.Usage())
	}
}

// cmd2pref copies data from the hal.Cmd and hal.Evt into a hal.Pref, resolving
// *'s on the way.
func cmd2pref(req *hal.SubCmdInst, evt *hal.Evt) (*hal.Pref, error) {
	var out hal.Pref

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			out.Key = stripAutoLinks(key)
		case "value":
			value, err = pi.String()
			out.Value = stripAutoLinks(value)
		case "room":
			out.Room = pi.DefString(evt.RoomId)
		case "user":
			out.User = pi.DefString(evt.UserId)
		case "broker":
			out.Broker = pi.DefString(evt.BrokerName())
		case "plugin":
			out.Plugin, _ = pi.String()
		}

		// return on the first error
		if err != nil {
			return nil, err
		}
	}

	return &out, nil
}

// cliList implements !pref list
func cliList(req *hal.SubCmdInst, evt *hal.Evt) {
	opts := hal.Pref{}
	prefs := opts.Find()

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			prefs = prefs.Key(stripAutoLinks(key))
		case "value":
			value, err = pi.String()
			prefs = prefs.Value(stripAutoLinks(value))
		case "room":
			prefs = prefs.Room(pi.DefString(evt.RoomId))
		case "user":
			prefs = prefs.User(pi.DefString(evt.UserId))
		case "broker":
			prefs = prefs.Broker(pi.DefString(evt.BrokerName()))
		case "plugin":
			prefs = prefs.Plugin(pi.DefString(NAME))
		}

		if err != nil {
			evt.Error(err)
			return
		}
	}

	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliFind implements !pref find
func cliFind(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	prefs := opts.Find()
	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliSet implements !pref set
func cliSet(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	if opts.Room != "" && !evt.Broker.LooksLikeRoomId(opts.Room) {
		opts.Room = evt.Broker.RoomNameToId(opts.Room)
	}

	if opts.User != "" && !evt.Broker.LooksLikeUserId(opts.User) {
		opts.User = evt.Broker.UserNameToId(opts.User)
	}

	// TODO: check plugin name validity
	// TODO: check broker name validity

	fmt.Printf("Setting pref: %q\n", opts.String())
	err = opts.Set()
	if err != nil {
		evt.Replyf("Failed to set pref: %q", err)
	} else {
		data := opts.GetPrefs().Table()
		evt.ReplyTable(data[0], data[1:])
	}
}

// cliRm implements !pref rm <id>
func cliRm(req *hal.SubCmdInst, evt *hal.Evt) {
	id, err := req.GetIdxParamInst(0).Int()
	if err != nil {
		panic(err) // TODO: placeholder
	}

	err = hal.RmPrefId(id)
	if err != nil {
		evt.Replyf("Failed to delete pref with id %d: %s", id, err)
	} else {
		evt.Replyf("Deleted pref id %d.", id)
	}
}

func stripAutoLinks(in string) string {
	if slackLinkRE.MatchString(in) {
		parts := strings.Split(strings.TrimSuffix(in, ">"), "|")
		if len(parts) == 2 {
			return parts[1]
		}
	}

	return in
}
<|endoftext|>


<commit_before>package postgres

import (
	"net/http"
	"testing"

	"github.com/nuveo/prest/api"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "dbname=$")
		So(where, ShouldContainSubstring, "test=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "prest")
		So(values, ShouldContain, "cool")
	})

	Convey("Where by request with jsonb field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})
}

func TestConnection(t *testing.T) {
	Convey("Verify database connection", t, func() {
		sqlx := Conn()
		So(sqlx, ShouldNotBeNil)
		err := sqlx.Ping()
		So(err, ShouldBeNil)
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where, err := PaginateIfPossible(r)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Insert("prest", "public", "test", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name=$1", []interface{}{"nuveo"})
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Update("prest", "public", "test", "name=$1", []interface{}{"prest"}, r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}
<commit_msg>test chkInvaidIdentifier<commit_after>package postgres

import (
	"net/http"
	"testing"

	"github.com/nuveo/prest/api"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "dbname=$")
		So(where, ShouldContainSubstring, "test=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "prest")
		So(values, ShouldContain, "cool")
	})

	Convey("Where by request with jsonb field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})
}

func TestConnection(t *testing.T) {
	Convey("Verify database connection", t, func() {
		sqlx := Conn()
		So(sqlx, ShouldNotBeNil)
		err := sqlx.Ping()
		So(err, ShouldBeNil)
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where, err := PaginateIfPossible(r)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Insert("prest", "public", "test", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name=$1", []interface{}{"nuveo"})
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Update("prest", "public", "test", "name=$1", []interface{}{"prest"}, r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestChkInvaidIdentifier(t *testing.T) {
	Convey("Check invalid character on identifier", t, func() {
		chk := chkInvaidIdentifier("fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvaidIdentifier("_9fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvaidIdentifier("_fild.Name")
		So(chk, ShouldBeFalse)

		chk = chkInvaidIdentifier("0fildName")
		So(chk, ShouldBeTrue)
		chk = chkInvaidIdentifier("fild'Name")
		So(chk, ShouldBeTrue)
		chk = chkInvaidIdentifier("fild\"Name")
		So(chk, ShouldBeTrue)
		chk = chkInvaidIdentifier("fild;Name")
		So(chk, ShouldBeTrue)
		chk = chkInvaidIdentifier("_123456789_123456789_123456789_123456789_123456789_123456789_12345")
		So(chk, ShouldBeTrue)

	})
}
<|endoftext|>