<commit_before>package kafka

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

type subManager struct {
	clientMap     map[string]*consumergroup.ConsumerGroup // key is client remote addr, a client can only sub 1 topic
	clientMapLock sync.RWMutex                            // TODO the lock is too big
}

func newSubManager() *subManager {
	return &subManager{
		clientMap: make(map[string]*consumergroup.ConsumerGroup, 500),
	}
}

func (this *subManager) PickConsumerGroup(cluster, topic, group, remoteAddr, realIp string,
	resetOffset string, permitStandby bool) (cg *consumergroup.ConsumerGroup, err error) {
	// find consumger group from cache
	var present bool
	this.clientMapLock.RLock()
	cg, present = this.clientMap[remoteAddr]
	this.clientMapLock.RUnlock()
	if present {
		return
	}

	if !permitStandby {
		// ensure concurrent sub threads didn't exceed partition count
		// the 1st barrier, consumer group is the final barrier
		onlineN := meta.Default.OnlineConsumersCount(cluster, topic, group)
		partitionN := len(meta.Default.TopicPartitions(cluster, topic))
		if partitionN > 0 && onlineN >= partitionN {
			err = store.ErrTooManyConsumers
			return
		}
	}

	this.clientMapLock.Lock()
	defer this.clientMapLock.Unlock()

	// double check lock
	cg, present = this.clientMap[remoteAddr]
	if present {
		return
	}

	// cache miss, create the consumer group for this client
	cf := consumergroup.NewConfig()
	cf.PermitStandby = permitStandby
	cf.OneToOne = true

	cf.Net.DialTimeout = time.Second * 10
	cf.Net.WriteTimeout = time.Second * 10
	cf.Net.ReadTimeout = time.Second * 10

	// kafka Fetch already batched into MessageSet，
	// this chan buf size influence on throughput is ignoreable
	cf.ChannelBufferSize = 0
	// kafka Fetch MaxWaitTime 250ms, MinByte=1 by default

	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = time.Second * 2 // chan recv timeout
	cf.Zookeeper.Chroot = meta.Default.ZkChroot(cluster)
	cf.Zookeeper.Timeout = zk.DefaultZkSessionTimeout()
	cf.Offsets.CommitInterval = time.Minute
	cf.Offsets.ProcessingTimeout = time.Second
	switch resetOffset {
	case "newest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetNewest
	case "oldest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetOldest
	default:
		cf.Offsets.ResetOffsets = false
		cf.Offsets.Initial = sarama.OffsetOldest
	}

	// runs in serial
	cg, err = consumergroup.JoinConsumerGroupRealIp(realIp, group, []string{topic},
		meta.Default.ZkAddrs(), cf)
	if err == nil {
		this.clientMap[remoteAddr] = cg
	}

	return
}

// For a given consumer client, it might be killed twice:
// 1. on socket level, the socket is closed
// 2. websocket/sub handler, conn closed or error occurs, explicitly kill the client
func (this *subManager) killClient(remoteAddr string) (err error) {
	this.clientMapLock.Lock()
	cg, present := this.clientMap[remoteAddr]
	if present {
		delete(this.clientMap, remoteAddr)
	}
	this.clientMapLock.Unlock()

	if !present {
		// e,g. client connects to Sub port but not Sub request(404), will lead to this case
		// e,g. Sub a non-exist topic
		return
	}

	if err = cg.Close(); err != nil {
		// will flush offset, must wait, otherwise offset is not guanranteed
		log.Error("cg[%s] close %s: %v", cg.Name(), remoteAddr, err)
	}

	return
}

func (this *subManager) Stop() {
	this.clientMapLock.Lock()
	defer this.clientMapLock.Unlock()

	var wg sync.WaitGroup
	for _, cg := range this.clientMap {
		wg.Add(1)
		go func(cg *consumergroup.ConsumerGroup) {
			cg.Close() // will commit inflight offsets
			wg.Done()
		}(cg)
	}

	wg.Wait()
	log.Trace("all consumer offsets committed")
}
<commit_msg>fix bad conditions<commit_after>package kafka

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

type subManager struct {
	clientMap     map[string]*consumergroup.ConsumerGroup // key is client remote addr, a client can only sub 1 topic
	clientMapLock sync.RWMutex                            // TODO the lock is too big
}

func newSubManager() *subManager {
	return &subManager{
		clientMap: make(map[string]*consumergroup.ConsumerGroup, 500),
	}
}

func (this *subManager) PickConsumerGroup(cluster, topic, group, remoteAddr, realIp string,
	resetOffset string, permitStandby bool) (cg *consumergroup.ConsumerGroup, err error) {
	// find consumger group from cache
	var present bool
	this.clientMapLock.RLock()
	cg, present = this.clientMap[remoteAddr]
	this.clientMapLock.RUnlock()
	if present {
		return
	}

	if !permitStandby {
		// ensure concurrent sub threads didn't exceed partition count
		// the 1st barrier, consumer group is the final barrier
		onlineN := meta.Default.OnlineConsumersCount(cluster, topic, group)
		partitionN := len(meta.Default.TopicPartitions(cluster, topic))
		if partitionN == 0 {
			err = store.ErrInvalidTopic
			return
		}
		if onlineN >= partitionN {
			err = store.ErrTooManyConsumers
			return
		}
	}

	this.clientMapLock.Lock()
	defer this.clientMapLock.Unlock()

	// double check lock
	cg, present = this.clientMap[remoteAddr]
	if present {
		return
	}

	// cache miss, create the consumer group for this client
	cf := consumergroup.NewConfig()
	cf.PermitStandby = permitStandby
	cf.OneToOne = true

	cf.Net.DialTimeout = time.Second * 10
	cf.Net.WriteTimeout = time.Second * 10
	cf.Net.ReadTimeout = time.Second * 10

	// kafka Fetch already batched into MessageSet，
	// this chan buf size influence on throughput is ignoreable
	cf.ChannelBufferSize = 0
	// kafka Fetch MaxWaitTime 250ms, MinByte=1 by default

	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = time.Second * 2 // chan recv timeout
	cf.Zookeeper.Chroot = meta.Default.ZkChroot(cluster)
	cf.Zookeeper.Timeout = zk.DefaultZkSessionTimeout()
	cf.Offsets.CommitInterval = time.Minute
	cf.Offsets.ProcessingTimeout = time.Second
	switch resetOffset {
	case "newest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetNewest
	case "oldest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetOldest
	default:
		cf.Offsets.ResetOffsets = false
		cf.Offsets.Initial = sarama.OffsetOldest
	}

	// runs in serial
	cg, err = consumergroup.JoinConsumerGroupRealIp(realIp, group, []string{topic},
		meta.Default.ZkAddrs(), cf)
	if err == nil {
		this.clientMap[remoteAddr] = cg
	}

	return
}

// For a given consumer client, it might be killed twice:
// 1. on socket level, the socket is closed
// 2. websocket/sub handler, conn closed or error occurs, explicitly kill the client
func (this *subManager) killClient(remoteAddr string) (err error) {
	this.clientMapLock.Lock()
	cg, present := this.clientMap[remoteAddr]
	if present {
		delete(this.clientMap, remoteAddr)
	}
	this.clientMapLock.Unlock()

	if !present {
		// e,g. client connects to Sub port but not Sub request(404), will lead to this case
		// e,g. Sub a non-exist topic
		return
	}

	if err = cg.Close(); err != nil {
		// will flush offset, must wait, otherwise offset is not guanranteed
		log.Error("cg[%s] close %s: %v", cg.Name(), remoteAddr, err)
	}

	return
}

func (this *subManager) Stop() {
	this.clientMapLock.Lock()
	defer this.clientMapLock.Unlock()

	var wg sync.WaitGroup
	for _, cg := range this.clientMap {
		wg.Add(1)
		go func(cg *consumergroup.ConsumerGroup) {
			cg.Close() // will commit inflight offsets
			wg.Done()
		}(cg)
	}

	wg.Wait()
	log.Trace("all consumer offsets committed")
}
<|endoftext|>


<commit_before>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"
	"net"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sFQDNTest", func() {
	var (
		kubectl          *helpers.Kubectl
		backgroundCancel context.CancelFunc = func() {}
		backgroundError  error

		demoManifest   = ""
		ciliumFilename string

		apps    = []string{helpers.App2, helpers.App3}
		appPods map[string]string

		// The IPs are updated in BeforeAll
		worldTarget          = "http://vagrant-cache.ci.cilium.io"
		worldTargetIP        = "147.75.38.95"
		worldInvalidTarget   = "http://jenkins.cilium.io"
		worldInvalidTargetIP = "104.198.14.52"
	)

	BeforeAll(func() {
		// In case the IPs changed from above, update them here
		var lookupErr error
		err := helpers.WithTimeout(func() bool {
			addrs, err2 := net.LookupHost("vagrant-cache.ci.cilium.io")
			if err2 != nil {
				lookupErr = fmt.Errorf("error looking up vagrant-cache.ci.cilium.io: %s", err2)
				return false
			}
			worldTargetIP = addrs[0]
			return true
		}, "Could not get vagrant-cache.ci.cilium.io IP", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		Expect(err).Should(BeNil(), "Error obtaining IP for test: %s", lookupErr)

		lookupErr = nil
		err = helpers.WithTimeout(func() bool {
			addrs, err2 := net.LookupHost("jenkins.cilium.io")
			if err2 != nil {
				lookupErr = fmt.Errorf("error looking up jenkins.cilium.io: %s", err2)
				return false
			}
			worldInvalidTargetIP = addrs[0]
			return true
		}, "Could not get jenkins.cilium.io IP", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		Expect(err).Should(BeNil(), "Error obtaining IP for test: %s", lookupErr)

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoManifest = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml")

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		By("Applying demo manifest")
		res := kubectl.ApplyDefault(demoManifest)
		res.ExpectSuccess("Demo config cannot be deployed")

		err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=testapp", helpers.HelperTimeout)
		Expect(err).Should(BeNil(), "Testapp is not ready after timeout")

		appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "id")
	})

	AfterFailed(func() {
		kubectl.CiliumReport("cilium service list", "cilium endpoint list")
	})

	AfterAll(func() {
		_ = kubectl.Delete(demoManifest)
		ExpectAllPodsTerminated(kubectl)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	JustBeforeEach(func() {
		backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
		Expect(backgroundError).To(BeNil(), "Cannot start background report process")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		backgroundCancel()
	})

	AfterEach(func() {
		_ = kubectl.Exec(fmt.Sprintf("%s delete --all cnp", helpers.KubectlCmd))
	})

	SkipItIf(helpers.SkipQuarantined, "Restart Cilium validate that FQDN is still working", func() {
		// Test functionality:
		// - When Cilium is running) Connectivity from App2 application can
		// connect to DNS because dns-proxy filter the DNS request. If the
		// connection is made correctly the IP is whitelisted by the FQDN rule
		// until the DNS TTL expires.
		// - When Cilium is not running) The DNS-proxy is not working, so the IP
		// connectivity to an existing IP that was queried before will work,
		// meanwhile connections using new DNS request will fail.
		// - On restart) Cilium will restore the IPS that were white-listted in
		// the FQDN and connection will work as normal.

		ciliumPodK8s1, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.K8s1)
		Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1")
		monitorRes1, monitorCancel1 := kubectl.MonitorStart(ciliumPodK8s1)
		ciliumPodK8s2, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.K8s2)
		Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s2")
		monitorRes2, monitorCancel2 := kubectl.MonitorStart(ciliumPodK8s2)
		defer func() {
			monitorCancel1()
			monitorCancel2()
			helpers.WriteToReportFile(monitorRes1.CombineOutput().Bytes(), "fqdn-restart-cilium-monitor-k8s1.log")
			helpers.WriteToReportFile(monitorRes2.CombineOutput().Bytes(), "fqdn-restart-cilium-monitor-k8s2.log")
		}()

		connectivityTest := func() {

			By("Testing that connection from %q to %q should work",
				appPods[helpers.App2], worldTarget)
			res := kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldTarget))
			ExpectWithOffset(1, res).To(helpers.CMDSuccess(), "%q cannot curl to %q",
				appPods[helpers.App2], worldTarget)

			By("Testing that connection from %q to %q shouldn't work",
				appPods[helpers.App2], worldInvalidTarget)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldInvalidTarget))
			ExpectWithOffset(1, res).ShouldNot(helpers.CMDSuccess(),
				"%q can curl to %q when it should fail", appPods[helpers.App2], worldInvalidTarget)

			By("Testing that connection from %q to %q works",
				appPods[helpers.App2], worldTargetIP)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldTargetIP))
			res.ExpectSuccess("%q cannot curl to %q during restart", helpers.App2, worldTargetIP)

			By("Testing that connection from %q to %q should not work",
				appPods[helpers.App2], worldInvalidTargetIP)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldInvalidTargetIP))
			res.ExpectFail("%q can connect when it should not work", helpers.App2)
		}

		fqndProxyPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-policy.yaml")

		_, err = kubectl.CiliumPolicyAction(
			helpers.DefaultNamespace, fqndProxyPolicy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "Cannot install fqdn proxy policy")

		connectivityTest()
		By("restarting cilium pods")

		// kill pid 1 in each cilium pod
		cmd := fmt.Sprintf("%[1]s get pods -l k8s-app=cilium -n %[2]s |  tail -n +2 | cut -d ' ' -f 1 | xargs -I{} %[1]s exec -n %[2]s {} -- kill 1",
			helpers.KubectlCmd, helpers.CiliumNamespace)
		quit, run := kubectl.RepeatCommandInBackground(cmd)
		channelClosed := false
		defer func() {
			if !channelClosed {
				close(quit)
			}
		}()
		<-run // waiting for first run to finish

		By("Testing connectivity when cilium is restoring using IPS without DNS")
		res := kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldTargetIP))
		res.ExpectSuccess("%q cannot curl to %q during restart", helpers.App2, worldTargetIP)

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldInvalidTargetIP))
		res.ExpectFail("%q can connect when it should not work", helpers.App2)

		// Re-run connectivity test while Cilium is still restarting. This should succeed as the same
		// DNS names were used in a connectivity test before the restart.
		connectivityTest()

		channelClosed = true
		close(quit)

		ExpectAllPodsTerminated(kubectl)
		ExpectCiliumReady(kubectl)

		// Restart monitoring after Cilium restart
		monitorRes1After, monitorCancel1After := kubectl.MonitorStart(ciliumPodK8s1)
		monitorRes2After, monitorCancel2After := kubectl.MonitorStart(ciliumPodK8s2)
		defer func() {
			monitorCancel1After()
			monitorCancel2After()
			helpers.WriteToReportFile(monitorRes1After.CombineOutput().Bytes(), "fqdn-after-restart-cilium-monitor-k8s1.log")
			helpers.WriteToReportFile(monitorRes2After.CombineOutput().Bytes(), "fqdn-after-restart-cilium-monitor-k8s2.log")
		}()

		// @TODO This endpoint ready call SHOULD NOT be here
		// Here some packets can be lost due to two different scenarios:
		//
		// 1) On restore the endpoint/fqdn policies, the identity ID for the
		// CIDRSet can be different, so if one endpoint start to regenerate and
		// other still have the old identity things can mess around and some
		// IPs are not white listed correctly. To prevent this, a restore for
		// local-identities will be added in the future.
		//
		// 2) On restore, the Kubernetes watcher is sending the CNP back to
		// Cilium, and before the endoint is restored the CNP can be applied
		// without the ToCIDRSet, this means that there is no TOCIDR rule in
		// the cilium policy and traffic will be drop.

		// As mentioned above, these endpoints ready should not be there, the only
		// reason to have this piece of code here is to reduce a flaky test.
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after Cilium restarts")

		By("Testing connectivity when cilium is *restored* using IPS without DNS")
		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldTargetIP))
		res.ExpectSuccess("%q cannot curl to %q after restart", helpers.App2, worldTargetIP)

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldInvalidTargetIP))
		res.ExpectFail("%q can connect when it should not work", helpers.App2)

		By("Testing connectivity using DNS request when cilium is restored correctly")
		connectivityTest()
	})

	It("Validate that multiple specs are working correctly", func() {
		// To make sure that UUID in multiple specs are plumbed correctly to
		// Cilium Policy
		fqdnPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-multiple-specs.yaml")
		world1Target := worldTarget
		world2Target := worldInvalidTarget

		_, err := kubectl.CiliumPolicyAction(
			helpers.DefaultNamespace, fqdnPolicy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "Cannot install fqdn proxy policy")

		By("Validating APP2 policy connectivity")
		res := kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail("--retry 5 "+world1Target))
		res.ExpectSuccess("Can't connect to to a valid target when it should work")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(world2Target))
		res.ExpectFail("Can connect to a valid target when it should NOT work")

		By("Validating APP3 policy connectivity")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App3],
			helpers.CurlWithRetries(world2Target, 5, true))
		res.ExpectSuccess("Can't connect to to a valid target when it should work")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App3],
			helpers.CurlFail(world1Target))
		res.ExpectFail("Can connect to to a valid target when it should NOT work")
	})
})
<commit_msg>test: disable fqdn connectivity test during restart<commit_after>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"context"
	"fmt"
	"net"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sFQDNTest", func() {
	var (
		kubectl          *helpers.Kubectl
		backgroundCancel context.CancelFunc = func() {}
		backgroundError  error

		demoManifest   = ""
		ciliumFilename string

		apps    = []string{helpers.App2, helpers.App3}
		appPods map[string]string

		// The IPs are updated in BeforeAll
		worldTarget          = "http://vagrant-cache.ci.cilium.io"
		worldTargetIP        = "147.75.38.95"
		worldInvalidTarget   = "http://jenkins.cilium.io"
		worldInvalidTargetIP = "104.198.14.52"
	)

	BeforeAll(func() {
		// In case the IPs changed from above, update them here
		var lookupErr error
		err := helpers.WithTimeout(func() bool {
			addrs, err2 := net.LookupHost("vagrant-cache.ci.cilium.io")
			if err2 != nil {
				lookupErr = fmt.Errorf("error looking up vagrant-cache.ci.cilium.io: %s", err2)
				return false
			}
			worldTargetIP = addrs[0]
			return true
		}, "Could not get vagrant-cache.ci.cilium.io IP", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		Expect(err).Should(BeNil(), "Error obtaining IP for test: %s", lookupErr)

		lookupErr = nil
		err = helpers.WithTimeout(func() bool {
			addrs, err2 := net.LookupHost("jenkins.cilium.io")
			if err2 != nil {
				lookupErr = fmt.Errorf("error looking up jenkins.cilium.io: %s", err2)
				return false
			}
			worldInvalidTargetIP = addrs[0]
			return true
		}, "Could not get jenkins.cilium.io IP", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		Expect(err).Should(BeNil(), "Error obtaining IP for test: %s", lookupErr)

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoManifest = helpers.ManifestGet(kubectl.BasePath(), "demo.yaml")

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		By("Applying demo manifest")
		res := kubectl.ApplyDefault(demoManifest)
		res.ExpectSuccess("Demo config cannot be deployed")

		err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=testapp", helpers.HelperTimeout)
		Expect(err).Should(BeNil(), "Testapp is not ready after timeout")

		appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "id")
	})

	AfterFailed(func() {
		kubectl.CiliumReport("cilium service list", "cilium endpoint list")
	})

	AfterAll(func() {
		_ = kubectl.Delete(demoManifest)
		ExpectAllPodsTerminated(kubectl)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	JustBeforeEach(func() {
		backgroundCancel, backgroundError = kubectl.BackgroundReport("uptime")
		Expect(backgroundError).To(BeNil(), "Cannot start background report process")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		backgroundCancel()
	})

	AfterEach(func() {
		_ = kubectl.Exec(fmt.Sprintf("%s delete --all cnp", helpers.KubectlCmd))
	})

	SkipItIf(helpers.SkipQuarantined, "Restart Cilium validate that FQDN is still working", func() {
		// Test functionality:
		// - When Cilium is running) Connectivity from App2 application can
		// connect to DNS because dns-proxy filter the DNS request. If the
		// connection is made correctly the IP is whitelisted by the FQDN rule
		// until the DNS TTL expires.
		// - When Cilium is not running) The DNS-proxy is not working, so the IP
		// connectivity to an existing IP that was queried before will work,
		// meanwhile connections using new DNS request will fail.
		// - On restart) Cilium will restore the IPS that were white-listted in
		// the FQDN and connection will work as normal.

		ciliumPodK8s1, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.K8s1)
		Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1")
		monitorRes1, monitorCancel1 := kubectl.MonitorStart(ciliumPodK8s1)
		ciliumPodK8s2, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.K8s2)
		Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s2")
		monitorRes2, monitorCancel2 := kubectl.MonitorStart(ciliumPodK8s2)
		defer func() {
			monitorCancel1()
			monitorCancel2()
			helpers.WriteToReportFile(monitorRes1.CombineOutput().Bytes(), "fqdn-restart-cilium-monitor-k8s1.log")
			helpers.WriteToReportFile(monitorRes2.CombineOutput().Bytes(), "fqdn-restart-cilium-monitor-k8s2.log")
		}()

		connectivityTest := func() {
			By("Testing that connection from %q to %q should work",
				appPods[helpers.App2], worldTarget)
			res := kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldTarget))
			ExpectWithOffset(1, res).To(helpers.CMDSuccess(), "%q cannot curl to %q",
				appPods[helpers.App2], worldTarget)

			By("Testing that connection from %q to %q shouldn't work",
				appPods[helpers.App2], worldInvalidTarget)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldInvalidTarget))
			ExpectWithOffset(1, res).ShouldNot(helpers.CMDSuccess(),
				"%q can curl to %q when it should fail", appPods[helpers.App2], worldInvalidTarget)

			By("Testing that connection from %q to %q works",
				appPods[helpers.App2], worldTargetIP)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldTargetIP))
			res.ExpectSuccess("%q cannot curl to %q during restart", helpers.App2, worldTargetIP)

			By("Testing that connection from %q to %q should not work",
				appPods[helpers.App2], worldInvalidTargetIP)
			res = kubectl.ExecPodCmd(
				helpers.DefaultNamespace, appPods[helpers.App2],
				helpers.CurlFail(worldInvalidTargetIP))
			res.ExpectFail("%q can connect when it should not work", helpers.App2)
		}

		fqndProxyPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-policy.yaml")

		_, err = kubectl.CiliumPolicyAction(
			helpers.DefaultNamespace, fqndProxyPolicy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "Cannot install fqdn proxy policy")

		connectivityTest()
		By("restarting cilium pods")

		// kill pid 1 in each cilium pod
		cmd := fmt.Sprintf("%[1]s get pods -l k8s-app=cilium -n %[2]s |  tail -n +2 | cut -d ' ' -f 1 | xargs -I{} %[1]s exec -n %[2]s {} -- kill 1",
			helpers.KubectlCmd, helpers.CiliumNamespace)
		quit, run := kubectl.RepeatCommandInBackground(cmd)
		channelClosed := false
		defer func() {
			if !channelClosed {
				close(quit)
			}
		}()
		<-run // waiting for first run to finish

		By("Testing connectivity when cilium is restoring using IPS without DNS")
		res := kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldTargetIP))
		res.ExpectSuccess("%q cannot curl to %q during restart", helpers.App2, worldTargetIP)

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldInvalidTargetIP))
		res.ExpectFail("%q can connect when it should not work", helpers.App2)

		// This test is failing consistently in quarantine, see #11213. Disable it for now
		// to verify the rest of the test is running stable in quarantine. Once this is the
		// case we could move the rest of the test out of quarantine and quarantine only
		// this part.
		if false {
			// Re-run connectivity test while Cilium is still restarting. This should succeed as the same
			// DNS names were used in a connectivity test before the restart.
			connectivityTest()
		}

		channelClosed = true
		close(quit)

		ExpectAllPodsTerminated(kubectl)
		ExpectCiliumReady(kubectl)

		// Restart monitoring after Cilium restart
		monitorRes1After, monitorCancel1After := kubectl.MonitorStart(ciliumPodK8s1)
		monitorRes2After, monitorCancel2After := kubectl.MonitorStart(ciliumPodK8s2)
		defer func() {
			monitorCancel1After()
			monitorCancel2After()
			helpers.WriteToReportFile(monitorRes1After.CombineOutput().Bytes(), "fqdn-after-restart-cilium-monitor-k8s1.log")
			helpers.WriteToReportFile(monitorRes2After.CombineOutput().Bytes(), "fqdn-after-restart-cilium-monitor-k8s2.log")
		}()

		// @TODO This endpoint ready call SHOULD NOT be here
		// Here some packets can be lost due to two different scenarios:
		//
		// 1) On restore the endpoint/fqdn policies, the identity ID for the
		// CIDRSet can be different, so if one endpoint start to regenerate and
		// other still have the old identity things can mess around and some
		// IPs are not white listed correctly. To prevent this, a restore for
		// local-identities will be added in the future.
		//
		// 2) On restore, the Kubernetes watcher is sending the CNP back to
		// Cilium, and before the endoint is restored the CNP can be applied
		// without the ToCIDRSet, this means that there is no TOCIDR rule in
		// the cilium policy and traffic will be drop.

		// As mentioned above, these endpoints ready should not be there, the only
		// reason to have this piece of code here is to reduce a flaky test.
		err = kubectl.CiliumEndpointWaitReady()
		Expect(err).To(BeNil(), "Endpoints are not ready after Cilium restarts")

		By("Testing connectivity when cilium is *restored* using IPS without DNS")
		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldTargetIP))
		res.ExpectSuccess("%q cannot curl to %q after restart", helpers.App2, worldTargetIP)

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(worldInvalidTargetIP))
		res.ExpectFail("%q can connect when it should not work", helpers.App2)

		By("Testing connectivity using DNS request when cilium is restored correctly")
		connectivityTest()
	})

	It("Validate that multiple specs are working correctly", func() {
		// To make sure that UUID in multiple specs are plumbed correctly to
		// Cilium Policy
		fqdnPolicy := helpers.ManifestGet(kubectl.BasePath(), "fqdn-proxy-multiple-specs.yaml")
		world1Target := worldTarget
		world2Target := worldInvalidTarget

		_, err := kubectl.CiliumPolicyAction(
			helpers.DefaultNamespace, fqdnPolicy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "Cannot install fqdn proxy policy")

		By("Validating APP2 policy connectivity")
		res := kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail("--retry 5 "+world1Target))
		res.ExpectSuccess("Can't connect to to a valid target when it should work")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App2],
			helpers.CurlFail(world2Target))
		res.ExpectFail("Can connect to a valid target when it should NOT work")

		By("Validating APP3 policy connectivity")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App3],
			helpers.CurlWithRetries(world2Target, 5, true))
		res.ExpectSuccess("Can't connect to to a valid target when it should work")

		res = kubectl.ExecPodCmd(
			helpers.DefaultNamespace, appPods[helpers.App3],
			helpers.CurlFail(world1Target))
		res.ExpectFail("Can connect to to a valid target when it should NOT work")
	})
})
<|endoftext|>


<commit_before>package kateway

import (
	"bytes"
	"fmt"
	"os"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

var (
	javaSdkPubErr       = []byte("send msg error")
	javaSdkPubStatusErr = []byte("StatusLine is null")
)

func init() {
	monitor.RegisterWatcher("kateway.apperr", func() monitor.Watcher {
		return &WatchAppError{}
	})
}

// WatchAppError monitors app err log to find all pubsub related err.
type WatchAppError struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Wg     *sync.WaitGroup
}

func (this *WatchAppError) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchAppError) Run() {
	defer this.Wg.Done()

	appError := metrics.NewRegisteredCounter("kateway.apperr", nil)
	msgChan := make(chan *sarama.ConsumerMessage, 2000)

	if err := this.consumeAppErrLogs(msgChan); err != nil {
		close(msgChan)

		log.Error("%v", err)
		return
	}

	for {
		select {
		case <-this.Stop:
			log.Info("kateway.apperr stopped")
			return

		case msg, ok := <-msgChan:
			if !ok {
				return
			}

			appError.Inc(1)
			log.Warn("%d/%d %s", msg.Partition, msg.Offset, string(msg.Value))
		}
	}
}

func (this *WatchAppError) consumeAppErrLogs(msgChan chan<- *sarama.ConsumerMessage) error {
	var (
		cluster = os.Getenv("APPLOG_CLUSTER")
		topic   = os.Getenv("APPLOG_TOPIC")
	)

	if cluster == "" || topic == "" {
		return fmt.Errorf("empty cluster/topic params provided, kateway.apperr disabled")
	}

	zkcluster := this.Zkzone.NewCluster(cluster)
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		return fmt.Errorf("cluster[%s] has empty brokers", cluster)
	}
	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		return err
	}
	defer kfk.Close()

	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		return err
	}
	defer consumer.Close()

	partitions, err := kfk.Partitions(topic)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	for _, p := range partitions {
		wg.Add(1)
		go this.consumePartition(zkcluster, consumer, topic, p, sarama.OffsetNewest, msgChan, &wg)
	}

	wg.Wait()
	return nil
}

func (this *WatchAppError) consumePartition(zkcluster *zk.ZkCluster, consumer sarama.Consumer,
	topic string, partitionId int32, offset int64, msgCh chan<- *sarama.ConsumerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	p, err := consumer.ConsumePartition(topic, partitionId, offset)
	if err != nil {
		log.Error("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err)
		return
	}
	defer p.Close()

	for {
		select {
		case <-this.Stop:
			return

		case err := <-p.Errors():
			log.Critical("cluster[%s] %s/%d: %s", zkcluster.Name(), topic, partitionId, err)
			return

		case msg := <-p.Messages():
			if msg != nil && this.predicate(msg.Value) {
				msgCh <- msg
			}

		}
	}
}

func (this *WatchAppError) predicate(msg []byte) bool {
	switch {
	case bytes.Contains(msg, javaSdkPubErr):
		return true

	case bytes.Contains(msg, javaSdkPubStatusErr):
		return true

	default:
		return false
	}
}
<commit_msg>disable kguard apperr<commit_after>package kateway

import (
	"bytes"
	"fmt"
	"os"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

var (
	javaSdkPubErr       = []byte("send msg error")
	javaSdkPubStatusErr = []byte("StatusLine is null")
)

func init() {
	monitor.RegisterWatcher("kateway.apperr", func() monitor.Watcher {
		return &WatchAppError{}
	})
}

// WatchAppError monitors app err log to find all pubsub related err.
type WatchAppError struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Wg     *sync.WaitGroup
}

func (this *WatchAppError) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchAppError) Run() {
	defer this.Wg.Done()

	return // disable for now TODO

	appError := metrics.NewRegisteredCounter("kateway.apperr", nil)
	msgChan := make(chan *sarama.ConsumerMessage, 2000)

	if err := this.consumeAppErrLogs(msgChan); err != nil {
		close(msgChan)

		log.Error("%v", err)
		return
	}

	for {
		select {
		case <-this.Stop:
			log.Info("kateway.apperr stopped")
			return

		case msg, ok := <-msgChan:
			if !ok {
				return
			}

			appError.Inc(1)
			log.Warn("%d/%d %s", msg.Partition, msg.Offset, string(msg.Value))
		}
	}
}

func (this *WatchAppError) consumeAppErrLogs(msgChan chan<- *sarama.ConsumerMessage) error {
	var (
		cluster = os.Getenv("APPLOG_CLUSTER")
		topic   = os.Getenv("APPLOG_TOPIC")
	)

	if cluster == "" || topic == "" {
		return fmt.Errorf("empty cluster/topic params provided, kateway.apperr disabled")
	}

	zkcluster := this.Zkzone.NewCluster(cluster)
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		return fmt.Errorf("cluster[%s] has empty brokers", cluster)
	}
	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		return err
	}
	defer kfk.Close()

	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		return err
	}
	defer consumer.Close()

	partitions, err := kfk.Partitions(topic)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	for _, p := range partitions {
		wg.Add(1)
		go this.consumePartition(zkcluster, consumer, topic, p, sarama.OffsetNewest, msgChan, &wg)
	}

	wg.Wait()
	return nil
}

func (this *WatchAppError) consumePartition(zkcluster *zk.ZkCluster, consumer sarama.Consumer,
	topic string, partitionId int32, offset int64, msgCh chan<- *sarama.ConsumerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	p, err := consumer.ConsumePartition(topic, partitionId, offset)
	if err != nil {
		log.Error("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err)
		return
	}
	defer p.Close()

	for {
		select {
		case <-this.Stop:
			return

		case err := <-p.Errors():
			log.Critical("cluster[%s] %s/%d: %s", zkcluster.Name(), topic, partitionId, err)
			return

		case msg := <-p.Messages():
			if msg != nil && this.predicate(msg.Value) {
				msgCh <- msg
			}

		}
	}
}

func (this *WatchAppError) predicate(msg []byte) bool {
	switch {
	case bytes.Contains(msg, javaSdkPubErr):
		return true

	case bytes.Contains(msg, javaSdkPubStatusErr):
		return true

	default:
		return false
	}
}
<|endoftext|>


<commit_before>package goat

import (
	"log"
	"net/http"
	"os"
	"strconv"
	"syscall"
	"time"
)

// Application name
const App = "goat"

// Application version
const Version = "git-master"

// Manager is responsible for coordinating the application
func Manager(killChan chan bool, exitChan chan int) {
	// Set up logging flags
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	log.Println("Starting " + App + " " + Version)

	// Grab initial server status
	stat := getServerStatus()
	if stat == (serverStatus{}) {
		log.Println("Could not print get startup status")
	} else {
		log.Printf("%s - %s_%s (%d CPU) [pid: %d]", stat.Hostname, stat.Platform, stat.Architecture, stat.NumCPU, stat.PID)
	}

	// Load configuration
	config := loadConfig()
	if config == (conf{}) {
		log.Println("Cannot load configuration, exiting now.")
		os.Exit(1)
	}
	static.Config = config

	// Check for sane announce interval (10 minutes or more)
	if static.Config.Interval <= 600 {
		log.Println("Announce interval must be at least 600 seconds.")
		os.Exit(1)
	}

	// Attempt database connection
	if !dbPing() {
		log.Println("Cannot connect to database", dbName(), ", exiting now.")
		os.Exit(1)
	}
	log.Println("Database", dbName(), ": OK")

	// If configured, attempt redis connection
	if static.Config.Redis {
		if !redisPing() {
			log.Println("Cannot connect to Redis, exiting now.")
			os.Exit(1)
		}
		log.Println("Redis : OK")
	}

	// Start cron manager
	go cronManager()

	// Set up graceful shutdown channels
	httpSendChan := make(chan bool)
	httpRecvChan := make(chan bool)
	httpsSendChan := make(chan bool)
	httpsRecvChan := make(chan bool)
	udpSendChan := make(chan bool)
	udpRecvChan := make(chan bool)

	// Set up HTTP(S) route
	http.HandleFunc("/", parseHTTP)

	// Launch listeners as configured
	if static.Config.HTTP {
		go listenHTTP(httpSendChan, httpRecvChan)
		log.Println("HTTP listener launched on port " + strconv.Itoa(static.Config.Port))
	}
	if static.Config.HTTPS {
		go listenHTTPS(httpsSendChan, httpsRecvChan)
		log.Println("HTTPS listener launched on port " + strconv.Itoa(static.Config.SSL.Port))
	}
	if static.Config.UDP {
		go listenUDP(udpSendChan, udpRecvChan)
		log.Println("UDP listener launched on port " + strconv.Itoa(static.Config.Port))
	}

	// Wait for shutdown signal
	for {
		select {
		case <-killChan:
			// Trigger a graceful shutdown
			log.Println("Triggering graceful shutdown, press Ctrl+C again to force halt")

			// If program hangs for more than 10 seconds, trigger a force halt
			go func() {
				time.Sleep(10 * time.Second)
				log.Println("Timeout reached, triggering force halt")
				if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil {
					log.Println(err.Error())
				}
			}()

			// Stop listeners
			if static.Config.HTTP {
				log.Println("Stopping HTTP listener")
				httpSendChan <- true
				<-httpRecvChan
			}
			if static.Config.HTTPS {
				log.Println("Stopping HTTPS listener")
				httpsSendChan <- true
				<-httpsRecvChan
			}
			if static.Config.UDP {
				log.Println("Stopping UDP listener")
				udpSendChan <- true
				<-udpRecvChan
			}

			log.Println("Closing database connection")
			dbCloseFunc()

			// Report that program should exit gracefully
			exitChan <- 0
		}
	}
}
<commit_msg>Use time.After instead of time.Sleep<commit_after>package goat

import (
	"log"
	"net/http"
	"os"
	"strconv"
	"syscall"
	"time"
)

// Application name
const App = "goat"

// Application version
const Version = "git-master"

// Manager is responsible for coordinating the application
func Manager(killChan chan bool, exitChan chan int) {
	// Set up logging flags
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	log.Println("Starting " + App + " " + Version)

	// Grab initial server status
	stat := getServerStatus()
	if stat == (serverStatus{}) {
		log.Println("Could not print get startup status")
	} else {
		log.Printf("%s - %s_%s (%d CPU) [pid: %d]", stat.Hostname, stat.Platform, stat.Architecture, stat.NumCPU, stat.PID)
	}

	// Load configuration
	config := loadConfig()
	if config == (conf{}) {
		log.Println("Cannot load configuration, exiting now.")
		os.Exit(1)
	}
	static.Config = config

	// Check for sane announce interval (10 minutes or more)
	if static.Config.Interval <= 600 {
		log.Println("Announce interval must be at least 600 seconds.")
		os.Exit(1)
	}

	// Attempt database connection
	if !dbPing() {
		log.Println("Cannot connect to database", dbName(), ", exiting now.")
		os.Exit(1)
	}
	log.Println("Database", dbName(), ": OK")

	// If configured, attempt redis connection
	if static.Config.Redis {
		if !redisPing() {
			log.Println("Cannot connect to Redis, exiting now.")
			os.Exit(1)
		}
		log.Println("Redis : OK")
	}

	// Start cron manager
	go cronManager()

	// Set up graceful shutdown channels
	httpSendChan := make(chan bool)
	httpRecvChan := make(chan bool)
	httpsSendChan := make(chan bool)
	httpsRecvChan := make(chan bool)
	udpSendChan := make(chan bool)
	udpRecvChan := make(chan bool)

	// Set up HTTP(S) route
	http.HandleFunc("/", parseHTTP)

	// Launch listeners as configured
	if static.Config.HTTP {
		go listenHTTP(httpSendChan, httpRecvChan)
		log.Println("HTTP listener launched on port " + strconv.Itoa(static.Config.Port))
	}
	if static.Config.HTTPS {
		go listenHTTPS(httpsSendChan, httpsRecvChan)
		log.Println("HTTPS listener launched on port " + strconv.Itoa(static.Config.SSL.Port))
	}
	if static.Config.UDP {
		go listenUDP(udpSendChan, udpRecvChan)
		log.Println("UDP listener launched on port " + strconv.Itoa(static.Config.Port))
	}

	// Wait for shutdown signal
	for {
		select {
		case <-killChan:
			// Trigger a graceful shutdown
			log.Println("Triggering graceful shutdown, press Ctrl+C again to force halt")

			// If program hangs for more than 10 seconds, trigger a force halt
			go func() {
				<-time.After(10 * time.Second)
				log.Println("Timeout reached, triggering force halt")
				if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil {
					log.Println(err.Error())
				}
			}()

			// Stop listeners
			if static.Config.HTTP {
				log.Println("Stopping HTTP listener")
				httpSendChan <- true
				<-httpRecvChan
			}
			if static.Config.HTTPS {
				log.Println("Stopping HTTPS listener")
				httpsSendChan <- true
				<-httpsRecvChan
			}
			if static.Config.UDP {
				log.Println("Stopping UDP listener")
				udpSendChan <- true
				<-udpRecvChan
			}

			log.Println("Closing database connection")
			dbCloseFunc()

			// Report that program should exit gracefully
			exitChan <- 0
		}
	}
}
<|endoftext|>


<commit_before>package files

import (
	"errors"
	"os"
	"path/filepath"
	"sync"
)

// A TreeEntry is passed along the channel to the process function.
type TreeEntry struct {
	Path  string
	Finfo os.FileInfo
}

// ProcessFunc is the signature for a function that processes each file entry.
type ProcessFunc func(done <-chan struct{}, f <-chan TreeEntry, result chan<- string)

// IgnoreFunc is a function definition that tells walkFiles when it should
// ignore files and directory paths.
type IgnoreFunc func(path string, finfo os.FileInfo) bool

// walkFiles walks a directory tree. It ignores files and paths with permission errors. All other errors
// will cause it to stop walking the tree. For every path segment it will call ignorePathFn and skip that
// file or sub directory if it returns true. Otherwise all regular files are sent along the filesChan
// for further processing.
func walkFiles(done <-chan struct{}, root string, ignorePathFn IgnoreFunc) (<-chan TreeEntry, <-chan error) {
	// filesChan channel is where we send regular files that aren't ignored.
	filesChan := make(chan TreeEntry)

	// errChan sends errors found while walking the tree.
	errChan := make(chan error, 1)

	// Walk the tree in separate go routine. Send back results and errors in the filesChan and errChan channels.
	go func() {
		defer close(filesChan)
		errChan <- filepath.Walk(root, func(path string, finfo os.FileInfo, err error) error {
			switch {
			case err != nil && os.IsPermission(err):
				// Permission errors are ignored. Just continue walking the tree
				// without processing the file or directory.
				return nil

			case err != nil:
				// All other errors cause walking to stop.
				return err

			case ignorePathFn(path, finfo):
				// if ignorePathFn returns true then skip processing this entry.
				if finfo.IsDir() {
					// If entry is a directory, then skip processing that
					// entry sub tree.
					return filepath.SkipDir
				}
				return nil

			case !finfo.Mode().IsRegular():
				// Only process regular files. This means the following types
				// are not processed: Dir, Symbol Link, Named Pip, Socket,
				// and Device files.
				return nil

			default:
				// This is a regular file, so send it along the fileChan for
				// potential processing.
				entry := TreeEntry{
					Path:  path,
					Finfo: finfo,
				}
				select {
				case filesChan <- entry:
				case <-done:
					// if we receive a message on the done channel, then stop
					// walking the tree.
					return errors.New("walk canceled")
				}
				return nil
			}

		})
	}()
	return filesChan, errChan
}

// PWalk will walk a tree and process files in it in parallel. fn is the process function. "n" go routines will
// be started with fn for processing. The ignorePathFn determines whether a file or directory path should be
// ignored. ignorePathFn can be nil. If it is nil then no files or paths are ignored.
func PWalk(root string, n int, fn ProcessFunc, ignorePathFn IgnoreFunc) (<-chan string, <-chan error) {
	done := make(chan struct{})
	defer close(done)
	// default to never ignoring files and directory paths.
	ignoreFn := neverIgnore
	if ignorePathFn != nil {
		// user supplied a function to test for ignoring entries.
		ignoreFn = ignorePathFn
	}

	filesChan, errChan := walkFiles(done, root, ignoreFn)

	// results holds the results of processing each entry. Its value is
	// ignored and only exists so that PWalk will block until all
	// processing has completed.
	results := make(chan string)

	// Setup a WaitGroup equal to the number of routines processing
	// file entries.
	var wg sync.WaitGroup
	wg.Add(n)

	// Start "n" fn routines to process files. When each one completes
	// processing all entries along its channel signal done.
	for i := 0; i < n; i++ {
		go func() {
			fn(done, filesChan, results)
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(results)
	}()
	for r := range results {
		var _ = r
	}
	return results, errChan
}

// neverIgnore is the default method for checking if an entry should be ignored.
// It never ignores an entry.
func neverIgnore(path string, finfo os.FileInfo) bool {
	return false
}
<commit_msg>Move configuration of PWalk into a structure and hang methods off of the structure. This decreases the signature of the PWalk method, and allows easier communication of parameters to private methods.<commit_after>package files

import (
	"errors"
	"os"
	"path/filepath"
	"sync"
)

// A TreeEntry is passed along the channel to the process function.
type TreeEntry struct {
	Path  string
	Finfo os.FileInfo
}

// ProcessFunc is the signature for a function that processes each file entry.
type ProcessFunc func(done <-chan struct{}, f <-chan TreeEntry, result chan<- string)

// IgnoreFunc is a function definition that tells walkFiles when it should
// ignore files and directory paths.
type IgnoreFunc func(path string, finfo os.FileInfo) bool

// PWalker implements a parallel walker. It holds the options and processing
// methods to use during the walk.
type PWalker struct {
	// NumParallel is the number of go routines to start for processing.
	NumParallel int

	// ProcessFn is the function to use to process entries.
	ProcessFn ProcessFunc

	// IgnoreFn is the method to test if an entry should be ignored. If
	// IgnoreFn is nil then all entries will be process.
	IgnoreFn IgnoreFunc

	// ProcessDirs tells the walker whether directory entries should also
	// be passed to ProcessFn. It defaults to false, so by default directories
	// are not passed to ProcessFn.
	ProcessDirs bool
}

// walkFiles walks a directory tree. It ignores files and paths with permission errors. All other errors
// will cause it to stop walking the tree. For every path segment it will call ignorePathFn and skip that
// file or sub directory if it returns true. Otherwise all regular files are sent along the filesChan
// for further processing.
func (p *PWalker) walkFiles(done <-chan struct{}, root string) (<-chan TreeEntry, <-chan error) {
	// filesChan channel is where we send regular files that aren't ignored.
	filesChan := make(chan TreeEntry)

	// errChan sends errors found while walking the tree.
	errChan := make(chan error, 1)

	// Walk the tree in separate go routine. Send back results and errors in the filesChan and errChan channels.
	go func() {
		defer close(filesChan)
		errChan <- filepath.Walk(root, func(path string, finfo os.FileInfo, err error) error {
			switch {
			case err != nil && os.IsPermission(err):
				// Permission errors are ignored. Just continue walking the tree
				// without processing the file or directory.
				return nil

			case err != nil:
				// All other errors cause walking to stop.
				return err

			case p.IgnoreFn(path, finfo):
				// if ignorePathFn returns true then skip processing this entry.
				if finfo.IsDir() {
					// If entry is a directory, then skip processing that
					// entry sub tree.
					return filepath.SkipDir
				}
				return nil

			case !finfo.Mode().IsRegular() && !finfo.IsDir():
				// Only process regular files. This means the following types
				// are not processed: Symbol Link, Named Pip, Socket,
				// and Device files.
				//
				// Directories are a special case and are handled in the default
				// clause, where regulars files are also handled.
				return nil

			default:
				// Directories aren't being processed, so skip this entry.
				if finfo.IsDir() && !p.ProcessDirs {
					return nil
				}

				// This is an entry we can process, so send it along the fileChan
				entry := TreeEntry{
					Path:  path,
					Finfo: finfo,
				}
				select {
				case filesChan <- entry:
				case <-done:
					// if we receive a message on the done channel, then stop
					// walking the tree.
					return errors.New("walk canceled")
				}
				return nil
			}

		})
	}()
	return filesChan, errChan
}

// PWalk will walk a tree and process files in it in parallel. fn is the process function. "n" go routines will
// be started with fn for processing. The ignorePathFn determines whether a file or directory path should be
// ignored. ignorePathFn can be nil. If it is nil then no files or paths are ignored.
func (p *PWalker) PWalk(root string) (<-chan string, <-chan error) {
	done := make(chan struct{})
	defer close(done)

	if p.IgnoreFn == nil {
		p.IgnoreFn = neverIgnore
	}

	if p.NumParallel < 1 {
		p.NumParallel = 3
	}

	filesChan, errChan := p.walkFiles(done, root)

	// results holds the results of processing each entry. Its value is
	// ignored and only exists so that PWalk will block until all
	// processing has completed.
	results := make(chan string)

	// Setup a WaitGroup equal to the number of routines processing
	// file entries.
	var wg sync.WaitGroup
	wg.Add(p.NumParallel)

	// Start "n" fn routines to process files. When each one completes
	// processing all entries along its channel signal done.
	for i := 0; i < p.NumParallel; i++ {
		go func() {
			p.ProcessFn(done, filesChan, results)
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(results)
	}()
	for r := range results {
		var _ = r
	}
	return results, errChan
}

// neverIgnore is the default method for checking if an entry should be ignored.
// It never ignores an entry.
func neverIgnore(_ string, _ os.FileInfo) bool {
	return false
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"github.com/versent/saml2aws/helper/credentials"
	"github.com/versent/saml2aws/helper/osxkeychain"
)

func init() {
	fmt.Println("Loading osx keychain helper")
	credentials.CurrentHelper = &osxkeychain.Osxkeychain{}
}
<commit_msg>fix(Helper) Remove helper load message actually fixes #53.<commit_after>package commands

import (
	"github.com/versent/saml2aws/helper/credentials"
	"github.com/versent/saml2aws/helper/osxkeychain"
)

func init() {
	credentials.CurrentHelper = &osxkeychain.Osxkeychain{}
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Adapted from code.google.com/p/google-api-go-client/examples/main.go

package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"hash/fnv"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"code.google.com/p/goauth2/oauth"
)

type Options struct {
	scope        string
	cacheToken   bool
	debug        bool
	clientId     string
	clientSecret string
}

func getClient(options *Options) *http.Client {

	config := &oauth.Config{
		ClientId:     options.clientId,
		ClientSecret: options.clientSecret,
		Scope:        options.scope,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
	}
	config.Scope = options.scope
	return getOAuthClient(config, options)
}

var (
	demoFunc  = make(map[string]func(*http.Client, []string))
	demoScope = make(map[string]string)
)

func registerDemo(name, scope string, main func(c *http.Client, argv []string)) {
	if demoFunc[name] != nil {
		panic(name + " already registered")
	}
	demoFunc[name] = main
	demoScope[name] = scope
}

func osUserCacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(os.Getenv("HOME"), "Library", "Caches")
	case "linux", "freebsd":
		return filepath.Join(os.Getenv("HOME"), ".cache")
	}
	log.Printf("TODO: osUserCacheDir on GOOS %q", runtime.GOOS)
	return "."
}

func tokenCacheFile(config *oauth.Config) string {
	hash := fnv.New32a()
	hash.Write([]byte(config.ClientId))
	hash.Write([]byte(config.ClientSecret))
	hash.Write([]byte(config.Scope))
	fn := fmt.Sprintf("go-api-demo-tok%v", hash.Sum32())
	return filepath.Join(osUserCacheDir(), url.QueryEscape(fn))
}

func tokenFromFile(file string, cacheToken bool) (*oauth.Token, error) {
	if !cacheToken {
		return nil, errors.New("--cachetoken is false")
	}
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	t := new(oauth.Token)
	err = gob.NewDecoder(f).Decode(t)
	return t, err
}

func saveToken(file string, token *oauth.Token) {
	f, err := os.Create(file)
	if err != nil {
		log.Printf("Warning: failed to cache oauth token: %v", err)
		return
	}
	defer f.Close()
	gob.NewEncoder(f).Encode(token)
}

func condDebugTransport(debug bool) http.RoundTripper {
	if debug {
		return &logTransport{http.DefaultTransport}
	} else {
		return http.DefaultTransport
	}
}

func getOAuthClient(config *oauth.Config, options *Options) *http.Client {
	cacheFile := tokenCacheFile(config)
	token, err := tokenFromFile(cacheFile, options.cacheToken)
	if err != nil {
		token = tokenFromWeb(config, options)
		saveToken(cacheFile, token)
	} else {
		log.Printf("Using cached token %#v from %q", token, cacheFile)
	}

	t := &oauth.Transport{
		Token:     token,
		Config:    config,
		Transport: condDebugTransport(options.debug),
	}
	return t.Client()
}

func tokenFromWeb(config *oauth.Config, options *Options) *oauth.Token {
	ch := make(chan string)
	randState := fmt.Sprintf("st%d", time.Now().UnixNano())
	ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/favicon.ico" {
			http.Error(rw, "", 404)
			return
		}
		if req.FormValue("state") != randState {
			log.Printf("State doesn't match: req = %#v", req)
			http.Error(rw, "", 500)
			return
		}
		if code := req.FormValue("code"); code != "" {
			fmt.Fprintf(rw, "
Success
Authorized.")
			rw.(http.Flusher).Flush()
			ch <- code
			return
		}
		log.Printf("no code")
		http.Error(rw, "", 500)
	}))
	defer ts.Close()

	config.RedirectURL = ts.URL
	authUrl := config.AuthCodeURL(randState)
	go openUrl(authUrl)
	log.Printf("Authorize this app at: %s", authUrl)
	code := <-ch
	log.Printf("Got code: %s", code)

	t := &oauth.Transport{
		Config:    config,
		Transport: condDebugTransport(options.debug),
	}
	_, err := t.Exchange(code)
	if err != nil {
		log.Fatalf("Token exchange error: %v", err)
	}
	return t.Token
}

func openUrl(url string) {
	try := []string{"xdg-open", "google-chrome", "open"}
	for _, bin := range try {
		err := exec.Command(bin, url).Run()
		if err == nil {
			return
		}
	}
	log.Printf("Error opening URL in browser.")
}
<commit_msg>Comment out echoing of token<commit_after>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Adapted from code.google.com/p/google-api-go-client/examples/main.go

package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"hash/fnv"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"code.google.com/p/goauth2/oauth"
)

type Options struct {
	scope        string
	cacheToken   bool
	debug        bool
	clientId     string
	clientSecret string
}

func getClient(options *Options) *http.Client {

	config := &oauth.Config{
		ClientId:     options.clientId,
		ClientSecret: options.clientSecret,
		Scope:        options.scope,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
	}
	config.Scope = options.scope
	return getOAuthClient(config, options)
}

var (
	demoFunc  = make(map[string]func(*http.Client, []string))
	demoScope = make(map[string]string)
)

func registerDemo(name, scope string, main func(c *http.Client, argv []string)) {
	if demoFunc[name] != nil {
		panic(name + " already registered")
	}
	demoFunc[name] = main
	demoScope[name] = scope
}

func osUserCacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(os.Getenv("HOME"), "Library", "Caches")
	case "linux", "freebsd":
		return filepath.Join(os.Getenv("HOME"), ".cache")
	}
	log.Printf("TODO: osUserCacheDir on GOOS %q", runtime.GOOS)
	return "."
}

func tokenCacheFile(config *oauth.Config) string {
	hash := fnv.New32a()
	hash.Write([]byte(config.ClientId))
	hash.Write([]byte(config.ClientSecret))
	hash.Write([]byte(config.Scope))
	fn := fmt.Sprintf("go-api-demo-tok%v", hash.Sum32())
	return filepath.Join(osUserCacheDir(), url.QueryEscape(fn))
}

func tokenFromFile(file string, cacheToken bool) (*oauth.Token, error) {
	if !cacheToken {
		return nil, errors.New("--cachetoken is false")
	}
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	t := new(oauth.Token)
	err = gob.NewDecoder(f).Decode(t)
	return t, err
}

func saveToken(file string, token *oauth.Token) {
	f, err := os.Create(file)
	if err != nil {
		log.Printf("Warning: failed to cache oauth token: %v", err)
		return
	}
	defer f.Close()
	gob.NewEncoder(f).Encode(token)
}

func condDebugTransport(debug bool) http.RoundTripper {
	if debug {
		return &logTransport{http.DefaultTransport}
	} else {
		return http.DefaultTransport
	}
}

func getOAuthClient(config *oauth.Config, options *Options) *http.Client {
	cacheFile := tokenCacheFile(config)
	token, err := tokenFromFile(cacheFile, options.cacheToken)
	if err != nil {
		token = tokenFromWeb(config, options)
		saveToken(cacheFile, token)
	} else {
		// log.Printf("Using cached token %#v from %q", token, cacheFile)
	}

	t := &oauth.Transport{
		Token:     token,
		Config:    config,
		Transport: condDebugTransport(options.debug),
	}
	return t.Client()
}

func tokenFromWeb(config *oauth.Config, options *Options) *oauth.Token {
	ch := make(chan string)
	randState := fmt.Sprintf("st%d", time.Now().UnixNano())
	ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/favicon.ico" {
			http.Error(rw, "", 404)
			return
		}
		if req.FormValue("state") != randState {
			log.Printf("State doesn't match: req = %#v", req)
			http.Error(rw, "", 500)
			return
		}
		if code := req.FormValue("code"); code != "" {
			fmt.Fprintf(rw, "
Success
Authorized.")
			rw.(http.Flusher).Flush()
			ch <- code
			return
		}
		log.Printf("no code")
		http.Error(rw, "", 500)
	}))
	defer ts.Close()

	config.RedirectURL = ts.URL
	authUrl := config.AuthCodeURL(randState)
	go openUrl(authUrl)
	log.Printf("Authorize this app at: %s", authUrl)
	code := <-ch
	log.Printf("Got code: %s", code)

	t := &oauth.Transport{
		Config:    config,
		Transport: condDebugTransport(options.debug),
	}
	_, err := t.Exchange(code)
	if err != nil {
		log.Fatalf("Token exchange error: %v", err)
	}
	return t.Token
}

func openUrl(url string) {
	try := []string{"xdg-open", "google-chrome", "open"}
	for _, bin := range try {
		err := exec.Command(bin, url).Run()
		if err == nil {
			return
		}
	}
	log.Printf("Error opening URL in browser.")
}
<|endoftext|>


<commit_before>package crypto

// xlattice_go/crypto/gocheck.go

import (
	. "launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { TestingT(t) }

type XLSuite struct{}

var _ = Suite(&XLSuite{})
<commit_msg>helloAndReply*go moved to ../msg<commit_after>package crypto

// xlattice_go/crypto/gocheck.go

import (
	. "launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { TestingT(t) }

type XLSuite struct{}

var _ = Suite(&XLSuite{})

<|endoftext|>


<commit_before>package godnsbl

import (
  "testing"
  "sync"
)

func TestGetDnsblRespFunc(t *testing.T) {
  var wg sync.WaitGroup
  var resp dnsblReturn
  wg.Add(1)
  go getDnsblResp(&wg, &resp, getFlipIP("127.0.0.2"), "b.barracudacentral.org")
  wg.Wait()

  if len(resp.Responses) != 1{
    t.Errorf("Expected length of responses to be 1, got: %d", len(resp.Responses))
  }

  if resp.Responses[0].Status != "ok"{
    t.Errorf("Expected status to be ok, got : %s", resp.Responses[0].Status)
  }

  if resp.Responses[0].Msg != ""{
    t.Errorf("No message expected, got : %s", resp.Responses[0].Msg)
  }

  if resp.Responses[0].Listed != true{
    t.Errorf("Expected blacklist to be true, got : %s", resp.Responses[0].Listed)
  }

  if resp.Responses[0].Name != "b.barracudacentral.org"{
    t.Errorf("Expected name to be b.barracudacentral.org, got : %s", resp.Responses[0].Name)
  }

  if resp.Responses[0].RespTime == 0{
    t.Errorf("Response time should not be zero")
  }

  if resp.Responses[0].Resp != "127.0.0.2"{
    t.Errorf("Resp should be 127.0.0.2, got: %s", resp.Responses[0].Resp)
  }

}

func TestCheckBlacklistFunc(t *testing.T) {
  resp := CheckBlacklist("127.0.0.2")

  if len(resp.Responses) != len(blacklistDomains){
    t.Errorf("Expected length of responses to be %d, got: %d", len(blacklistDomains), len(resp.Responses))
  }

  if len(resp.Listed) != true{
    t.Errorf("Expected blacklist to be true, got : %s", resp.Listed)
  }

}

func TestGetflipIPFunc(t *testing.T) {
  flipped := getFlipIP("10.11.12.13")
  if flipped != "13.12.11.10"{
		t.Errorf("IP did not flip properly, expecting 13.12.11.10 got: %s", flipped)
	}
}
<commit_msg>Not sure how that pasts my local tests...<commit_after>package godnsbl

import (
  "testing"
  "sync"
)

func TestGetDnsblRespFunc(t *testing.T) {
  var wg sync.WaitGroup
  var resp dnsblReturn
  wg.Add(1)
  go getDnsblResp(&wg, &resp, getFlipIP("127.0.0.2"), "b.barracudacentral.org")
  wg.Wait()

  if len(resp.Responses) != 1{
    t.Errorf("Expected length of responses to be 1, got: %d", len(resp.Responses))
  }

  if resp.Responses[0].Status != "ok"{
    t.Errorf("Expected status to be ok, got : %s", resp.Responses[0].Status)
  }

  if resp.Responses[0].Msg != ""{
    t.Errorf("No message expected, got : %s", resp.Responses[0].Msg)
  }

  if resp.Responses[0].Listed != true{
    t.Errorf("Expected blacklist to be true, got : %s", resp.Responses[0].Listed)
  }

  if resp.Responses[0].Name != "b.barracudacentral.org"{
    t.Errorf("Expected name to be b.barracudacentral.org, got : %s", resp.Responses[0].Name)
  }

  if resp.Responses[0].RespTime == 0{
    t.Errorf("Response time should not be zero")
  }

  if resp.Responses[0].Resp != "127.0.0.2"{
    t.Errorf("Resp should be 127.0.0.2, got: %s", resp.Responses[0].Resp)
  }

}

func TestCheckBlacklistFunc(t *testing.T) {
  resp := CheckBlacklist("127.0.0.2")

  if len(resp.Responses) != len(blacklistDomains){
    t.Errorf("Expected length of responses to be %d, got: %d", len(blacklistDomains), len(resp.Responses))
  }

  if resp.Listed != true{
    t.Errorf("Expected blacklist to be true, got : %s", resp.Listed)
  }

}

func TestGetflipIPFunc(t *testing.T) {
  flipped := getFlipIP("10.11.12.13")
  if flipped != "13.12.11.10"{
		t.Errorf("IP did not flip properly, expecting 13.12.11.10 got: %s", flipped)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "encoding/json"
    "flag"
    "fmt"
    "github.com/cailei/gopm_index/gopm_index"
    "io/ioutil"
    "log"
    "os"
)

func cmd_publish(args []string) {
    // parse flags
    var help bool
    var verbose bool

    f := flag.NewFlagSet("publish_flags", flag.ExitOnError)
    f.BoolVar(&help, "help", false, "show help info")
    f.BoolVar(&help, "h", false, "show help info")
    f.BoolVar(&verbose, "verbose", false, "verbose")
    f.BoolVar(&verbose, "v", false, "verbose")
    f.Usage = print_publish_help
    f.Parse(args)

    if help {
        print_publish_help()
        return
    }

    // get package folder
    json_names := f.Args()
    if len(json_names) == 0 {
        fmt.Print("\nPlease provide a <package>.json file to publish.\n")
        print_publish_help()
        return
    }

    for i := 0; i < len(json_names); i++ {
        publish_package(json_names[i], verbose)
    }
}

func publish_package(json_name string, verbose bool) {
    // read json content
    json_file, err := os.Open(json_name)
    if err != nil {
        log.Fatalln(err)
    }
    defer json_file.Close()

    json_content, err := ioutil.ReadAll(json_file)
    if err != nil {
        log.Fatalln(err)
    }

    // unmarshal to PackageMeta object
    var meta gopm_index.PackageMeta
    if err := json.Unmarshal(json_content, &meta); err != nil {
        log.Fatalln(err)
    }

    // check mandatory fields
    if meta.Name == "" {
        log.Fatalf("%v: 'name' is empty\n", json_name)
    }
    if meta.Description == "" {
        log.Fatalf("%v: 'description' is empty\n", json_name)
    }
    if meta.Author.Name == "" {
        log.Fatalf("%v: 'author.name' is empty\n", json_name)
    }
    if meta.Repositories == nil {
        log.Fatalf("%v: 'repositories' is missing\n", json_name)
    }
    if len(meta.Repositories) == 0 {
        log.Fatalf("%v: 'repositories' is empty\n", json_name)
    }

    // check name uniqueness on the server
    exists := agent_package_name_exists(meta.Name)
    if exists {
        log.Fatalf("The package name '%v' already exists in the index.\n", meta.Name)
    }

    // post the json to the server
    agent_upload_package(meta)

    fmt.Printf("Successfully publish package '%v'\n", meta.Name)
}

func print_publish_help() {
    fmt.Print(`
gopm publish <package name>:
    publish your package to the central index database.

options:
    -v, -verbose    verbose
    -h, -help       show help info

`)

}
<commit_msg>move package existence check to the server side<commit_after>package main

import (
    "encoding/json"
    "flag"
    "fmt"
    "github.com/cailei/gopm_index/gopm_index"
    "io/ioutil"
    "log"
    "os"
)

func cmd_publish(args []string) {
    // parse flags
    var help bool
    var verbose bool

    f := flag.NewFlagSet("publish_flags", flag.ExitOnError)
    f.BoolVar(&help, "help", false, "show help info")
    f.BoolVar(&help, "h", false, "show help info")
    f.BoolVar(&verbose, "verbose", false, "verbose")
    f.BoolVar(&verbose, "v", false, "verbose")
    f.Usage = print_publish_help
    f.Parse(args)

    if help {
        print_publish_help()
        return
    }

    // get package folder
    json_names := f.Args()
    if len(json_names) == 0 {
        fmt.Print("\nPlease provide a <package>.json file to publish.\n")
        print_publish_help()
        return
    }

    for i := 0; i < len(json_names); i++ {
        publish_package(json_names[i], verbose)
    }
}

func publish_package(json_name string, verbose bool) {
    // read json content
    json_file, err := os.Open(json_name)
    if err != nil {
        log.Fatalln(err)
    }
    defer json_file.Close()

    json_content, err := ioutil.ReadAll(json_file)
    if err != nil {
        log.Fatalln(err)
    }

    // unmarshal to PackageMeta object
    var meta gopm_index.PackageMeta
    if err := json.Unmarshal(json_content, &meta); err != nil {
        log.Fatalln(err)
    }

    // check mandatory fields
    if meta.Name == "" {
        log.Fatalf("%v: 'name' is empty\n", json_name)
    }
    if meta.Description == "" {
        log.Fatalf("%v: 'description' is empty\n", json_name)
    }
    if meta.Author.Name == "" {
        log.Fatalf("%v: 'author.name' is empty\n", json_name)
    }
    if meta.Repositories == nil {
        log.Fatalf("%v: 'repositories' is missing\n", json_name)
    }
    if len(meta.Repositories) == 0 {
        log.Fatalf("%v: 'repositories' is empty\n", json_name)
    }

    // post the json to the server
    agent_upload_package(meta)

    fmt.Printf("Successfully publish package '%v'\n", meta.Name)
}

func print_publish_help() {
    fmt.Print(`
gopm publish <package name>:
    publish your package to the central index database.

options:
    -v, -verbose    verbose
    -h, -help       show help info

`)

}
<|endoftext|>


<commit_before>package gopushbullet

//ErrorResponse (any non-200 error code) contain information on the kind of error that happened.
type ErrorResponse struct {
	Error struct {
		Message string `json:"message"`
		Type    string `json:"type"`
		Cat     string `json:"cat"`
	} `json:"error"`
}

//PushMessage describes a message to be sent via PushBullet. Only one of the first 4 properties may be specified with a message being sent.
type PushMessage struct {
	DeviceID   string `json:"device_iden"`
	Email      string `json:"email"`
	ChannelTag string `json:"channel_tag"`
	ClientID   string `json:"client_iden"`

	// Type indicates the type of push message being sent
	Type    string   `json:"type"`    // note, link, address, checklist, or file
	Title   string   `json:"title"`   // Title is used withn note, link, and checklist types
	Body    string   `json:"body"`    // Body is used with link, note, or file types
	URL     string   `json:"url"`     // URL is used for link types
	Address string   `json:"address"` // Address is used with address type
	Items   []string `json:"items"`   // Items are used with checklist types
	// The following are used with file types
	FileName       string `json:"file_name"`
	FileType       string `json:"file_type"` // MIME type of the file
	FileURL        string `json:"file_url"`
	SourceDeviceID string `json:"source_device_iden"`
}

//Device describes a registered device (phone, stream).
type Device struct {
	ID           string  `json:"iden"`
	PushToken    string  `json:"push_token"`
	AppVersion   int     `json:"app_version"`
	Fingerprint  string  `json:"fingerprint"`
	Active       bool    `json:"active"`
	Nickname     string  `json:"nickname"`
	Manufacturer string  `json:"manufacturer"`
	Type         string  `json:"type"`
	Created      float32 `json:"created"`
	Modified     float32 `json:"modified"`
	Model        string  `json:"model"`
	Pushable     bool    `json:"pushable"`
}

//Contact describes a contact entry.
type Contact struct {
	ID              string  `json:"iden"`
	Name            string  `json:"name"`
	Created         float32 `json:"created"`
	Modified        float32 `json:"modified"`
	Email           string  `json:"email"`
	EmailNormalized string  `json:"email_normalized"`
	Active          bool    `json:"active"`
}

//Subscription describes a channel subscription.
type Subscription struct {
	ID       string  `json:"iden"`
	Created  float32 `json:"created"`
	Modified float32 `json:"modified"`
	Active   bool    `json:"active"`
	Channel  Channel `json:"channel"`
}

//Channel describes a channel on a subscription.
type Channel struct {
	ID          string `json:"iden"`
	Tag         string `json:"tag"`
	Name        string `json:"name"`
	Description string `json:"description"`
	ImageURL    string `json:"image_url"`
}
<commit_msg>Add User struct<commit_after>package gopushbullet

//ErrorResponse (any non-200 error code) contain information on the kind of error that happened.
type ErrorResponse struct {
	Error struct {
		Message string `json:"message"`
		Type    string `json:"type"`
		Cat     string `json:"cat"`
	} `json:"error"`
}

//PushMessage describes a message to be sent via PushBullet. Only one of the first 4 properties may be specified with a message being sent.
type PushMessage struct {
	DeviceID   string `json:"device_iden"`
	Email      string `json:"email"`
	ChannelTag string `json:"channel_tag"`
	ClientID   string `json:"client_iden"`

	// Type indicates the type of push message being sent
	Type    string   `json:"type"`    // note, link, address, checklist, or file
	Title   string   `json:"title"`   // Title is used withn note, link, and checklist types
	Body    string   `json:"body"`    // Body is used with link, note, or file types
	URL     string   `json:"url"`     // URL is used for link types
	Address string   `json:"address"` // Address is used with address type
	Items   []string `json:"items"`   // Items are used with checklist types
	// The following are used with file types
	FileName       string `json:"file_name"`
	FileType       string `json:"file_type"` // MIME type of the file
	FileURL        string `json:"file_url"`
	SourceDeviceID string `json:"source_device_iden"`
}

//Device describes a registered device (phone, stream).
type Device struct {
	ID           string  `json:"iden"`
	PushToken    string  `json:"push_token"`
	AppVersion   int     `json:"app_version"`
	Fingerprint  string  `json:"fingerprint"`
	Active       bool    `json:"active"`
	Nickname     string  `json:"nickname"`
	Manufacturer string  `json:"manufacturer"`
	Type         string  `json:"type"`
	Created      float32 `json:"created"`
	Modified     float32 `json:"modified"`
	Model        string  `json:"model"`
	Pushable     bool    `json:"pushable"`
}

//Contact describes a contact entry.
type Contact struct {
	ID              string  `json:"iden"`
	Name            string  `json:"name"`
	Created         float32 `json:"created"`
	Modified        float32 `json:"modified"`
	Email           string  `json:"email"`
	EmailNormalized string  `json:"email_normalized"`
	Active          bool    `json:"active"`
}

//Subscription describes a channel subscription.
type Subscription struct {
	ID       string  `json:"iden"`
	Created  float32 `json:"created"`
	Modified float32 `json:"modified"`
	Active   bool    `json:"active"`
	Channel  Channel `json:"channel"`
}

//Channel describes a channel on a subscription.
type Channel struct {
	ID          string `json:"iden"`
	Tag         string `json:"tag"`
	Name        string `json:"name"`
	Description string `json:"description"`
	ImageURL    string `json:"image_url"`
}

//User describes the authenticated user.
type User struct {
	ID              string  `json:"iden"`
	Email           string  `json:"email"`
	EmailNormalized string  `json:"email_normalized"`
	Created         float32 `json:"created"`
	Modified        float32 `json:"modified"`
	Name            string  `json:"name"`
	ImageURL        string  `json:"image_url"`
	Preferences     struct {
		Onboarding struct {
			App       bool `json:"app"`
			Friends   bool `json:"friends"`
			Extension bool `json:"extension"`
		} `json:"onboarding"`
		Social bool   `json:"social"`
		Cat    string `json:"cat"`
	} `json:"preferences"`
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"

	// Imports so that goinstall automatically installs them.
	_ "code.google.com/p/go-tour/pic"
	_ "code.google.com/p/go-tour/tree"
	_ "code.google.com/p/go-tour/wc"
)

const basePkg = "code.google.com/p/go-tour/"

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)
)

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find and serve the go tour files
	p, err := build.Default.Import(basePkg, "", build.FindOnly)
	if err != nil {
		log.Fatalf("Couldn't find tour files: %v", err)
	}
	root := p.Dir
	log.Println("Serving content from", root)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/favicon.ico" || r.URL.Path == "/" {
			fn := filepath.Join(root, "static", r.URL.Path[1:])
			http.ServeFile(w, r, fn)
			return
		}
		http.Error(w, "not found", 404)
	})
	http.Handle("/static/", http.FileServer(http.Dir(root)))
	http.HandleFunc("/kill", kill)

	if !strings.HasPrefix(*httpListen, "127.0.0.1") &&
		!strings.HasPrefix(*httpListen, "localhost") {
		log.Print(localhostWarning)
	}

	log.Printf("Serving at http://%s/", *httpListen)
	log.Fatal(http.ListenAndServe(*httpListen, nil))
}

const localhostWarning = `
WARNING!  WARNING!  WARNING!

I appear to be listening on an address that is not localhost.
Anyone with access to this address and port will have access
to this machine as the user running gotour.

If you don't understand this message, hit Control-C to terminate this process.

WARNING!  WARNING!  WARNING!
`

var running struct {
	sync.Mutex
	cmd *exec.Cmd
}

func stopRun() {
	running.Lock()
	if running.cmd != nil {
		running.cmd.Process.Kill()
		running.cmd = nil
	}
	running.Unlock()
}

func kill(w http.ResponseWriter, r *http.Request) {
	stopRun()
}

var (
	commentRe = regexp.MustCompile(`(?m)^#.*\n`)
	tmpdir    string
)

func init() {
	// find real temporary directory (for rewriting filename in output)
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

func compile(req *http.Request) (out []byte, err error) {
	stopRun()

	// x is the base name for .go, .6, executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// rewrite filename in error output
	defer func() {
		if err != nil {
			// drop messages from the go tool like '# _/compile0'
			out = commentRe.ReplaceAll(out, nil)
		}
		out = bytes.Replace(out, []byte(src+":"), []byte("main.go:"), -1)
	}()

	// write body to x.go
	body := []byte(req.FormValue("body"))
	defer os.Remove(src)
	if err = ioutil.WriteFile(src, body, 0666); err != nil {
		return
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	out, err = run(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return
	}

	// run x
	return run("", bin)
}

// run executes the specified command and returns its output and an error.
func run(dir string, args ...string) ([]byte, error) {
	var buf bytes.Buffer
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = &buf
	cmd.Stderr = cmd.Stdout

	// Start command and leave in 'running'.
	running.Lock()
	if running.cmd != nil {
		defer running.Unlock()
		return nil, fmt.Errorf("already running %s", running.cmd.Path)
	}
	if err := cmd.Start(); err != nil {
		running.Unlock()
		return nil, err
	}
	running.cmd = cmd
	running.Unlock()

	// Wait for the command.  Clean up,
	err := cmd.Wait()
	running.Lock()
	if running.cmd == cmd {
		running.cmd = nil
	}
	running.Unlock()
	return buf.Bytes(), err
}
<commit_msg>gotour: make the console messages more descriptive<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"

	// Imports so that goinstall automatically installs them.
	_ "code.google.com/p/go-tour/pic"
	_ "code.google.com/p/go-tour/tree"
	_ "code.google.com/p/go-tour/wc"
)

const basePkg = "code.google.com/p/go-tour/"

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)
)

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find and serve the go tour files
	p, err := build.Default.Import(basePkg, "", build.FindOnly)
	if err != nil {
		log.Fatalf("Couldn't find tour files: %v", err)
	}
	root := p.Dir
	log.Println("Serving content from", root)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/favicon.ico" || r.URL.Path == "/" {
			fn := filepath.Join(root, "static", r.URL.Path[1:])
			http.ServeFile(w, r, fn)
			return
		}
		http.Error(w, "not found", 404)
	})
	http.Handle("/static/", http.FileServer(http.Dir(root)))
	http.HandleFunc("/kill", kill)

	if !strings.HasPrefix(*httpListen, "127.0.0.1") &&
		!strings.HasPrefix(*httpListen, "localhost") {
		log.Print(localhostWarning)
	}

	log.Printf("Open your web browser and visit http://%s/", *httpListen)
	log.Fatal(http.ListenAndServe(*httpListen, nil))
}

const localhostWarning = `
WARNING!  WARNING!  WARNING!

I appear to be listening on an address that is not localhost.
Anyone with access to this address and port will have access
to this machine as the user running gotour.

If you don't understand this message, hit Control-C to terminate this process.

WARNING!  WARNING!  WARNING!
`

var running struct {
	sync.Mutex
	cmd *exec.Cmd
}

func stopRun() {
	running.Lock()
	if running.cmd != nil {
		running.cmd.Process.Kill()
		running.cmd = nil
	}
	running.Unlock()
}

func kill(w http.ResponseWriter, r *http.Request) {
	stopRun()
}

var (
	commentRe = regexp.MustCompile(`(?m)^#.*\n`)
	tmpdir    string
)

func init() {
	// find real temporary directory (for rewriting filename in output)
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

func compile(req *http.Request) (out []byte, err error) {
	stopRun()

	// x is the base name for .go, .6, executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// rewrite filename in error output
	defer func() {
		if err != nil {
			// drop messages from the go tool like '# _/compile0'
			out = commentRe.ReplaceAll(out, nil)
		}
		out = bytes.Replace(out, []byte(src+":"), []byte("main.go:"), -1)
	}()

	// write body to x.go
	body := []byte(req.FormValue("body"))
	defer os.Remove(src)
	if err = ioutil.WriteFile(src, body, 0666); err != nil {
		return
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	out, err = run(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return
	}

	// run x
	return run("", bin)
}

// run executes the specified command and returns its output and an error.
func run(dir string, args ...string) ([]byte, error) {
	var buf bytes.Buffer
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = &buf
	cmd.Stderr = cmd.Stdout

	// Start command and leave in 'running'.
	running.Lock()
	if running.cmd != nil {
		defer running.Unlock()
		return nil, fmt.Errorf("already running %s", running.cmd.Path)
	}
	if err := cmd.Start(); err != nil {
		running.Unlock()
		return nil, err
	}
	running.cmd = cmd
	running.Unlock()

	// Wait for the command.  Clean up,
	err := cmd.Wait()
	running.Lock()
	if running.cmd == cmd {
		running.cmd = nil
	}
	running.Unlock()
	return buf.Bytes(), err
}
<|endoftext|>


<commit_before>package server

import (
	"context"

	"github.com/rancher/helm-controller/pkg/generated/controllers/helm.cattle.io"
	"github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/apps"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/batch"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
	"github.com/rancher/wrangler/pkg/apply"
	"github.com/rancher/wrangler/pkg/crd"
	"github.com/rancher/wrangler/pkg/start"
	"k8s.io/client-go/rest"
	"k8s.io/kubernetes/staging/src/k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd"
)

type Context struct {
	K3s   *k3s.Factory
	Helm  *helm.Factory
	Batch *batch.Factory
	Apps  *apps.Factory
	Auth  *rbac.Factory
	Core  *core.Factory
	K8s   kubernetes.Interface
	Apply apply.Apply
}

func (c *Context) Start(ctx context.Context) error {
	return start.All(ctx, 5, c.K3s, c.Helm, c.Apps, c.Auth, c.Batch, c.Core)
}

func newContext(ctx context.Context, cfg string) (*Context, error) {
	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
	if err != nil {
		return nil, err
	}

	if err := crds(ctx, restConfig); err != nil {
		return nil, err
	}

	k8s := kubernetes.NewForConfigOrDie(restConfig)
	return &Context{
		K3s:   k3s.NewFactoryFromConfigOrDie(restConfig),
		Helm:  helm.NewFactoryFromConfigOrDie(restConfig),
		K8s:   k8s,
		Auth:  rbac.NewFactoryFromConfigOrDie(restConfig),
		Apps:  apps.NewFactoryFromConfigOrDie(restConfig),
		Batch: batch.NewFactoryFromConfigOrDie(restConfig),
		Core:  core.NewFactoryFromConfigOrDie(restConfig),
		Apply: apply.New(k8s, apply.NewClientFactory(restConfig)),
	}, nil
}

func crds(ctx context.Context, config *rest.Config) error {
	factory, err := crd.NewFactoryFromClient(config)
	if err != nil {
		return err
	}

	factory.BatchCreateCRDs(ctx, crd.NamespacedTypes(
		"ListenerConfig.k3s.cattle.io/v1",
		"Addon.k3s.cattle.io/v1",
		"HelmChart.helm.cattle.io/v1")...)

	return factory.BatchWait()
}
<commit_msg>Wrong import<commit_after>package server

import (
	"context"

	"github.com/rancher/helm-controller/pkg/generated/controllers/helm.cattle.io"
	"github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/apps"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/batch"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
	"github.com/rancher/wrangler/pkg/apply"
	"github.com/rancher/wrangler/pkg/crd"
	"github.com/rancher/wrangler/pkg/start"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

type Context struct {
	K3s   *k3s.Factory
	Helm  *helm.Factory
	Batch *batch.Factory
	Apps  *apps.Factory
	Auth  *rbac.Factory
	Core  *core.Factory
	K8s   kubernetes.Interface
	Apply apply.Apply
}

func (c *Context) Start(ctx context.Context) error {
	return start.All(ctx, 5, c.K3s, c.Helm, c.Apps, c.Auth, c.Batch, c.Core)
}

func newContext(ctx context.Context, cfg string) (*Context, error) {
	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
	if err != nil {
		return nil, err
	}

	if err := crds(ctx, restConfig); err != nil {
		return nil, err
	}

	k8s := kubernetes.NewForConfigOrDie(restConfig)
	return &Context{
		K3s:   k3s.NewFactoryFromConfigOrDie(restConfig),
		Helm:  helm.NewFactoryFromConfigOrDie(restConfig),
		K8s:   k8s,
		Auth:  rbac.NewFactoryFromConfigOrDie(restConfig),
		Apps:  apps.NewFactoryFromConfigOrDie(restConfig),
		Batch: batch.NewFactoryFromConfigOrDie(restConfig),
		Core:  core.NewFactoryFromConfigOrDie(restConfig),
		Apply: apply.New(k8s, apply.NewClientFactory(restConfig)),
	}, nil
}

func crds(ctx context.Context, config *rest.Config) error {
	factory, err := crd.NewFactoryFromClient(config)
	if err != nil {
		return err
	}

	factory.BatchCreateCRDs(ctx, crd.NamespacedTypes(
		"ListenerConfig.k3s.cattle.io/v1",
		"Addon.k3s.cattle.io/v1",
		"HelmChart.helm.cattle.io/v1")...)

	return factory.BatchWait()
}
<|endoftext|>