<commit_before>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// ensure that an added file shows up in docker diff
func TestDiffFilenameShownInOutput(t *testing.T) {
	containerCmd := `echo foo > /root/bar`
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
	cid, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to start the container: %v", err))

	cleanCID := stripTrailingCharacters(cid)

	diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
	out, _, err := runCommandWithOutput(diffCmd)
	errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))

	found := false
	for _, line := range strings.Split(out, "\n") {
		if strings.Contains("A /root/bar", line) {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("couldn't find the new file in docker diff's output: %v", out)
	}
	go deleteContainer(cleanCID)

	logDone("diff - check if created file shows up")
}

// test to ensure GH #3840 doesn't occur any more
func TestDiffEnsureDockerinitFilesAreIgnored(t *testing.T) {
	// this is a list of files which shouldn't show up in `docker diff`
	dockerinitFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerinit", "/.dockerenv"}

	// we might not run into this problem from the first run, so start a few containers
	for i := 0; i < 20; i++ {
		containerCmd := `echo foo > /root/bar`
		runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
		cid, _, err := runCommandWithOutput(runCmd)
		errorOut(err, t, fmt.Sprintf("%s", err))

		cleanCID := stripTrailingCharacters(cid)

		diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
		out, _, err := runCommandWithOutput(diffCmd)
		errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))

		go deleteContainer(cleanCID)

		for _, filename := range dockerinitFiles {
			if strings.Contains(out, filename) {
				t.Errorf("found file which should've been ignored %v in diff output", filename)
			}
		}
	}

	logDone("diff - check if ignored files show up in diff")
}
<commit_msg>add test<commit_after>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// ensure that an added file shows up in docker diff
func TestDiffFilenameShownInOutput(t *testing.T) {
	containerCmd := `echo foo > /root/bar`
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
	cid, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to start the container: %v", err))

	cleanCID := stripTrailingCharacters(cid)

	diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
	out, _, err := runCommandWithOutput(diffCmd)
	errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))

	found := false
	for _, line := range strings.Split(out, "\n") {
		if strings.Contains("A /root/bar", line) {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("couldn't find the new file in docker diff's output: %v", out)
	}
	go deleteContainer(cleanCID)

	logDone("diff - check if created file shows up")
}

// test to ensure GH #3840 doesn't occur any more
func TestDiffEnsureDockerinitFilesAreIgnored(t *testing.T) {
	// this is a list of files which shouldn't show up in `docker diff`
	dockerinitFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerinit", "/.dockerenv"}

	// we might not run into this problem from the first run, so start a few containers
	for i := 0; i < 20; i++ {
		containerCmd := `echo foo > /root/bar`
		runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
		cid, _, err := runCommandWithOutput(runCmd)
		errorOut(err, t, fmt.Sprintf("%s", err))

		cleanCID := stripTrailingCharacters(cid)

		diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
		out, _, err := runCommandWithOutput(diffCmd)
		errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))

		go deleteContainer(cleanCID)

		for _, filename := range dockerinitFiles {
			if strings.Contains(out, filename) {
				t.Errorf("found file which should've been ignored %v in diff output", filename)
			}
		}
	}

	logDone("diff - check if ignored files show up in diff")
}

func TestDiffEnsureOnlyKmsgAndPtmx(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sleep 0")
	cid, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("%s", err))
	cleanCID := stripTrailingCharacters(cid)

	diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
	out, _, err := runCommandWithOutput(diffCmd)
	errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))
	go deleteContainer(cleanCID)

	expected := map[string]bool{
		"C /dev":      true,
		"A /dev/full": true, // busybox
		"C /dev/ptmx": true, // libcontainer
		"A /dev/kmsg": true, // lxc
	}

	for _, line := range strings.Split(out, "\n") {
		if line != "" && !expected[line] {
			t.Errorf("'%s' is shown in the diff but shouldn't", line)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/sapcc/kubernikus/pkg/api/client/operations"
	etcd_util "github.com/sapcc/kubernikus/pkg/util/etcd"
	"github.com/sapcc/kubernikus/test/e2e/framework"
)

const (
	CleanupBackupContainerDeleteInterval = 1 * time.Second
	CleanupBackupContainerDeleteTimeout  = 1 * time.Minute
)

type PyrolisisTests struct {
	Kubernikus *framework.Kubernikus
	OpenStack  *framework.OpenStack
	Reuse      bool
}

func (p *PyrolisisTests) Run(t *testing.T) {
	if p.Reuse == false {
		quota := t.Run("SettingKlustersOnFire", p.SettingKlustersOnFire)
		require.True(t, quota, "Klusters must burn")

		t.Run("Wait", func(t *testing.T) {
			t.Run("Klusters", p.WaitForE2EKlustersTerminated)
		})

		t.Run("CleanupVolumes", p.CleanupVolumes)
		t.Run("CleanupInstances", p.CleanupInstances)

		cleanupStorageContainer := t.Run("CleanupBackupStorageContainers", p.CleanupBackupStorageContainers)
		require.True(t, cleanupStorageContainer, "Etcd backup storage container cleanup failed")

		t.Run("CleanupLoadbalancers", p.CleanupLoadbalancers)
	}
}

func (p *PyrolisisTests) SettingKlustersOnFire(t *testing.T) {
	res, err := p.Kubernikus.Client.Operations.ListClusters(
		operations.NewListClustersParams(),
		p.Kubernikus.AuthInfo,
	)
	require.NoError(t, err, "There should be no error while listing klusters")

	for _, kluster := range res.Payload {
		if strings.HasPrefix(kluster.Name, "e2e-") {
			t.Run(fmt.Sprintf("TerminatingKluster-%v", kluster.Name), func(t *testing.T) {
				_, err := p.Kubernikus.Client.Operations.TerminateCluster(
					operations.NewTerminateClusterParams().WithName(kluster.Name),
					p.Kubernikus.AuthInfo,
				)
				assert.NoError(t, err, "There should be no error while terminating klusters")
			})
		}
	}
}

func (p *PyrolisisTests) WaitForE2EKlustersTerminated(t *testing.T) {
	err := p.Kubernikus.WaitForKlusters("e2e-", 0, WaitForKlusterToBeDeletedTimeout)
	assert.NoError(t, err, "E2E Klusters didn't burn down in time")
}

func (p *PyrolisisTests) CleanupBackupStorageContainers(t *testing.T) {
	storageClient, err := openstack.NewObjectStorageV1(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create object storage client")

	containersListOpts := containers.ListOpts{
		Full: false,
	}
	allPages, err := containers.List(storageClient, containersListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing storage containers")

	allContainers, err := containers.ExtractNames(allPages)
	require.NoError(t, err, "There should be no error while extracting storage containers")

	objectsListOpts := objects.ListOpts{
		Full: false,
	}
	for _, container := range allContainers {
		if strings.HasPrefix(container, etcd_util.BackupStorageContainerBase) {
			allPages, err := objects.List(storageClient, container, objectsListOpts).AllPages()
			require.NoError(t, err, "There should be no error while lising objetcs in container %s", container)

			allObjects, err := objects.ExtractNames(allPages)
			require.NoError(t, err, "There should be no error while extracting objetcs names for container %s", container)

			for _, object := range allObjects {
				_, err := objects.Delete(storageClient, container, object, objects.DeleteOpts{}).Extract()
				//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
				if _, ok := err.(gophercloud.ErrDefault404); ok {
					continue
				}
				require.NoError(t, err, "There should be no error while deleting object %s/%s", container, object)
			}

			err = wait.PollImmediate(CleanupBackupContainerDeleteInterval, CleanupBackupContainerDeleteTimeout,
				func() (bool, error) {
					_, err := containers.Delete(storageClient, container).Extract()
					if errResponseCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok && errResponseCode.Actual == 409 {
						return false, nil
					}
					//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
					if _, ok := err.(gophercloud.ErrDefault404); ok {
						return true, nil
					}
					return true, err
				})
			require.NoError(t, err, "There should be no error while deleting storage container: %s", container)
		}
	}
}

func (p *PyrolisisTests) CleanupVolumes(t *testing.T) {
	storageClient, err := openstack.NewBlockStorageV3(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create block storage client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	volumeListOpts := volumes.ListOpts{
		TenantID: project.ID,
	}

	allPages, err := volumes.List(storageClient, volumeListOpts).AllPages()
	require.NoError(t, err, "There should be no error while retrieving volume pages")

	allVolumes, err := volumes.ExtractVolumes(allPages)
	require.NoError(t, err, "There should be no error while extracting volumes")

	for _, vol := range allVolumes {
		// in-tree
		if strings.HasPrefix(vol.Name, "kubernetes-dynamic-pvc-") &&
			strings.HasPrefix(vol.Metadata["kubernetes.io/created-for/pvc/namespace"], "e2e-volumes-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
		// CSI
		if strings.HasPrefix(vol.Name, "pv-e2e-") &&
			strings.HasPrefix(vol.Metadata["cinder.csi.openstack.org/cluster"], "e2e-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
	}
}

func (p *PyrolisisTests) CleanupInstances(t *testing.T) {
	computeClient, err := openstack.NewComputeV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error creating compute client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	serversListOpts := servers.ListOpts{
		Name:     "e2e-",
		TenantID: project.ID,
	}

	allPages, err := servers.List(computeClient, serversListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing all servers")

	allServers, err := servers.ExtractServers(allPages)
	require.NoError(t, err, "There should be no error while extracting all servers")

	for _, srv := range allServers {
		err := servers.Delete(computeClient, srv.ID).ExtractErr()
		require.NoError(t, err, "There should be no error while deleting server %s (%s)", srv.Name, srv.ID)
	}
}

func (p *PyrolisisTests) CleanupLoadbalancers(t *testing.T) {
	lbClient, err := openstack.NewLoadBalancerV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error getting a loadbalancer client")

	allPages, err := loadbalancers.List(lbClient, loadbalancers.ListOpts{}).AllPages()
	require.NoError(t, err, "There should be no error while listing loadbalancers")

	allLoadbalancers, err := loadbalancers.ExtractLoadBalancers(allPages)
	require.NoError(t, err, "There should be no error while extracting loadbalancers")

	for _, lb := range allLoadbalancers {
		if strings.HasSuffix(lb.Name, "_e2e-lb") {
			err := loadbalancers.Delete(lbClient, lb.ID, loadbalancers.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting loadbalancer %s", lb.Name)
		}
	}
}
<commit_msg>pyrolisis: rescursivly delete loadbalancer<commit_after>package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/sapcc/kubernikus/pkg/api/client/operations"
	etcd_util "github.com/sapcc/kubernikus/pkg/util/etcd"
	"github.com/sapcc/kubernikus/test/e2e/framework"
)

const (
	CleanupBackupContainerDeleteInterval = 1 * time.Second
	CleanupBackupContainerDeleteTimeout  = 1 * time.Minute
)

type PyrolisisTests struct {
	Kubernikus *framework.Kubernikus
	OpenStack  *framework.OpenStack
	Reuse      bool
}

func (p *PyrolisisTests) Run(t *testing.T) {
	if p.Reuse == false {
		quota := t.Run("SettingKlustersOnFire", p.SettingKlustersOnFire)
		require.True(t, quota, "Klusters must burn")

		t.Run("Wait", func(t *testing.T) {
			t.Run("Klusters", p.WaitForE2EKlustersTerminated)
		})

		t.Run("CleanupVolumes", p.CleanupVolumes)
		t.Run("CleanupInstances", p.CleanupInstances)

		cleanupStorageContainer := t.Run("CleanupBackupStorageContainers", p.CleanupBackupStorageContainers)
		require.True(t, cleanupStorageContainer, "Etcd backup storage container cleanup failed")

		t.Run("CleanupLoadbalancers", p.CleanupLoadbalancers)
	}
}

func (p *PyrolisisTests) SettingKlustersOnFire(t *testing.T) {
	res, err := p.Kubernikus.Client.Operations.ListClusters(
		operations.NewListClustersParams(),
		p.Kubernikus.AuthInfo,
	)
	require.NoError(t, err, "There should be no error while listing klusters")

	for _, kluster := range res.Payload {
		if strings.HasPrefix(kluster.Name, "e2e-") {
			t.Run(fmt.Sprintf("TerminatingKluster-%v", kluster.Name), func(t *testing.T) {
				_, err := p.Kubernikus.Client.Operations.TerminateCluster(
					operations.NewTerminateClusterParams().WithName(kluster.Name),
					p.Kubernikus.AuthInfo,
				)
				assert.NoError(t, err, "There should be no error while terminating klusters")
			})
		}
	}
}

func (p *PyrolisisTests) WaitForE2EKlustersTerminated(t *testing.T) {
	err := p.Kubernikus.WaitForKlusters("e2e-", 0, WaitForKlusterToBeDeletedTimeout)
	assert.NoError(t, err, "E2E Klusters didn't burn down in time")
}

func (p *PyrolisisTests) CleanupBackupStorageContainers(t *testing.T) {
	storageClient, err := openstack.NewObjectStorageV1(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create object storage client")

	containersListOpts := containers.ListOpts{
		Full: false,
	}
	allPages, err := containers.List(storageClient, containersListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing storage containers")

	allContainers, err := containers.ExtractNames(allPages)
	require.NoError(t, err, "There should be no error while extracting storage containers")

	objectsListOpts := objects.ListOpts{
		Full: false,
	}
	for _, container := range allContainers {
		if strings.HasPrefix(container, etcd_util.BackupStorageContainerBase) {
			allPages, err := objects.List(storageClient, container, objectsListOpts).AllPages()
			require.NoError(t, err, "There should be no error while lising objetcs in container %s", container)

			allObjects, err := objects.ExtractNames(allPages)
			require.NoError(t, err, "There should be no error while extracting objetcs names for container %s", container)

			for _, object := range allObjects {
				_, err := objects.Delete(storageClient, container, object, objects.DeleteOpts{}).Extract()
				//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
				if _, ok := err.(gophercloud.ErrDefault404); ok {
					continue
				}
				require.NoError(t, err, "There should be no error while deleting object %s/%s", container, object)
			}

			err = wait.PollImmediate(CleanupBackupContainerDeleteInterval, CleanupBackupContainerDeleteTimeout,
				func() (bool, error) {
					_, err := containers.Delete(storageClient, container).Extract()
					if errResponseCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok && errResponseCode.Actual == 409 {
						return false, nil
					}
					//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
					if _, ok := err.(gophercloud.ErrDefault404); ok {
						return true, nil
					}
					return true, err
				})
			require.NoError(t, err, "There should be no error while deleting storage container: %s", container)
		}
	}
}

func (p *PyrolisisTests) CleanupVolumes(t *testing.T) {
	storageClient, err := openstack.NewBlockStorageV3(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create block storage client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	volumeListOpts := volumes.ListOpts{
		TenantID: project.ID,
	}

	allPages, err := volumes.List(storageClient, volumeListOpts).AllPages()
	require.NoError(t, err, "There should be no error while retrieving volume pages")

	allVolumes, err := volumes.ExtractVolumes(allPages)
	require.NoError(t, err, "There should be no error while extracting volumes")

	for _, vol := range allVolumes {
		// in-tree
		if strings.HasPrefix(vol.Name, "kubernetes-dynamic-pvc-") &&
			strings.HasPrefix(vol.Metadata["kubernetes.io/created-for/pvc/namespace"], "e2e-volumes-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
		// CSI
		if strings.HasPrefix(vol.Name, "pv-e2e-") &&
			strings.HasPrefix(vol.Metadata["cinder.csi.openstack.org/cluster"], "e2e-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
	}
}

func (p *PyrolisisTests) CleanupInstances(t *testing.T) {
	computeClient, err := openstack.NewComputeV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error creating compute client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	serversListOpts := servers.ListOpts{
		Name:     "e2e-",
		TenantID: project.ID,
	}

	allPages, err := servers.List(computeClient, serversListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing all servers")

	allServers, err := servers.ExtractServers(allPages)
	require.NoError(t, err, "There should be no error while extracting all servers")

	for _, srv := range allServers {
		err := servers.Delete(computeClient, srv.ID).ExtractErr()
		require.NoError(t, err, "There should be no error while deleting server %s (%s)", srv.Name, srv.ID)
	}
}

func (p *PyrolisisTests) CleanupLoadbalancers(t *testing.T) {
	lbClient, err := openstack.NewLoadBalancerV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error getting a loadbalancer client")

	allPages, err := loadbalancers.List(lbClient, loadbalancers.ListOpts{}).AllPages()
	require.NoError(t, err, "There should be no error while listing loadbalancers")

	allLoadbalancers, err := loadbalancers.ExtractLoadBalancers(allPages)
	require.NoError(t, err, "There should be no error while extracting loadbalancers")

	for _, lb := range allLoadbalancers {
		if strings.HasSuffix(lb.Name, "_e2e-lb") {
			err := loadbalancers.Delete(lbClient, lb.ID, loadbalancers.DeleteOpts{Cascade: true}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting loadbalancer %s", lb.Name)
		}
	}
}
<|endoftext|>


<commit_before>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2017
	All Rights Reserved
    Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package gopi // import "github.com/djthorpe/gopi"

import (
	"fmt"
	"os"
	"path"

	"github.com/djthorpe/gopi/util"
)

////////////////////////////////////////////////////////////////////////////////
// TYPES

// ModuleType defines the type of module
type ModuleType uint

// Module is a structure which determines details about a module
type Module struct {
	Name string
	Type ModuleType
	New  ModuleNewFunc
}

// ModuleNewFunc is the signature for creating a new module instance
type ModuleNewFunc func(*AppConfig) (Driver, error)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	MODULE_TYPE_NONE ModuleType = iota
	MODULE_TYPE_OTHER
	MODULE_TYPE_LOGGER
	MODULE_TYPE_HARDWARE
	MODULE_TYPE_DISPLAY
	MODULE_TYPE_BITMAP
	MODULE_TYPE_VECTOR
	MODULE_TYPE_VGFONT
	MODULE_TYPE_OPENGL
	MODULE_TYPE_GPIO
	MODULE_TYPE_I2C
	MODULE_TYPE_SPI
	MODULE_TYPE_INPUT
)

////////////////////////////////////////////////////////////////////////////////
// GLOBAL VARIABLES

var (
	flags           = util.NewFlags(path.Base(os.Args[0]))
	modules_by_name = make(map[string]Module)
	modules_by_type = make(map[ModuleType]Module)
)

////////////////////////////////////////////////////////////////////////////////
// INIT METHOD

func init() {
	flags.FlagBool("debug", false, "Set debugging mode")
}

////////////////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

func RegisterModule(module Module) *util.Flags {
	// Register by name
	if _, exists := modules_by_name[module.Name]; exists {
		fmt.Fprintln(os.Stderr, "Duplicate Module registered:", &module)
		os.Exit(-1)
	} else {
		modules_by_name[module.Name] = module
	}
	// Register by type if module type is not None or Other
	if module.Type != MODULE_TYPE_OTHER && module.Type != MODULE_TYPE_NONE {
		if _, exists := modules_by_type[module.Type]; exists {
			fmt.Fprintln(os.Stderr, "Duplicate Module registered:", &module)
			os.Exit(-1)
		} else {
			modules_by_type[module.Type] = module
		}
	}
	// Return flags to add configuration options onto
	return flags
}

func ModuleByType(t ModuleType) (*Module, error) {
	if module, exists := modules_by_type[t]; exists {
		return &module, nil
	} else {
		return nil, ErrModuleNotFound
	}
}

func ModuleByName(n string) (*Module, error) {
	if module, exists := modules_by_name[n]; exists {
		return &module, nil
	} else {
		return nil, ErrModuleNotFound
	}
}

////////////////////////////////////////////////////////////////////////////////
// STRINGIFY

func (t ModuleType) String() string {
	switch t {
	case MODULE_TYPE_NONE:
		return "MODULE_TYPE_NONE"
	case MODULE_TYPE_OTHER:
		return "MODULE_TYPE_OTHER"
	case MODULE_TYPE_LOGGER:
		return "MODULE_TYPE_LOGGER"
	case MODULE_TYPE_HARDWARE:
		return "MODULE_TYPE_HARDWARE"
	case MODULE_TYPE_DISPLAY:
		return "MODULE_TYPE_DISPLAY"
	case MODULE_TYPE_BITMAP:
		return "MODULE_TYPE_BITMAP"
	case MODULE_TYPE_VECTOR:
		return "MODULE_TYPE_VECTOR"
	case MODULE_TYPE_VGFONT:
		return "MODULE_TYPE_VGFONT"
	case MODULE_TYPE_OPENGL:
		return "MODULE_TYPE_OPENGL"
	case MODULE_TYPE_GPIO:
		return "MODULE_TYPE_GPIO"
	case MODULE_TYPE_I2C:
		return "MODULE_TYPE_I2C"
	case MODULE_TYPE_SPI:
		return "MODULE_TYPE_SPI"
	case MODULE_TYPE_INPUT:
		return "MODULE_TYPE_INPUT"
	default:
		return "[Invalid ModuleType value]"
	}
}

func (this *Module) String() string {
	return fmt.Sprintf("gopi.Module{ name=%v type=%v }", this.Name, this.Type)
}
<commit_msg>Updated modules for layout<commit_after>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2017
	All Rights Reserved
    Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package gopi // import "github.com/djthorpe/gopi"

import (
	"fmt"
	"os"
	"path"

	"github.com/djthorpe/gopi/util"
)

////////////////////////////////////////////////////////////////////////////////
// TYPES

// ModuleType defines the type of module
type ModuleType uint

// Module is a structure which determines details about a module
type Module struct {
	Name string
	Type ModuleType
	New  ModuleNewFunc
}

// ModuleNewFunc is the signature for creating a new module instance
type ModuleNewFunc func(*AppConfig) (Driver, error)

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	MODULE_TYPE_NONE ModuleType = iota
	MODULE_TYPE_OTHER
	MODULE_TYPE_LOGGER
	MODULE_TYPE_HARDWARE
	MODULE_TYPE_DISPLAY
	MODULE_TYPE_BITMAP
	MODULE_TYPE_VECTOR
	MODULE_TYPE_VGFONT
	MODULE_TYPE_OPENGL
	MODULE_TYPE_LAYOUT
	MODULE_TYPE_GPIO
	MODULE_TYPE_I2C
	MODULE_TYPE_SPI
	MODULE_TYPE_INPUT
)

////////////////////////////////////////////////////////////////////////////////
// GLOBAL VARIABLES

var (
	flags           = util.NewFlags(path.Base(os.Args[0]))
	modules_by_name = make(map[string]Module)
	modules_by_type = make(map[ModuleType]Module)
)

////////////////////////////////////////////////////////////////////////////////
// INIT METHOD

func init() {
	flags.FlagBool("debug", false, "Set debugging mode")
}

////////////////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

func RegisterModule(module Module) *util.Flags {
	// Register by name
	if _, exists := modules_by_name[module.Name]; exists {
		fmt.Fprintln(os.Stderr, "Duplicate Module registered:", &module)
		os.Exit(-1)
	} else {
		modules_by_name[module.Name] = module
	}
	// Register by type if module type is not None or Other
	if module.Type != MODULE_TYPE_OTHER && module.Type != MODULE_TYPE_NONE {
		if _, exists := modules_by_type[module.Type]; exists {
			fmt.Fprintln(os.Stderr, "Duplicate Module registered:", &module)
			os.Exit(-1)
		} else {
			modules_by_type[module.Type] = module
		}
	}
	// Return flags to add configuration options onto
	return flags
}

func ModuleByType(t ModuleType) (*Module, error) {
	if module, exists := modules_by_type[t]; exists {
		return &module, nil
	} else {
		return nil, ErrModuleNotFound
	}
}

func ModuleByName(n string) (*Module, error) {
	if module, exists := modules_by_name[n]; exists {
		return &module, nil
	} else {
		return nil, ErrModuleNotFound
	}
}

////////////////////////////////////////////////////////////////////////////////
// STRINGIFY

func (t ModuleType) String() string {
	switch t {
	case MODULE_TYPE_NONE:
		return "MODULE_TYPE_NONE"
	case MODULE_TYPE_OTHER:
		return "MODULE_TYPE_OTHER"
	case MODULE_TYPE_LOGGER:
		return "MODULE_TYPE_LOGGER"
	case MODULE_TYPE_HARDWARE:
		return "MODULE_TYPE_HARDWARE"
	case MODULE_TYPE_DISPLAY:
		return "MODULE_TYPE_DISPLAY"
	case MODULE_TYPE_BITMAP:
		return "MODULE_TYPE_BITMAP"
	case MODULE_TYPE_VECTOR:
		return "MODULE_TYPE_VECTOR"
	case MODULE_TYPE_VGFONT:
		return "MODULE_TYPE_VGFONT"
	case MODULE_TYPE_OPENGL:
		return "MODULE_TYPE_OPENGL"
	case MODULE_TYPE_LAYOUT:
		return "MODULE_TYPE_LAYOUT"
	case MODULE_TYPE_GPIO:
		return "MODULE_TYPE_GPIO"
	case MODULE_TYPE_I2C:
		return "MODULE_TYPE_I2C"
	case MODULE_TYPE_SPI:
		return "MODULE_TYPE_SPI"
	case MODULE_TYPE_INPUT:
		return "MODULE_TYPE_INPUT"
	default:
		return "[Invalid ModuleType value]"
	}
}

func (this *Module) String() string {
	return fmt.Sprintf("gopi.Module{ name=%v type=%v }", this.Name, this.Type)
}
<|endoftext|>


<commit_before>package terse

import (
	"html/template"
	"text/template/parse"
)

func compile(name string, funcs template.FuncMap, tt tokenTree) (map[string]*parse.Tree, error) {
	if tt.err != nil {
		return map[string]*parse.Tree{}, tt.err
	}

	var nodes []parse.Node
	if len(tt.roots) > 0 {
		nodes = tt.roots[0].Compile("")
		for _, rn := range tt.roots[1:] {
			nodes = append(nodes, rn.Compile("\n")...)
		}
	}

	pt := &parse.Tree{
		Name: name,
		Root: &parse.ListNode{
			NodeType: parse.NodeList,
			Pos:      parse.Pos(0),
			Nodes:    nodes,
		},
	}

	return map[string]*parse.Tree{name: pt}, nil
}
<commit_msg>Remove special casing for first root node<commit_after>package terse

import (
	"html/template"
	"text/template/parse"
)

func compile(name string, funcs template.FuncMap, tt tokenTree) (map[string]*parse.Tree, error) {
	if tt.err != nil {
		return map[string]*parse.Tree{}, tt.err
	}

	var nodes []parse.Node
	var prefix string
	if len(tt.roots) > 0 {
		for _, rn := range tt.roots {
			nodes = append(nodes, rn.Compile(prefix)...)
			prefix = "\n"
		}
	}

	pt := &parse.Tree{
		Name: name,
		Root: &parse.ListNode{
			NodeType: parse.NodeList,
			Pos:      parse.Pos(0),
			Nodes:    nodes,
		},
	}

	return map[string]*parse.Tree{name: pt}, nil
}
<|endoftext|>


<commit_before>package self_test

import (
	"crypto/tls"
	"fmt"
	"net"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/testdata"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type versioner interface {
	GetVersion() protocol.VersionNumber
}

var _ = Describe("Handshake tests", func() {
	var (
		server        quic.Listener
		serverConfig  *quic.Config
		acceptStopped chan struct{}
	)

	BeforeEach(func() {
		server = nil
		acceptStopped = make(chan struct{})
		serverConfig = &quic.Config{}
	})

	AfterEach(func() {
		if server != nil {
			server.Close()
			<-acceptStopped
		}
	})

	runServer := func() {
		var err error
		// start the server
		server, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), serverConfig)
		Expect(err).ToNot(HaveOccurred())

		go func() {
			defer GinkgoRecover()
			defer close(acceptStopped)
			for {
				_, err := server.Accept()
				if err != nil {
					return
				}
			}
		}()
	}

	Context("Version Negotiation", func() {
		var supportedVersions []protocol.VersionNumber

		BeforeEach(func() {
			supportedVersions = protocol.SupportedVersions
			protocol.SupportedVersions = append(protocol.SupportedVersions, []protocol.VersionNumber{7, 8, 9, 10}...)
		})

		AfterEach(func() {
			protocol.SupportedVersions = supportedVersions
		})

		It("when the server supports more versions than the client", func() {
			// the server doesn't support the highest supported version, which is the first one the client will try
			// but it supports a bunch of versions that the client doesn't speak
			serverConfig.Versions = []protocol.VersionNumber{7, 8, protocol.SupportedVersions[0], 9}
			runServer()
			sess, err := quic.DialAddr(server.Addr().String(), &tls.Config{InsecureSkipVerify: true}, nil)
			Expect(err).ToNot(HaveOccurred())
			Expect(sess.(versioner).GetVersion()).To(Equal(protocol.SupportedVersions[0]))
		})

		It("when the client supports more versions than the server supports", func() {
			// the server doesn't support the highest supported version, which is the first one the client will try
			// but it supports a bunch of versions that the client doesn't speak
			serverConfig.Versions = supportedVersions
			runServer()
			conf := &quic.Config{
				Versions: []protocol.VersionNumber{7, 8, 9, protocol.SupportedVersions[0], 10},
			}
			sess, err := quic.DialAddr(server.Addr().String(), &tls.Config{InsecureSkipVerify: true}, conf)
			Expect(err).ToNot(HaveOccurred())
			Expect(sess.(versioner).GetVersion()).To(Equal(protocol.SupportedVersions[0]))
		})
	})

	Context("Certifiate validation", func() {
		for _, v := range protocol.SupportedVersions {
			version := v

			Context(fmt.Sprintf("using %s", version), func() {
				var clientConfig *quic.Config

				BeforeEach(func() {
					serverConfig.Versions = []protocol.VersionNumber{version}
					clientConfig = &quic.Config{
						Versions: []protocol.VersionNumber{version},
					}
				})

				It("accepts the certificate", func() {
					runServer()
					_, err := quic.DialAddr(fmt.Sprintf("quic.clemente.io:%d", server.Addr().(*net.UDPAddr).Port), nil, clientConfig)
					Expect(err).ToNot(HaveOccurred())
				})

				It("errors if the server name doesn't match", func() {
					runServer()
					_, err := quic.DialAddr(fmt.Sprintf("127.0.0.1:%d", server.Addr().(*net.UDPAddr).Port), nil, clientConfig)
					Expect(err).To(HaveOccurred())
				})

				It("uses the ServerName in the tls.Config", func() {
					runServer()
					conf := &tls.Config{ServerName: "quic.clemente.io"}
					_, err := quic.DialAddr(fmt.Sprintf("127.0.0.1:%d", server.Addr().(*net.UDPAddr).Port), conf, clientConfig)
					Expect(err).ToNot(HaveOccurred())
				})
			})
		}
	})
})
<commit_msg>close the server in the version negotiation integration tests<commit_after>package self_test

import (
	"crypto/tls"
	"fmt"
	"net"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/testdata"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type versioner interface {
	GetVersion() protocol.VersionNumber
}

var _ = Describe("Handshake tests", func() {
	var (
		server        quic.Listener
		serverConfig  *quic.Config
		acceptStopped chan struct{}
	)

	BeforeEach(func() {
		server = nil
		acceptStopped = make(chan struct{})
		serverConfig = &quic.Config{}
	})

	AfterEach(func() {
		if server != nil {
			server.Close()
			<-acceptStopped
		}
	})

	runServer := func() quic.Listener {
		var err error
		// start the server
		server, err = quic.ListenAddr("localhost:0", testdata.GetTLSConfig(), serverConfig)
		Expect(err).ToNot(HaveOccurred())

		go func() {
			defer GinkgoRecover()
			defer close(acceptStopped)
			for {
				if _, err := server.Accept(); err != nil {
					return
				}
			}
		}()
		return server
	}

	Context("Version Negotiation", func() {
		var supportedVersions []protocol.VersionNumber

		BeforeEach(func() {
			supportedVersions = protocol.SupportedVersions
			protocol.SupportedVersions = append(protocol.SupportedVersions, []protocol.VersionNumber{7, 8, 9, 10}...)
		})

		AfterEach(func() {
			protocol.SupportedVersions = supportedVersions
		})

		It("when the server supports more versions than the client", func() {
			// the server doesn't support the highest supported version, which is the first one the client will try
			// but it supports a bunch of versions that the client doesn't speak
			serverConfig.Versions = []protocol.VersionNumber{7, 8, protocol.SupportedVersions[0], 9}
			server := runServer()
			defer server.Close()
			sess, err := quic.DialAddr(server.Addr().String(), &tls.Config{InsecureSkipVerify: true}, nil)
			Expect(err).ToNot(HaveOccurred())
			Expect(sess.(versioner).GetVersion()).To(Equal(protocol.SupportedVersions[0]))
		})

		It("when the client supports more versions than the server supports", func() {
			// the server doesn't support the highest supported version, which is the first one the client will try
			// but it supports a bunch of versions that the client doesn't speak
			serverConfig.Versions = supportedVersions
			server := runServer()
			defer server.Close()
			conf := &quic.Config{
				Versions: []protocol.VersionNumber{7, 8, 9, protocol.SupportedVersions[0], 10},
			}
			sess, err := quic.DialAddr(server.Addr().String(), &tls.Config{InsecureSkipVerify: true}, conf)
			Expect(err).ToNot(HaveOccurred())
			Expect(sess.(versioner).GetVersion()).To(Equal(protocol.SupportedVersions[0]))
		})
	})

	Context("Certifiate validation", func() {
		for _, v := range protocol.SupportedVersions {
			version := v

			Context(fmt.Sprintf("using %s", version), func() {
				var clientConfig *quic.Config

				BeforeEach(func() {
					serverConfig.Versions = []protocol.VersionNumber{version}
					clientConfig = &quic.Config{
						Versions: []protocol.VersionNumber{version},
					}
				})

				It("accepts the certificate", func() {
					runServer()
					_, err := quic.DialAddr(fmt.Sprintf("quic.clemente.io:%d", server.Addr().(*net.UDPAddr).Port), nil, clientConfig)
					Expect(err).ToNot(HaveOccurred())
				})

				It("errors if the server name doesn't match", func() {
					runServer()
					_, err := quic.DialAddr(fmt.Sprintf("127.0.0.1:%d", server.Addr().(*net.UDPAddr).Port), nil, clientConfig)
					Expect(err).To(HaveOccurred())
				})

				It("uses the ServerName in the tls.Config", func() {
					runServer()
					conf := &tls.Config{ServerName: "quic.clemente.io"}
					_, err := quic.DialAddr(fmt.Sprintf("127.0.0.1:%d", server.Addr().(*net.UDPAddr).Port), conf, clientConfig)
					Expect(err).ToNot(HaveOccurred())
				})
			})
		}
	})
})
<|endoftext|>


<commit_before>package fusekernel_test

import (
	"os"
	"testing"

	fuse "github.com/jacobsa/bazilfuse"
)

func TestOpenFlagsAccmodeMask(t *testing.T) {
	var f = fuse.OpenFlags(os.O_RDWR | os.O_SYNC)
	if g, e := f&fuse.OpenAccessModeMask, fuse.OpenReadWrite; g != e {
		t.Fatalf("OpenAccessModeMask behaves wrong: %v: %o != %o", f, g, e)
	}
	if f.IsReadOnly() {
		t.Fatalf("IsReadOnly is wrong: %v", f)
	}
	if f.IsWriteOnly() {
		t.Fatalf("IsWriteOnly is wrong: %v", f)
	}
	if !f.IsReadWrite() {
		t.Fatalf("IsReadWrite is wrong: %v", f)
	}
}

func TestOpenFlagsString(t *testing.T) {
	var f = fuse.OpenFlags(os.O_RDWR | os.O_SYNC | os.O_APPEND)
	if g, e := f.String(), "OpenReadWrite+OpenAppend+OpenSync"; g != e {
		t.Fatalf("OpenFlags.String: %q != %q", g, e)
	}
}
<commit_msg>Deleted a test that references the old package.<commit_after><|endoftext|>


<commit_before>package pkglib

// Thin wrappers around git CLI invocations

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	log "github.com/sirupsen/logrus"
)

// 040000 tree 7804129bd06218b72c298139a25698a748d253c6\tpkg/init
var treeHashRe *regexp.Regexp

func init() {
	treeHashRe = regexp.MustCompile("^[0-7]{6} [^ ]+ ([0-9a-f]{40})\t.+\n$")
}

type git struct {
	dir string
}

// Returns git==nil and no error if the path is not within a git repository
func newGit(dir string) (*git, error) {
	g := &git{dir}

	// Check if dir really is within a git directory
	ok, err := g.isWorkTree(dir)
	if err != nil {
		return nil, err
	}
	if !ok {
		return nil, nil
	}
	return g, nil
}

func (g git) mkCmd(args ...string) *exec.Cmd {
	return exec.Command("git", append([]string{"-C", g.dir}, args...)...)
}

func (g git) commandStdout(stderr io.Writer, args ...string) (string, error) {
	cmd := g.mkCmd(args...)
	cmd.Stderr = stderr
	log.Debugf("Executing: %v", cmd.Args)

	out, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func (g git) command(args ...string) error {
	cmd := g.mkCmd(args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	log.Debugf("Executing: %v", cmd.Args)

	return cmd.Run()
}

func (g git) isWorkTree(pkg string) (bool, error) {
	tf, err := g.commandStdout(nil, "rev-parse", "--is-inside-work-tree")
	if err != nil {
		// If we executed git ok but it errored then that's because this isn't a git repo
		if _, ok := err.(*exec.ExitError); ok {
			return false, nil
		}
		return false, err
	}

	tf = strings.TrimSpace(tf)

	if tf == "true" {
		return true, nil
	}

	return false, fmt.Errorf("unexpected output from git rev-parse --is-inside-work-tree: %s", tf)
}

func (g git) contentHash() (string, error) {
	hash := sha256.New()
	out, err := g.commandStdout(nil, "ls-files")
	if err != nil {
		return "", err
	}
	scanner := bufio.NewScanner(strings.NewReader(strings.TrimSpace(out)))
	for scanner.Scan() {
		filename := filepath.Join(g.dir, scanner.Text())
		info, err := os.Lstat(filename)
		if err != nil {
			log.Debugf("cannot stat %s: %s, skipped", filename, err)
			continue
		}
		if info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
			// we do not want to calculate hash of directory or symlinks
			continue
		}
		f, err := os.Open(filename)
		if err != nil {
			log.Debugf("cannot open %s: %s, skipped", filename, err)
			continue
		}
		if _, err := io.Copy(hash, f); err != nil {
			_ = f.Close()
			return "", err
		}
		if err = f.Close(); err != nil {
			return "", err
		}
	}
	if err = scanner.Err(); err != nil {
		return "", err
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

func (g git) treeHash(pkg, commit string) (string, error) {
	// we have to check if pkg is at the top level of the git tree,
	// if that's the case we need to use tree hash from the commit itself
	out, err := g.commandStdout(nil, "rev-parse", "--prefix", pkg, "--show-toplevel")
	if err != nil {
		return "", err
	}
	if strings.TrimSpace(out) == pkg {
		out, err = g.commandStdout(nil, "show", "--format=%T", "-s", commit)
		if err != nil {
			return "", err
		}
		return strings.TrimSpace(out), nil
	}

	out, err = g.commandStdout(os.Stderr, "ls-tree", "--full-tree", commit, "--", pkg)
	if err != nil {
		return "", err
	}

	if out == "" {
		return "", fmt.Errorf("Package %s is not in git", pkg)
	}

	matches := treeHashRe.FindStringSubmatch(out)
	if len(matches) != 2 {
		return "", fmt.Errorf("Unable to parse ls-tree output: %q", out)
	}

	return matches[1], nil
}

func (g git) commitHash(commit string) (string, error) {
	out, err := g.commandStdout(os.Stderr, "rev-parse", commit)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(out), nil
}

func (g git) commitTag(commit string) (string, error) {
	out, err := g.commandStdout(os.Stderr, "tag", "-l", "--points-at", commit)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(out), nil
}

func (g git) isDirty(pkg, commit string) (bool, error) {
	// If it isn't HEAD it can't be dirty
	if commit != "HEAD" {
		return false, nil
	}

	// Update cache, otherwise files which have an updated
	// timestamp but no actual changes are marked as changes
	// because `git diff-index` only uses the `lstat` result and
	// not the actual file contents. Running `git update-index
	// --refresh` updates the cache.
	if err := g.command("update-index", "-q", "--refresh"); err != nil {
		return false, err
	}

	err := g.command("diff-index", "--quiet", commit, "--", pkg)
	if err == nil {
		return false, nil
	}
	switch err.(type) {
	case *exec.ExitError:
		// diff-index exits with an error if there are differences
		return true, nil
	default:
		return false, err
	}
}
<commit_msg>catch new untracked and unignored files (#3875)<commit_after>package pkglib

// Thin wrappers around git CLI invocations

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	log "github.com/sirupsen/logrus"
)

// 040000 tree 7804129bd06218b72c298139a25698a748d253c6\tpkg/init
var treeHashRe *regexp.Regexp

func init() {
	treeHashRe = regexp.MustCompile("^[0-7]{6} [^ ]+ ([0-9a-f]{40})\t.+\n$")
}

type git struct {
	dir string
}

// Returns git==nil and no error if the path is not within a git repository
func newGit(dir string) (*git, error) {
	g := &git{dir}

	// Check if dir really is within a git directory
	ok, err := g.isWorkTree(dir)
	if err != nil {
		return nil, err
	}
	if !ok {
		return nil, nil
	}
	return g, nil
}

func (g git) mkCmd(args ...string) *exec.Cmd {
	return exec.Command("git", append([]string{"-C", g.dir}, args...)...)
}

func (g git) commandStdout(stderr io.Writer, args ...string) (string, error) {
	cmd := g.mkCmd(args...)
	cmd.Stderr = stderr
	log.Debugf("Executing: %v", cmd.Args)

	out, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func (g git) command(args ...string) error {
	cmd := g.mkCmd(args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	log.Debugf("Executing: %v", cmd.Args)

	return cmd.Run()
}

func (g git) isWorkTree(pkg string) (bool, error) {
	tf, err := g.commandStdout(nil, "rev-parse", "--is-inside-work-tree")
	if err != nil {
		// If we executed git ok but it errored then that's because this isn't a git repo
		if _, ok := err.(*exec.ExitError); ok {
			return false, nil
		}
		return false, err
	}

	tf = strings.TrimSpace(tf)

	if tf == "true" {
		return true, nil
	}

	return false, fmt.Errorf("unexpected output from git rev-parse --is-inside-work-tree: %s", tf)
}

func (g git) contentHash() (string, error) {
	hash := sha256.New()
	// list of files tracked by git that might have changed
	trackedFiles, err := g.commandStdout(nil, "ls-files")
	if err != nil {
		return "", err
	}
	untrackedFiles, err := g.commandStdout(nil, "ls-files", "--exclude-standard", "--others")
	if err != nil {
		return "", err
	}
	allFiles := strings.Join([]string{trackedFiles, untrackedFiles}, "\n")
	scanner := bufio.NewScanner(strings.NewReader(strings.TrimSpace(allFiles)))
	for scanner.Scan() {
		filename := filepath.Join(g.dir, scanner.Text())
		info, err := os.Lstat(filename)
		if err != nil {
			log.Debugf("cannot stat %s: %s, skipped", filename, err)
			continue
		}
		if info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
			// we do not want to calculate hash of directory or symlinks
			continue
		}
		f, err := os.Open(filename)
		if err != nil {
			log.Debugf("cannot open %s: %s, skipped", filename, err)
			continue
		}
		if _, err := io.Copy(hash, f); err != nil {
			_ = f.Close()
			return "", err
		}
		if err = f.Close(); err != nil {
			return "", err
		}
	}
	if err = scanner.Err(); err != nil {
		return "", err
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

func (g git) treeHash(pkg, commit string) (string, error) {
	// we have to check if pkg is at the top level of the git tree,
	// if that's the case we need to use tree hash from the commit itself
	out, err := g.commandStdout(nil, "rev-parse", "--prefix", pkg, "--show-toplevel")
	if err != nil {
		return "", err
	}
	if strings.TrimSpace(out) == pkg {
		out, err = g.commandStdout(nil, "show", "--format=%T", "-s", commit)
		if err != nil {
			return "", err
		}
		return strings.TrimSpace(out), nil
	}

	out, err = g.commandStdout(os.Stderr, "ls-tree", "--full-tree", commit, "--", pkg)
	if err != nil {
		return "", err
	}

	if out == "" {
		return "", fmt.Errorf("Package %s is not in git", pkg)
	}

	matches := treeHashRe.FindStringSubmatch(out)
	if len(matches) != 2 {
		return "", fmt.Errorf("Unable to parse ls-tree output: %q", out)
	}

	return matches[1], nil
}

func (g git) commitHash(commit string) (string, error) {
	out, err := g.commandStdout(os.Stderr, "rev-parse", commit)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(out), nil
}

func (g git) commitTag(commit string) (string, error) {
	out, err := g.commandStdout(os.Stderr, "tag", "-l", "--points-at", commit)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(out), nil
}

func (g git) isDirty(pkg, commit string) (bool, error) {
	// If it isn't HEAD it can't be dirty
	if commit != "HEAD" {
		return false, nil
	}

	// Update cache, otherwise files which have an updated
	// timestamp but no actual changes are marked as changes
	// because `git diff-index` only uses the `lstat` result and
	// not the actual file contents. Running `git update-index
	// --refresh` updates the cache.
	if err := g.command("update-index", "-q", "--refresh"); err != nil {
		return false, err
	}

	// diff-index works pretty well, except that
	err := g.command("diff-index", "--quiet", commit, "--", pkg)
	if err == nil {
		// this returns an error if there are *no* untracked files, which is strange, but we can work with it
		if _, err := g.commandStdout(nil, "ls-files", "--exclude-standard", "--others", "--error-unmatch", "--", pkg); err != nil {
			return false, nil
		}
		return true, nil
	}
	switch err.(type) {
	case *exec.ExitError:
		// diff-index exits with an error if there are differences
		return true, nil
	default:
		return false, err
	}
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"

	"istio.io/istio/istioctl/pkg/clioptions"
	"istio.io/istio/istioctl/pkg/util/handlers"
	"istio.io/istio/pilot/pkg/xds"
	"istio.io/istio/pkg/config"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/kube"
)

var (
	forFlag      string
	nameflag     string
	threshold    float32
	timeout      time.Duration
	generation   string
	verbose      bool
	targetSchema collection.Schema
	clientGetter func(string, string) (dynamic.Interface, error)
)

const pollInterval = time.Second

// waitCmd represents the wait command
func waitCmd() *cobra.Command {
	var opts clioptions.ControlPlaneOptions
	cmd := &cobra.Command{
		Use:   "wait [flags] <type> <name>[.<namespace>]",
		Short: "Wait for an Istio resource",
		Long:  `Waits for the specified condition to be true of an Istio resource.`,
		Example: `  # Wait until the bookinfo virtual service has been distributed to all proxies in the mesh
  istioctl experimental wait --for=distribution virtualservice bookinfo.default

  # Wait until 99% of the proxies receive the distribution, timing out after 5 minutes
  istioctl experimental wait --for=distribution --threshold=.99 --timeout=300s virtualservice bookinfo.default
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			printVerbosef(cmd, "kubeconfig %s", kubeconfig)
			printVerbosef(cmd, "ctx %s", configContext)
			if forFlag == "delete" {
				return errors.New("wait for delete is not yet implemented")
			} else if forFlag != "distribution" {
				return fmt.Errorf("--for must be 'delete' or 'distribution', got: %s", forFlag)
			}
			var w *watcher
			ctx, cancel := context.WithTimeout(context.Background(), timeout)
			defer cancel()
			if generation == "" {
				w = getAndWatchResource(ctx) // setup version getter from kubernetes
			} else {
				w = withContext(ctx)
				w.Go(func(result chan string) error {
					result <- generation
					return nil
				})
			}
			// wait for all deployed versions to be contained in generations
			t := time.NewTicker(pollInterval)
			printVerbosef(cmd, "getting first version from chan")
			firstVersion, err := w.BlockingRead()
			if err != nil {
				return fmt.Errorf("unable to retrieve Kubernetes resource %s: %v", "", err)
			}
			generations := []string{firstVersion}
			targetResource := config.Key(
				targetSchema.Resource().Group(), targetSchema.Resource().Version(), targetSchema.Resource().Kind(),
				nameflag, namespace)
			for {
				// run the check here as soon as we start
				// because tickers won't run immediately
				present, notpresent, sdcnum, err := poll(cmd, generations, targetResource, opts)
				printVerbosef(cmd, "Received poll result: %d/%d", present, present+notpresent)
				if err != nil {
					return err
				} else if float32(present)/float32(present+notpresent) >= threshold {
					_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resource %s present on %d out of %d configurations for totally %d sidecars\n",
						targetResource, present, present+notpresent, sdcnum)
					return nil
				}
				select {
				case newVersion := <-w.resultsChan:
					printVerbosef(cmd, "received new target version: %s", newVersion)
					generations = append(generations, newVersion)
				case <-t.C:
					printVerbosef(cmd, "tick")
					continue
				case err = <-w.errorChan:
					return fmt.Errorf("unable to retrieve Kubernetes resource2 %s: %v", "", err)
				case <-ctx.Done():
					printVerbosef(cmd, "timeout")
					// I think this means the timeout has happened:
					t.Stop()
					return fmt.Errorf("timeout expired before resource %s became effective on all sidecars",
						targetResource)
				}
			}
		},
		Args: func(cmd *cobra.Command, args []string) error {
			if err := cobra.ExactArgs(2)(cmd, args); err != nil {
				return err
			}
			nameflag, namespace = handlers.InferPodInfo(args[1], handlers.HandleNamespace(namespace, defaultNamespace))
			return validateType(args[0])
		},
	}
	cmd.PersistentFlags().StringVar(&forFlag, "for", "distribution",
		"Wait condition, must be 'distribution' or 'delete'")
	cmd.PersistentFlags().DurationVar(&timeout, "timeout", time.Second*30,
		"The duration to wait before failing")
	cmd.PersistentFlags().Float32Var(&threshold, "threshold", 1,
		"The ratio of distribution required for success")
	cmd.PersistentFlags().StringVar(&generation, "generation", "",
		"Wait for a specific generation of config to become current, rather than using whatever is latest in "+
			"Kubernetes")
	cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose output")
	_ = cmd.PersistentFlags().MarkHidden("verbose")
	opts.AttachControlPlaneFlags(cmd)
	return cmd
}

func printVerbosef(cmd *cobra.Command, template string, args ...any) {
	if verbose {
		_, _ = fmt.Fprintf(cmd.OutOrStdout(), template+"\n", args...)
	}
}

func validateType(kind string) error {
	originalKind := kind

	// Remove any dashes.
	kind = strings.ReplaceAll(kind, "-", "")

	for _, s := range collections.Pilot.All() {
		if strings.EqualFold(kind, s.Resource().Kind()) {
			targetSchema = s
			return nil
		}
	}
	return fmt.Errorf("type %s is not recognized", originalKind)
}

func countVersions(versionCount map[string]int, configVersion string) {
	if count, ok := versionCount[configVersion]; ok {
		versionCount[configVersion] = count + 1
	} else {
		versionCount[configVersion] = 1
	}
}

func poll(cmd *cobra.Command,
	acceptedVersions []string,
	targetResource string,
	opts clioptions.ControlPlaneOptions,
) (present, notpresent, sdcnum int, err error) {
	kubeClient, err := kubeClientWithRevision(kubeconfig, configContext, opts.Revision)
	if err != nil {
		return 0, 0, 0, err
	}
	path := fmt.Sprintf("/debug/config_distribution?resource=%s", targetResource)
	pilotResponses, err := kubeClient.AllDiscoveryDo(context.TODO(), istioNamespace, path)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unable to query pilot for distribution "+
			"(are you using pilot version >= 1.4 with config distribution tracking on): %s", err)
	}
	sdcnum = 0
	versionCount := make(map[string]int)
	for _, response := range pilotResponses {
		var configVersions []xds.SyncedVersions
		err = json.Unmarshal(response, &configVersions)
		if err != nil {
			return 0, 0, 0, err
		}
		printVerbosef(cmd, "sync status: %+v", configVersions)
		sdcnum += len(configVersions)
		for _, configVersion := range configVersions {
			countVersions(versionCount, configVersion.ClusterVersion)
			countVersions(versionCount, configVersion.RouteVersion)
			countVersions(versionCount, configVersion.ListenerVersion)
		}
	}

	for version, count := range versionCount {
		if contains(acceptedVersions, version) {
			present += count
		} else {
			notpresent += count
		}
	}
	return present, notpresent, sdcnum, nil
}

func init() {
	clientGetter = func(kubeconfig, context string) (dynamic.Interface, error) {
		config, err := kube.DefaultRestConfig(kubeconfig, context)
		if err != nil {
			return nil, err
		}
		cfg := dynamic.ConfigFor(config)
		dclient, err := dynamic.NewForConfig(cfg)
		if err != nil {
			return nil, err
		}
		return dclient, nil
	}
}

// getAndWatchResource ensures that Generations always contains
// the current generation of the targetResource, adding new versions
// as they are created.
func getAndWatchResource(ictx context.Context) *watcher {
	g := withContext(ictx)
	// copy nameflag to avoid race
	nf := nameflag
	g.Go(func(result chan string) error {
		// retrieve latest generation from Kubernetes
		dclient, err := clientGetter(kubeconfig, configContext)
		if err != nil {
			return err
		}
		collectionParts := strings.Split(targetSchema.Name().String(), "/")
		group := targetSchema.Resource().Group()
		version := targetSchema.Resource().Version()
		resource := collectionParts[3]
		r := dclient.Resource(schema.GroupVersionResource{Group: group, Version: version, Resource: resource}).Namespace(namespace)
		watch, err := r.Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + nf})
		if err != nil {
			return err
		}
		for w := range watch.ResultChan() {
			o, ok := w.Object.(metav1.Object)
			if !ok {
				continue
			}
			if o.GetName() == nf {
				result <- strconv.FormatInt(o.GetGeneration(), 10)
			}
			select {
			case <-ictx.Done():
				return ictx.Err()
			default:
				continue
			}
		}

		return nil
	})
	return g
}

type watcher struct {
	resultsChan chan string
	errorChan   chan error
	ctx         context.Context
}

func withContext(ctx context.Context) *watcher {
	return &watcher{
		resultsChan: make(chan string, 1),
		errorChan:   make(chan error, 1),
		ctx:         ctx,
	}
}

func (w *watcher) Go(f func(chan string) error) {
	go func() {
		if err := f(w.resultsChan); err != nil {
			w.errorChan <- err
		}
	}()
}

func (w *watcher) BlockingRead() (string, error) {
	select {
	case err := <-w.errorChan:
		return "", err
	case res := <-w.resultsChan:
		return res, nil
	case <-w.ctx.Done():
		return "", w.ctx.Err()
	}
}
<commit_msg>Update message for number of sidecars (#40437)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"

	"istio.io/istio/istioctl/pkg/clioptions"
	"istio.io/istio/istioctl/pkg/util/handlers"
	"istio.io/istio/pilot/pkg/xds"
	"istio.io/istio/pkg/config"
	"istio.io/istio/pkg/config/schema/collection"
	"istio.io/istio/pkg/config/schema/collections"
	"istio.io/istio/pkg/kube"
)

var (
	forFlag      string
	nameflag     string
	threshold    float32
	timeout      time.Duration
	generation   string
	verbose      bool
	targetSchema collection.Schema
	clientGetter func(string, string) (dynamic.Interface, error)
)

const pollInterval = time.Second

// waitCmd represents the wait command
func waitCmd() *cobra.Command {
	var opts clioptions.ControlPlaneOptions
	cmd := &cobra.Command{
		Use:   "wait [flags] <type> <name>[.<namespace>]",
		Short: "Wait for an Istio resource",
		Long:  `Waits for the specified condition to be true of an Istio resource.`,
		Example: `  # Wait until the bookinfo virtual service has been distributed to all proxies in the mesh
  istioctl experimental wait --for=distribution virtualservice bookinfo.default

  # Wait until 99% of the proxies receive the distribution, timing out after 5 minutes
  istioctl experimental wait --for=distribution --threshold=.99 --timeout=300s virtualservice bookinfo.default
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			printVerbosef(cmd, "kubeconfig %s", kubeconfig)
			printVerbosef(cmd, "ctx %s", configContext)
			if forFlag == "delete" {
				return errors.New("wait for delete is not yet implemented")
			} else if forFlag != "distribution" {
				return fmt.Errorf("--for must be 'delete' or 'distribution', got: %s", forFlag)
			}
			var w *watcher
			ctx, cancel := context.WithTimeout(context.Background(), timeout)
			defer cancel()
			if generation == "" {
				w = getAndWatchResource(ctx) // setup version getter from kubernetes
			} else {
				w = withContext(ctx)
				w.Go(func(result chan string) error {
					result <- generation
					return nil
				})
			}
			// wait for all deployed versions to be contained in generations
			t := time.NewTicker(pollInterval)
			printVerbosef(cmd, "getting first version from chan")
			firstVersion, err := w.BlockingRead()
			if err != nil {
				return fmt.Errorf("unable to retrieve Kubernetes resource %s: %v", "", err)
			}
			generations := []string{firstVersion}
			targetResource := config.Key(
				targetSchema.Resource().Group(), targetSchema.Resource().Version(), targetSchema.Resource().Kind(),
				nameflag, namespace)
			for {
				// run the check here as soon as we start
				// because tickers won't run immediately
				present, notpresent, sdcnum, err := poll(cmd, generations, targetResource, opts)
				printVerbosef(cmd, "Received poll result: %d/%d", present, present+notpresent)
				if err != nil {
					return err
				} else if float32(present)/float32(present+notpresent) >= threshold {
					_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resource %s present on %d out of %d configurations across %d sidecars\n",
						targetResource, present, present+notpresent, sdcnum)
					return nil
				}
				select {
				case newVersion := <-w.resultsChan:
					printVerbosef(cmd, "received new target version: %s", newVersion)
					generations = append(generations, newVersion)
				case <-t.C:
					printVerbosef(cmd, "tick")
					continue
				case err = <-w.errorChan:
					return fmt.Errorf("unable to retrieve Kubernetes resource2 %s: %v", "", err)
				case <-ctx.Done():
					printVerbosef(cmd, "timeout")
					// I think this means the timeout has happened:
					t.Stop()
					return fmt.Errorf("timeout expired before resource %s became effective on all sidecars",
						targetResource)
				}
			}
		},
		Args: func(cmd *cobra.Command, args []string) error {
			if err := cobra.ExactArgs(2)(cmd, args); err != nil {
				return err
			}
			nameflag, namespace = handlers.InferPodInfo(args[1], handlers.HandleNamespace(namespace, defaultNamespace))
			return validateType(args[0])
		},
	}
	cmd.PersistentFlags().StringVar(&forFlag, "for", "distribution",
		"Wait condition, must be 'distribution' or 'delete'")
	cmd.PersistentFlags().DurationVar(&timeout, "timeout", time.Second*30,
		"The duration to wait before failing")
	cmd.PersistentFlags().Float32Var(&threshold, "threshold", 1,
		"The ratio of distribution required for success")
	cmd.PersistentFlags().StringVar(&generation, "generation", "",
		"Wait for a specific generation of config to become current, rather than using whatever is latest in "+
			"Kubernetes")
	cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "enables verbose output")
	_ = cmd.PersistentFlags().MarkHidden("verbose")
	opts.AttachControlPlaneFlags(cmd)
	return cmd
}

func printVerbosef(cmd *cobra.Command, template string, args ...any) {
	if verbose {
		_, _ = fmt.Fprintf(cmd.OutOrStdout(), template+"\n", args...)
	}
}

func validateType(kind string) error {
	originalKind := kind

	// Remove any dashes.
	kind = strings.ReplaceAll(kind, "-", "")

	for _, s := range collections.Pilot.All() {
		if strings.EqualFold(kind, s.Resource().Kind()) {
			targetSchema = s
			return nil
		}
	}
	return fmt.Errorf("type %s is not recognized", originalKind)
}

func countVersions(versionCount map[string]int, configVersion string) {
	if count, ok := versionCount[configVersion]; ok {
		versionCount[configVersion] = count + 1
	} else {
		versionCount[configVersion] = 1
	}
}

func poll(cmd *cobra.Command,
	acceptedVersions []string,
	targetResource string,
	opts clioptions.ControlPlaneOptions,
) (present, notpresent, sdcnum int, err error) {
	kubeClient, err := kubeClientWithRevision(kubeconfig, configContext, opts.Revision)
	if err != nil {
		return 0, 0, 0, err
	}
	path := fmt.Sprintf("/debug/config_distribution?resource=%s", targetResource)
	pilotResponses, err := kubeClient.AllDiscoveryDo(context.TODO(), istioNamespace, path)
	if err != nil {
		return 0, 0, 0, fmt.Errorf("unable to query pilot for distribution "+
			"(are you using pilot version >= 1.4 with config distribution tracking on): %s", err)
	}
	sdcnum = 0
	versionCount := make(map[string]int)
	for _, response := range pilotResponses {
		var configVersions []xds.SyncedVersions
		err = json.Unmarshal(response, &configVersions)
		if err != nil {
			return 0, 0, 0, err
		}
		printVerbosef(cmd, "sync status: %+v", configVersions)
		sdcnum += len(configVersions)
		for _, configVersion := range configVersions {
			countVersions(versionCount, configVersion.ClusterVersion)
			countVersions(versionCount, configVersion.RouteVersion)
			countVersions(versionCount, configVersion.ListenerVersion)
		}
	}

	for version, count := range versionCount {
		if contains(acceptedVersions, version) {
			present += count
		} else {
			notpresent += count
		}
	}
	return present, notpresent, sdcnum, nil
}

func init() {
	clientGetter = func(kubeconfig, context string) (dynamic.Interface, error) {
		config, err := kube.DefaultRestConfig(kubeconfig, context)
		if err != nil {
			return nil, err
		}
		cfg := dynamic.ConfigFor(config)
		dclient, err := dynamic.NewForConfig(cfg)
		if err != nil {
			return nil, err
		}
		return dclient, nil
	}
}

// getAndWatchResource ensures that Generations always contains
// the current generation of the targetResource, adding new versions
// as they are created.
func getAndWatchResource(ictx context.Context) *watcher {
	g := withContext(ictx)
	// copy nameflag to avoid race
	nf := nameflag
	g.Go(func(result chan string) error {
		// retrieve latest generation from Kubernetes
		dclient, err := clientGetter(kubeconfig, configContext)
		if err != nil {
			return err
		}
		collectionParts := strings.Split(targetSchema.Name().String(), "/")
		group := targetSchema.Resource().Group()
		version := targetSchema.Resource().Version()
		resource := collectionParts[3]
		r := dclient.Resource(schema.GroupVersionResource{Group: group, Version: version, Resource: resource}).Namespace(namespace)
		watch, err := r.Watch(context.TODO(), metav1.ListOptions{FieldSelector: "metadata.name=" + nf})
		if err != nil {
			return err
		}
		for w := range watch.ResultChan() {
			o, ok := w.Object.(metav1.Object)
			if !ok {
				continue
			}
			if o.GetName() == nf {
				result <- strconv.FormatInt(o.GetGeneration(), 10)
			}
			select {
			case <-ictx.Done():
				return ictx.Err()
			default:
				continue
			}
		}

		return nil
	})
	return g
}

type watcher struct {
	resultsChan chan string
	errorChan   chan error
	ctx         context.Context
}

func withContext(ctx context.Context) *watcher {
	return &watcher{
		resultsChan: make(chan string, 1),
		errorChan:   make(chan error, 1),
		ctx:         ctx,
	}
}

func (w *watcher) Go(f func(chan string) error) {
	go func() {
		if err := f(w.resultsChan); err != nil {
			w.errorChan <- err
		}
	}()
}

func (w *watcher) BlockingRead() (string, error) {
	select {
	case err := <-w.errorChan:
		return "", err
	case res := <-w.resultsChan:
		return res, nil
	case <-w.ctx.Done():
		return "", w.ctx.Err()
	}
}
<|endoftext|>


<commit_before>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with gm file,
// You can obtain one at https://github.com/gogf/gf.

package gset_test

import (
	"fmt"
	"github.com/gogf/gf/v2/container/gset"
	"github.com/gogf/gf/v2/frame/g"
)

func ExampleStrSet_Contains() {
	var set gset.StrSet
	set.Add("a")
	fmt.Println(set.Contains("a"))
	fmt.Println(set.Contains("A"))
	fmt.Println(set.ContainsI("A"))

	// Output:
	// true
	// false
	// true
}

func ExampleStrSet_Walk() {
	var (
		set    gset.StrSet
		names  = g.SliceStr{"user", "user_detail"}
		prefix = "gf_"
	)
	set.Add(names...)
	// Add prefix for given table names.
	set.Walk(func(item string) string {
		return prefix + item
	})
	fmt.Println(set.Slice())

	// May Output:
	// [gf_user gf_user_detail]
}
<commit_msg>update<commit_after>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with gm file,
// You can obtain one at https://github.com/gogf/gf.

package gset_test

import (
	"fmt"
	"github.com/gogf/gf/v2/container/gset"
	"github.com/gogf/gf/v2/frame/g"
)

func ExampleStrSet_NewStrSet() {
	strSet := gset.NewStrSet(true)
	strSet.Add([]string{"str1", "str2", "str3"}...)

	// Mya Output:
	//Iterator  str1
	//Iterator  str2
	//Iterator  str3
}

func ExampleStrSet_Add() {
	var strSet gset.StrSet
	strSet.Add([]string{"str1", "str2", "str3"}...)

	// Mya Output:
	//Iterator  str1
	//Iterator  str2
	//Iterator  str3
}

func ExampleStrSet_AddIfNotExist() {
	var strSet gset.StrSet
	fmt.Println(strSet.AddIfNotExist("str"))

	// Output:
	// true
}

func ExampleStrSet_AddIfNotExistFunc() {
	var strSet gset.StrSet
	fmt.Println(strSet.AddIfNotExistFunc("str", func() bool {
		return true
	}))

	// Output:
	// true
}

func ExampleStrSet_AddIfNotExistFuncLock() {
	var strSet gset.StrSet
	fmt.Println(strSet.AddIfNotExistFuncLock("str", func() bool {
		return true
	}))

	// Output:
	// true
}

func ExampleStrSet_Clear() {
	var strSet gset.StrSet
	strSet.Add([]string{"str1", "str2", "str3"}...)

	strSet.Clear()

	fmt.Println(strSet.Size())

	// Output:
	// 0
}

func ExampleStrSet_Complement() {
	strSet := gset.NewStrSet(true)
	strSet.Add([]string{"str1", "str2", "str3", "str4", "str5"}...)

	var s gset.StrSet
	s.Add([]string{"str1", "str2", "str3"}...)

	fmt.Println(s.Complement(strSet).Slice())

	// May Output:
	// [str4 str5]
}

func ExampleStrSet_Contains() {
	var set gset.StrSet
	set.Add("a")
	fmt.Println(set.Contains("a"))
	fmt.Println(set.Contains("A"))

	// Output:
	// true
	// false
}

func ExampleStrSet_ContainsI() {
	var set gset.StrSet
	set.Add("a")
	fmt.Println(set.ContainsI("a"))
	fmt.Println(set.ContainsI("A"))

	// Output:
	// true
	// true
}

func ExampleStrSet_Diff() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c"}...)
	var s2 gset.StrSet
	s2.Add([]string{"a", "b", "c", "d"}...)
	// 差集
	fmt.Println(s2.Diff(s1).Slice())

	// Output:
	// [d]
}

func ExampleStrSet_Equal() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c"}...)
	var s2 gset.StrSet
	s2.Add([]string{"a", "b", "c", "d"}...)
	fmt.Println(s2.Equal(s1))

	// Output:
	// false
}

func ExampleStrSet_Intersect() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c"}...)
	var s2 gset.StrSet
	s2.Add([]string{"a", "b", "c", "d"}...)
	// 交集
	fmt.Println(s2.Intersect(s1).Slice())

	// May Output:
	// [c a b]
}

func ExampleStrSet_IsSubsetOf() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c", "d"}...)
	var s2 gset.StrSet
	s2.Add([]string{"a", "b", "d"}...)
	fmt.Println(s2.IsSubsetOf(s1))

	// Output:
	// true
}

func ExampleStrSet_Iterator() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c", "d"}...)
	s1.Iterator(func(v string) bool {
		fmt.Println("Iterator", v)
		return true
	})
	// May Output:
	// Iterator a
	// Iterator b
	// Iterator c
	// Iterator d
}

func ExampleStrSet_Join() {
	s1 := gset.NewStrSet(true)
	s1.Add([]string{"a", "b", "c", "d"}...)
	fmt.Println(s1.Join(","))

	// May Output:
	// b,c,d,a
}

func ExampleStrSet_LockFunc() {

}

func ExampleStrSet_Walk() {
	var (
		set    gset.StrSet
		names  = g.SliceStr{"user", "user_detail"}
		prefix = "gf_"
	)
	set.Add(names...)
	// Add prefix for given table names.
	set.Walk(func(item string) string {
		return prefix + item
	})
	fmt.Println(set.Slice())

	// May Output:
	// [gf_user gf_user_detail]
}
<|endoftext|>


<commit_before>package shells

import (
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"errors"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)

type AbstractShell struct {
}

func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
	features.Artifacts = true
	features.Cache = true
}

func (b *AbstractShell) GetSupportedOptions() []string {
	return []string{"artifacts", "cache", "dependencies", "after_script"}
}

func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
	w.Cd(info.Build.FullProjectDir())
}

func (b *AbstractShell) writeExports(w ShellWriter, info common.ShellScriptInfo) {
	for _, variable := range info.Build.GetAllVariables() {
		w.Variable(variable)
	}
}

func (b *AbstractShell) writeTLSCAInfo(w ShellWriter, build *common.Build, key string) {
	if build.TLSCAChain != "" {
		w.Variable(common.BuildVariable{
			Key:      key,
			Value:    build.TLSCAChain,
			Public:   true,
			Internal: true,
			File:     true,
		})
	}
}

func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projectDir string) {
	w.RmDir(projectDir)
	if depth := build.GetGitDepth(); depth != "" {
		w.Notice("Cloning repository for %s with git depth set to %s...", build.RefName, depth)
		w.Command("git", "clone", build.RepoURL, projectDir, "--depth", depth, "--branch", build.RefName)
	} else {
		w.Notice("Cloning repository...")
		w.Command("git", "clone", build.RepoURL, projectDir)
	}
	w.Cd(projectDir)
}

func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
	depth := build.GetGitDepth()

	w.IfDirectory(gitDir)
	if depth != "" {
		w.Notice("Fetching changes for %s with git depth set to %s...", build.RefName, depth)
	} else {
		w.Notice("Fetching changes...")
	}
	w.Cd(projectDir)
	w.Command("git", "clean", "-ffdx")
	w.Command("git", "reset", "--hard")
	w.Command("git", "remote", "set-url", "origin", build.RepoURL)
	if depth != "" {
		var refspec string
		if build.Tag {
			refspec = "+refs/tags/" + build.RefName + ":refs/tags/" + build.RefName
		} else {
			refspec = "+refs/heads/" + build.RefName + ":refs/remotes/origin/" + build.RefName
		}
		w.Command("git", "fetch", "--depth", depth, "origin", "--prune", refspec)
	} else {
		w.Command("git", "fetch", "origin", "--prune", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*")
	}
	w.Else()
	b.writeCloneCmd(w, build, projectDir)
	w.EndIf()
}

func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
	w.Notice("Checking out %s as %s...", build.Sha[0:8], build.RefName)
	// We remove a git index file, this is required if `git checkout` is terminated
	w.RmFile(".git/index.lock")
	w.Command("git", "checkout", "-q", build.Sha)
}

func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
	if build.CacheDir == "" {
		return
	}

	// Deduce cache key
	key = path.Join(build.Name, build.RefName)
	if userKey != "" {
		key = build.GetAllVariables().ExpandValue(userKey)
	}

	// Ignore cache without the key
	if key == "" {
		return
	}

	file = path.Join(build.CacheDir, key, "cache.zip")
	file, err := filepath.Rel(build.BuildDir, file)
	if err != nil {
		return "", ""
	}
	return
}

func (o *archivingOptions) CommandArguments() (args []string) {
	for _, path := range o.Paths {
		args = append(args, "--path", path)
	}

	if o.Untracked {
		args = append(args, "--untracked")
	}
	return
}

func (b *AbstractShell) guardRunnerCommand(w ShellWriter, runnerCommand string, action string, f func()) {
	if runnerCommand == "" {
		w.Warning("%s is not supported by this executor.", action)
		return
	}

	w.IfCmd(runnerCommand, "--version")
	f()
	w.Else()
	w.Warning("Missing %s. %s is disabled.", runnerCommand, action)
	w.EndIf()
}

func (b *AbstractShell) cacheExtractor(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}

	// Skip restoring cache if no cache is defined
	if archiverArgs := options.CommandArguments(); len(archiverArgs) == 0 {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-extractor",
		"--file", cacheFile,
	}

	// Generate cache download address
	if url := getCacheDownloadURL(info.Build, cacheKey); url != nil {
		args = append(args, "--url", url.String())
	}

	// Execute archive command
	b.guardRunnerCommand(w, info.RunnerCommand, "Extracting cache", func() {
		w.Notice("Checking cache for %s...", cacheKey)
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) downloadArtifacts(w ShellWriter, build *common.BuildInfo, info common.ShellScriptInfo) {
	b.guardRunnerCommand(w, info.RunnerCommand, "Artifacts downloading", func() {
		args := []string{
			"artifacts-downloader",
			"--url",
			info.Build.Runner.URL,
			"--token",
			build.Token,
			"--id",
			strconv.Itoa(build.ID),
		}

		w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) downloadAllArtifacts(w ShellWriter, dependencies *dependencies, info common.ShellScriptInfo) {
	for _, otherBuild := range info.Build.DependsOnBuilds {
		if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
			continue
		}
		if !dependencies.IsDependent(otherBuild.Name) {
			continue
		}
		b.downloadArtifacts(w, &otherBuild, info)
	}
}

func (b *AbstractShell) writePrepareScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	b.writeExports(w, info)

	build := info.Build
	projectDir := build.FullProjectDir()
	gitDir := path.Join(build.FullProjectDir(), ".git")

	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	w.Command("git", "config", "--global", "fetch.recurseSubmodules", "false")
	switch info.Build.GetGitStrategy() {
	case common.GitFetch:
		b.writeFetchCmd(w, build, projectDir, gitDir)

	case common.GitClone:
		b.writeCloneCmd(w, build, projectDir)

	default:
		return errors.New("unknown GIT_STRATEGY")
	}

	b.writeCheckoutCmd(w, build)

	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	// Try to restore from main cache, if not found cache for master
	b.cacheExtractor(w, options.Cache, info)

	// Process all artifacts
	b.downloadAllArtifacts(w, options.Dependencies, info)
	return nil
}

func (b *AbstractShell) writeBuildScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	commands := info.Build.Commands
	commands = strings.TrimSpace(commands)
	for _, command := range strings.Split(commands, "\n") {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
		w.CheckForErrors()
	}

	return nil
}

func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-archiver",
		"--file", cacheFile,
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Generate cache upload address
	if url := getCacheUploadURL(info.Build, cacheKey); url != nil {
		args = append(args, "--url", url.String())
	}

	b.guardRunnerCommand(w, info.RunnerCommand, "Creating cache", func() {
		// Execute archive command
		w.Notice("Creating cache %s...", cacheKey)
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.Build.Runner.URL == "" {
		return
	}

	args := []string{
		"artifacts-uploader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		info.Build.Token,
		"--id",
		strconv.Itoa(info.Build.ID),
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Get artifacts:name
	if name, ok := info.Build.Options.GetString("artifacts", "name"); ok && name != "" {
		args = append(args, "--name", name)
	}

	// Get artifacts:expire_in
	if expireIn, ok := info.Build.Options.GetString("artifacts", "expire_in"); ok && expireIn != "" {
		args = append(args, "--expire-in", expireIn)
	}

	b.guardRunnerCommand(w, info.RunnerCommand, "Uploading artifacts", func() {
		w.Notice("Uploading artifacts...")
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptInfo) error {
	shellOptions := struct {
		AfterScript []string `json:"after_script"`
	}{}
	err := info.Build.Options.Decode(&shellOptions)
	if err != nil {
		return err
	}

	if len(shellOptions.AfterScript) == 0 {
		return nil
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	w.Notice("Running after script...")

	for _, command := range shellOptions.AfterScript {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
		w.CheckForErrors()
	}

	return nil
}

func (b *AbstractShell) writeArchiveCacheScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Find cached files and archive them
	b.cacheArchiver(w, options.Cache, info)
	return
}

func (b *AbstractShell) writeUploadArtifactsScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Upload artifacts
	b.uploadArtifacts(w, options.Artifacts, info)
	return
}

func (b *AbstractShell) writeScript(w ShellWriter, scriptType common.ShellScriptType, info common.ShellScriptInfo) (err error) {
	switch scriptType {
	case common.ShellPrepareScript:
		return b.writePrepareScript(w, info)

	case common.ShellBuildScript:
		return b.writeBuildScript(w, info)

	case common.ShellAfterScript:
		return b.writeAfterScript(w, info)

	case common.ShellArchiveCache:
		return b.writeArchiveCacheScript(w, info)

	case common.ShellUploadArtifacts:
		return b.writeUploadArtifactsScript(w, info)

	default:
		return errors.New("Not supported script type: " + string(scriptType))
	}
}
<commit_msg>Print failure for artifacts only once for all stages<commit_after>package shells

import (
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"errors"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)

type AbstractShell struct {
}

func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
	features.Artifacts = true
	features.Cache = true
}

func (b *AbstractShell) GetSupportedOptions() []string {
	return []string{"artifacts", "cache", "dependencies", "after_script"}
}

func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
	w.Cd(info.Build.FullProjectDir())
}

func (b *AbstractShell) writeExports(w ShellWriter, info common.ShellScriptInfo) {
	for _, variable := range info.Build.GetAllVariables() {
		w.Variable(variable)
	}
}

func (b *AbstractShell) writeTLSCAInfo(w ShellWriter, build *common.Build, key string) {
	if build.TLSCAChain != "" {
		w.Variable(common.BuildVariable{
			Key:      key,
			Value:    build.TLSCAChain,
			Public:   true,
			Internal: true,
			File:     true,
		})
	}
}

func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projectDir string) {
	w.RmDir(projectDir)
	if depth := build.GetGitDepth(); depth != "" {
		w.Notice("Cloning repository for %s with git depth set to %s...", build.RefName, depth)
		w.Command("git", "clone", build.RepoURL, projectDir, "--depth", depth, "--branch", build.RefName)
	} else {
		w.Notice("Cloning repository...")
		w.Command("git", "clone", build.RepoURL, projectDir)
	}
	w.Cd(projectDir)
}

func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
	depth := build.GetGitDepth()

	w.IfDirectory(gitDir)
	if depth != "" {
		w.Notice("Fetching changes for %s with git depth set to %s...", build.RefName, depth)
	} else {
		w.Notice("Fetching changes...")
	}
	w.Cd(projectDir)
	w.Command("git", "clean", "-ffdx")
	w.Command("git", "reset", "--hard")
	w.Command("git", "remote", "set-url", "origin", build.RepoURL)
	if depth != "" {
		var refspec string
		if build.Tag {
			refspec = "+refs/tags/" + build.RefName + ":refs/tags/" + build.RefName
		} else {
			refspec = "+refs/heads/" + build.RefName + ":refs/remotes/origin/" + build.RefName
		}
		w.Command("git", "fetch", "--depth", depth, "origin", "--prune", refspec)
	} else {
		w.Command("git", "fetch", "origin", "--prune", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*")
	}
	w.Else()
	b.writeCloneCmd(w, build, projectDir)
	w.EndIf()
}

func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
	w.Notice("Checking out %s as %s...", build.Sha[0:8], build.RefName)
	// We remove a git index file, this is required if `git checkout` is terminated
	w.RmFile(".git/index.lock")
	w.Command("git", "checkout", "-q", build.Sha)
}

func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
	if build.CacheDir == "" {
		return
	}

	// Deduce cache key
	key = path.Join(build.Name, build.RefName)
	if userKey != "" {
		key = build.GetAllVariables().ExpandValue(userKey)
	}

	// Ignore cache without the key
	if key == "" {
		return
	}

	file = path.Join(build.CacheDir, key, "cache.zip")
	file, err := filepath.Rel(build.BuildDir, file)
	if err != nil {
		return "", ""
	}
	return
}

func (o *archivingOptions) CommandArguments() (args []string) {
	for _, path := range o.Paths {
		args = append(args, "--path", path)
	}

	if o.Untracked {
		args = append(args, "--untracked")
	}
	return
}

func (b *AbstractShell) guardRunnerCommand(w ShellWriter, runnerCommand string, action string, f func()) {
	if runnerCommand == "" {
		w.Warning("%s is not supported by this executor.", action)
		return
	}

	w.IfCmd(runnerCommand, "--version")
	f()
	w.Else()
	w.Warning("Missing %s. %s is disabled.", runnerCommand, action)
	w.EndIf()
}

func (b *AbstractShell) cacheExtractor(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}

	// Skip restoring cache if no cache is defined
	if archiverArgs := options.CommandArguments(); len(archiverArgs) == 0 {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-extractor",
		"--file", cacheFile,
	}

	// Generate cache download address
	if url := getCacheDownloadURL(info.Build, cacheKey); url != nil {
		args = append(args, "--url", url.String())
	}

	// Execute archive command
	b.guardRunnerCommand(w, info.RunnerCommand, "Extracting cache", func() {
		w.Notice("Checking cache for %s...", cacheKey)
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) downloadArtifacts(w ShellWriter, build *common.BuildInfo, info common.ShellScriptInfo) {
	args := []string{
		"artifacts-downloader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		build.Token,
		"--id",
		strconv.Itoa(build.ID),
	}

	w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) downloadAllArtifacts(w ShellWriter, dependencies *dependencies, info common.ShellScriptInfo) {
	b.guardRunnerCommand(w, info.RunnerCommand, "Artifacts downloading", func() {
		for _, otherBuild := range info.Build.DependsOnBuilds {
			if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
				continue
			}
			if !dependencies.IsDependent(otherBuild.Name) {
				continue
			}
			b.downloadArtifacts(w, &otherBuild, info)
		}
	})
}

func (b *AbstractShell) writePrepareScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	b.writeExports(w, info)

	build := info.Build
	projectDir := build.FullProjectDir()
	gitDir := path.Join(build.FullProjectDir(), ".git")

	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	w.Command("git", "config", "--global", "fetch.recurseSubmodules", "false")
	switch info.Build.GetGitStrategy() {
	case common.GitFetch:
		b.writeFetchCmd(w, build, projectDir, gitDir)

	case common.GitClone:
		b.writeCloneCmd(w, build, projectDir)

	default:
		return errors.New("unknown GIT_STRATEGY")
	}

	b.writeCheckoutCmd(w, build)

	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	// Try to restore from main cache, if not found cache for master
	b.cacheExtractor(w, options.Cache, info)

	// Process all artifacts
	b.downloadAllArtifacts(w, options.Dependencies, info)
	return nil
}

func (b *AbstractShell) writeBuildScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	commands := info.Build.Commands
	commands = strings.TrimSpace(commands)
	for _, command := range strings.Split(commands, "\n") {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
		w.CheckForErrors()
	}

	return nil
}

func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-archiver",
		"--file", cacheFile,
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Generate cache upload address
	if url := getCacheUploadURL(info.Build, cacheKey); url != nil {
		args = append(args, "--url", url.String())
	}

	b.guardRunnerCommand(w, info.RunnerCommand, "Creating cache", func() {
		// Execute archive command
		w.Notice("Creating cache %s...", cacheKey)
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.Build.Runner.URL == "" {
		return
	}

	args := []string{
		"artifacts-uploader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		info.Build.Token,
		"--id",
		strconv.Itoa(info.Build.ID),
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Get artifacts:name
	if name, ok := info.Build.Options.GetString("artifacts", "name"); ok && name != "" {
		args = append(args, "--name", name)
	}

	// Get artifacts:expire_in
	if expireIn, ok := info.Build.Options.GetString("artifacts", "expire_in"); ok && expireIn != "" {
		args = append(args, "--expire-in", expireIn)
	}

	b.guardRunnerCommand(w, info.RunnerCommand, "Uploading artifacts", func() {
		w.Notice("Uploading artifacts...")
		w.Command(info.RunnerCommand, args...)
	})
}

func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptInfo) error {
	shellOptions := struct {
		AfterScript []string `json:"after_script"`
	}{}
	err := info.Build.Options.Decode(&shellOptions)
	if err != nil {
		return err
	}

	if len(shellOptions.AfterScript) == 0 {
		return nil
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	w.Notice("Running after script...")

	for _, command := range shellOptions.AfterScript {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
		w.CheckForErrors()
	}

	return nil
}

func (b *AbstractShell) writeArchiveCacheScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Find cached files and archive them
	b.cacheArchiver(w, options.Cache, info)
	return
}

func (b *AbstractShell) writeUploadArtifactsScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
	// Parse options
	var options shellOptions
	err = info.Build.Options.Decode(&options)
	if err != nil {
		return
	}

	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Upload artifacts
	b.uploadArtifacts(w, options.Artifacts, info)
	return
}

func (b *AbstractShell) writeScript(w ShellWriter, scriptType common.ShellScriptType, info common.ShellScriptInfo) (err error) {
	switch scriptType {
	case common.ShellPrepareScript:
		return b.writePrepareScript(w, info)

	case common.ShellBuildScript:
		return b.writeBuildScript(w, info)

	case common.ShellAfterScript:
		return b.writeAfterScript(w, info)

	case common.ShellArchiveCache:
		return b.writeArchiveCacheScript(w, info)

	case common.ShellUploadArtifacts:
		return b.writeUploadArtifactsScript(w, info)

	default:
		return errors.New("Not supported script type: " + string(scriptType))
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"database/sql"
	"reflect"
	"strings"
	"sync"
	"time"
)

// SQLScan helper class for reading sql to Struct
// Columns in struct must be marked with a `sql:"col_name"` tag
// Ex: in sql a column name is col1, in struct the col tag must be `sql:"col1"`
type SQLScan struct {
	sync.RWMutex
	columnNames []string
}

// Clear - clears the columns array.
// Used to be able to reuse the scan helper for another SQL
func (s *SQLScan) Clear() {
	s.Lock()
	defer s.Unlock()

	s.columnNames = nil
}

// Scan - reads sql statement into a struct
func (s *SQLScan) Scan(u *DbUtils, rows *sql.Rows, dest interface{}) error {
	s.Lock()
	defer s.Unlock()

	if s.columnNames == nil || len(s.columnNames) == 0 {
		cols, err := rows.Columns()
		if err != nil {
			return err
		}

		s.columnNames = cols
	}

	nrCols := len(s.columnNames)
	pointers := make([]interface{}, nrCols)
	fieldTypes := make([]reflect.Type, nrCols)

	structVal := reflect.ValueOf(dest).Elem()
	nFields := structVal.NumField()

	for i, colName := range s.columnNames {
		loweredColName := strings.ToLower(colName)
		for j := 0; j < nFields; j++ {
			typeField := structVal.Type().Field(j)
			tag := typeField.Tag

			if tag.Get("sql") == loweredColName {
				pointers[i] = structVal.Field(j).Addr().Interface()
				fieldTypes[i] = typeField.Type
				break
			}
		}
	}

	err := rows.Scan(pointers...)
	if err != nil {
		return err
	}

	if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
		// in oci, the timestamp is comming up as local time zone
		// even if you ask for the UTC
		dt := time.Now()
		dtType := reflect.TypeOf(dt)

		for i := 0; i < nrCols; i++ {
			if fieldTypes[i] == dtType {
				dtval := pointers[i].(*time.Time)
				strdt := Date2string(*dtval, ISODateTimestamp)
				*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
			}
		}
	}

	return nil
}
<commit_msg>Oracle has column names as upper<commit_after>package utils

import (
	"database/sql"
	"reflect"
	"strings"
	"sync"
	"time"
)

// SQLScan helper class for reading sql to Struct
// Columns in struct must be marked with a `sql:"col_name"` tag
// Ex: in sql a column name is col1, in struct the col tag must be `sql:"col1"`
type SQLScan struct {
	sync.RWMutex
	columnNames []string
}

// Clear - clears the columns array.
// Used to be able to reuse the scan helper for another SQL
func (s *SQLScan) Clear() {
	s.Lock()
	defer s.Unlock()

	s.columnNames = nil
}

// Scan - reads sql statement into a struct
func (s *SQLScan) Scan(u *DbUtils, rows *sql.Rows, dest interface{}) error {
	s.Lock()
	defer s.Unlock()

	if s.columnNames == nil || len(s.columnNames) == 0 {
		cols, err := rows.Columns()
		if err != nil {
			return err
		}

		s.columnNames = cols

		if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
			for i, colName := range s.columnNames {
				if colName[0:1] != "\"" {
					s.columnNames[i] = strings.ToLower(colName)
				}
			}
		}
	}

	nrCols := len(s.columnNames)
	pointers := make([]interface{}, nrCols)
	fieldTypes := make([]reflect.Type, nrCols)

	structVal := reflect.ValueOf(dest).Elem()
	nFields := structVal.NumField()

	for i, colName := range s.columnNames {
		for j := 0; j < nFields; j++ {
			typeField := structVal.Type().Field(j)
			tag := typeField.Tag

			if tag.Get("sql") == colName {
				pointers[i] = structVal.Field(j).Addr().Interface()
				fieldTypes[i] = typeField.Type
				break
			}
		}
	}

	err := rows.Scan(pointers...)
	if err != nil {
		return err
	}

	if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
		// in oci, the timestamp is comming up as local time zone
		// even if you ask for the UTC
		dt := time.Now()
		dtType := reflect.TypeOf(dt)

		for i := 0; i < nrCols; i++ {
			if fieldTypes[i] == dtType {
				dtval := pointers[i].(*time.Time)
				strdt := Date2string(*dtval, ISODateTimestamp)
				*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
			}
		}
	}

	return nil
}
<|endoftext|>