<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package builtins

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	appconfig "github.com/GoogleCloudPlatform/anthos-appconfig/appconfigmgrv2/api/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sapierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/types"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/webhook"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

var log = ctrl.Log.WithName("webhooks-builtins-pod")
var TODO_FIND_NAMESPACE = "appconfigmgrv2-system"
var localMgr ctrl.Manager

func SetupWebHook(mgr ctrl.Manager) {

	// Setup webhooks
	// entryLog.Info("setting up webhook server")
	hookServer := mgr.GetWebhookServer()
	localMgr = mgr

	// entryLog.Info("registering webhooks to the webhook server")
	hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: &podAnnotator{}})
	hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{}})

}

// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=upod.appconfigmgr.cft.dev

// podAnnotator annotates Pods
type podAnnotator struct {
	client  client.Client
	decoder *admission.Decoder
}

//func getJSONKey(client client.Client, ns string, secret string) {
//	client.
//}

func kubeSecretFromTemplate(ns string, name string, mapKey string, value string) *corev1.Secret {
	return kubeSecretFromTemplateBytes(ns, name, mapKey, []byte(value))
}

func kubeSecretFromTemplateBytes(ns string, name string, mapKey string, value []byte) *corev1.Secret {
	return &corev1.Secret{
		Type: corev1.SecretTypeOpaque,
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
		},
		Data: map[string][]byte{
			mapKey: []byte(value),
		},
	}
}

func kubeSecretReponse(ns string, name string) *corev1.Secret {
	return &corev1.Secret{
		Type: corev1.SecretTypeOpaque,
	}
}

func getSecretName(ns string) string {
	log.Info("getSecretName:Start:" + "demo-" + ns + "-key")
	return "demo-" + ns + "-key"
}

func updateSecretsVolume(pod *corev1.Pod, secretName string) {
	log.V(1).Info("updateSecretsVolume", "secretName", secretName)
	found := false
	index := -1
	for i, element := range pod.Spec.Volumes {
		if element.Name == "google-auth-token" {
			log.V(1).Info("updateSecretsVolume:volumeFound", "element.Name", element.Name)
			found = true
			index = i
		}
	}
	element := &corev1.Volume{
		Name: "google-auth-token",
		VolumeSource: corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: secretName,
			},
		},
	}
	if !found {
		index = len(pod.Spec.Volumes)
		pod.Spec.Volumes = append(pod.Spec.Volumes, *element)
	} else {
		pod.Spec.Volumes[index] = *element
	}

}

func updateContainers(pod *corev1.Pod, containerName string, mountName string, mountPath string, envName string) {
	log.V(1).Info("updateContainers",
		"containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath,
	)

	for index, element := range pod.Spec.Containers {
		if element.Name == containerName {
			log.V(1).Info("updateContainers:found",
				"containerName", containerName,
				"mountName", mountName,
				"mountPath", mountPath)
			updateContainerMounts(&element, containerName, mountName, mountPath)
			updateContainerEnv(&element, containerName, envName, mountPath+"/key.json")
			pod.Spec.Containers[index] = element
			return
		}
	}

	//TODO - Decide how to fail or just warning
	log.Info("updateContainers:containerNotFound", "containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath)
}

func updateContainerMounts(container *corev1.Container, containerName string, mountName string, mountPath string) {
	log.V(1).Info("updateContainerMounts",
		"containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath,
	)
	found := false
	index := -1
	for i, element := range container.VolumeMounts {
		if element.Name == mountName {
			found = true
			index = i
			log.V(1).Info("updateContainerMounts:found",
				"containerName", containerName,
				"mountName", mountName,
				"mountPath", mountPath,
			)
		}
	}
	element := &corev1.VolumeMount{
		Name:      mountName,
		MountPath: mountPath,
	}

	if !found {
		// Append The Mount
		log.V(1).Info("updateContainerMounts:addMount",
			"containerName", containerName,
			"mountName", mountName,
			"mountPath", mountPath,
		)
		index = len(container.VolumeMounts)
		container.VolumeMounts = append(container.VolumeMounts, *element)
	} else {
		container.VolumeMounts[index] = *element
	}

	log.V(1).Info("updateContainerMounts:exit",
		"containerInfo", container,
	)

}

func updateContainerEnv(container *corev1.Container, containerName string, envName string, mountPath string) {
	log.V(1).Info("updateContainerEnv",
		"containerName", containerName,
		"envName", envName,
		"mountPath", mountPath,
	)
	found := false
	index := -1
	for i, element := range container.Env {
		if element.Name == envName {
			found = true
			index = i
			log.V(1).Info("updateContainerEnv:found",
				"containerName", containerName,
				"envName", envName,
				"mountPath", mountPath,
			)
		}
	}
	element := &corev1.EnvVar{
		Name:  envName,
		Value: mountPath,
	}

	if !found {
		// Append The Mount
		log.V(1).Info("updateContainerEnv:addMount",
			"containerName", containerName,
			"envName", envName,
			"mountPath", mountPath,
		)
		index = len(container.Env)
		container.Env = append(container.Env, *element)
	} else {
		container.Env[index] = *element
	}

	log.V(1).Info("updateContainerEnv:exit",
		"containerInfo", container,
	)

}

func (a *podAnnotator) handleGCPSecretIfNeeded(ctx context.Context, pod *corev1.Pod, app *appconfig.AppEnvConfigTemplateV2) error {

	log.Info("podAnnotator:handleGCPSecretIfNeeded")
	if app.Spec.Auth == nil ||
		app.Spec.Auth.GCPAccess == nil ||
		app.Spec.Auth.GCPAccess.AccessType != "secret" {

		return nil
	}

	secretName := app.Spec.Auth.GCPAccess.SecretInfo.Name
	secretNamespace := TODO_FIND_NAMESPACE
	secret := &corev1.Secret{}

	cl := localMgr.GetClient()
	err := cl.Get(ctx, types.NamespacedName{Name: secretName, Namespace: secretNamespace}, secret)
	if err != nil {
		log.Error(err, "Get Google Key from Secret to generate token")
		return errors.New("Secret Not Found")
		//	Try Create
		//err = cl.Create(ctx, kubeSecretFromTemplate(req.Namespace, "google-cloud-key"))
		//if err != nil {
		//	log.Error(err, "Secret:Create")
		//	return admission.Errored(http.StatusBadRequest, err)
		//}
	}
	log.Info("HandleUpdate:Secret", "secret", secret.Name)
	token := string(secret.Data["key.json"])

	appSecret := &corev1.Secret{}
	err = cl.Get(ctx, types.NamespacedName{Name: "google-cloud-token", Namespace: app.Namespace}, appSecret)
	if err != nil {
		if k8sapierrors.IsNotFound(err) {
			err = cl.Create(ctx, kubeSecretFromTemplate(app.Namespace, "google-cloud-token", "key.json", token))
			if err != nil {
				return err
			}
		}
		return err
	} else {
		appSecret.Data["key.json"] = []byte(token)
		err = cl.Update(ctx, appSecret)
		if err != nil {
			return err
		}
	}

	updateSecretsVolume(pod, "google-cloud-token")
	updateContainers(pod, "hello-app-pubsub", "google-auth-token",
		"/var/run/secrets/google/token", "GOOGLE_APPLICATION_CREDENTIALS")

	return nil

}

func getApplicationName(pod *corev1.Pod) (string, error) {
	if pod.Annotations == nil {
		return "", errors.New("Annotation not found, empty annotations")
	}

	if val, ok := pod.Annotations["appconfigmgr.cft.dev/application"]; ok {
		return val, nil
	}

	return "", errors.New("Annotation not found, empty annotations")
}

// podAnnotator adds an annotation to every incoming pods.
func (a *podAnnotator) Handle(ctx context.Context, req admission.Request) admission.Response {

	pod := &corev1.Pod{}

	log.Info("HandleUpdate:Start", req.Name, req.Namespace)

	err := a.decoder.Decode(req, pod)
	if err != nil {
		return admission.Errored(http.StatusBadRequest, err)
	}

	app := &appconfig.AppEnvConfigTemplateV2{}

	applicationName, err := getApplicationName(pod)
	if err != nil {
		log.Error(err, "Application annotation not found")

	}

	log.Info("HandleUpdate:applicationName", "applicationName", applicationName,
		"req.Namespace", req.Namespace, "req.Operation", req.Operation)

	err = localMgr.GetClient().Get(ctx, types.NamespacedName{Name: applicationName, Namespace: req.Namespace}, app)
	if err != nil {
		log.Error(err, "Application Does not Exist - working to see why it is not in scheme, hardcoded app to pubsub")
		//return admission.Errored(http.StatusBadRequest, err)
	}

	if req.Operation == "CREATE" {
		err := a.handleGCPSecretIfNeeded(ctx, pod, app)
		if err != nil {
			log.Error(err, "Application GCP Secret could not be handled see error")
			return admission.Errored(http.StatusBadRequest, err)
		}
	}

	if pod.Annotations == nil {
		pod.Annotations = map[string]string{}
	}
	pod.Annotations["example-mutating-admission-webhook"] = "foo"

	marshaledPod, err := json.Marshal(pod)
	if err != nil {
		return admission.Errored(http.StatusInternalServerError, err)
	}

	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
}

// podAnnotator implements inject.Client.
// A client will be automatically injected.

// InjectClient injects the client.
func (a *podAnnotator) InjectClient(c client.Client) error {
	a.client = c
	return nil
}

// podAnnotator implements admission.DecoderInjector.
// A decoder will be automatically injected.

// InjectDecoder injects the decoder.
func (a *podAnnotator) InjectDecoder(d *admission.Decoder) error {
	a.decoder = d
	return nil
}

// +kubebuilder:webhook:path=/validate-v1-pod,mutating=false,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod.appconfigmgr.cft.dev

// podValidator validates Pods
type podValidator struct {
	client  client.Client
	decoder *admission.Decoder
}

// podValidator admits a pod iff a specific annotation exists.
func (v *podValidator) Handle(ctx context.Context, req admission.Request) admission.Response {
	pod := &corev1.Pod{}

	err := v.decoder.Decode(req, pod)
	if err != nil {
		return admission.Errored(http.StatusBadRequest, err)
	}

	key := "example-mutating-admission-webhook"
	anno, found := pod.Annotations[key]
	if !found {
		return admission.Denied(fmt.Sprintf("missing annotation %s", key))
	}
	if anno != "foo" {
		return admission.Denied(fmt.Sprintf("annotation %s did not have value %q", key, "foo"))
	}

	return admission.Allowed("")
}

// podValidator implements inject.Client.
// A client will be automatically injected.

// InjectClient injects the client.
func (v *podValidator) InjectClient(c client.Client) error {
	v.client = c
	return nil
}

// podValidator implements admission.DecoderInjector.
// A decoder will be automatically injected.

// InjectDecoder injects the decoder.
func (v *podValidator) InjectDecoder(d *admission.Decoder) error {
	v.decoder = d
	return nil
}
<commit_msg>fix suggested type of error check<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package builtins

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	appconfig "github.com/GoogleCloudPlatform/anthos-appconfig/appconfigmgrv2/api/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sapierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/types"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/webhook"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

var log = ctrl.Log.WithName("webhooks-builtins-pod")
var TODO_FIND_NAMESPACE = "appconfigmgrv2-system"
var localMgr ctrl.Manager

func SetupWebHook(mgr ctrl.Manager) {

	// Setup webhooks
	// entryLog.Info("setting up webhook server")
	hookServer := mgr.GetWebhookServer()
	localMgr = mgr

	// entryLog.Info("registering webhooks to the webhook server")
	hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: &podAnnotator{}})
	hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{}})

}

// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=upod.appconfigmgr.cft.dev

// podAnnotator annotates Pods
type podAnnotator struct {
	client  client.Client
	decoder *admission.Decoder
}

//func getJSONKey(client client.Client, ns string, secret string) {
//	client.
//}

func kubeSecretFromTemplate(ns string, name string, mapKey string, value string) *corev1.Secret {
	return kubeSecretFromTemplateBytes(ns, name, mapKey, []byte(value))
}

func kubeSecretFromTemplateBytes(ns string, name string, mapKey string, value []byte) *corev1.Secret {
	return &corev1.Secret{
		Type: corev1.SecretTypeOpaque,
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: ns,
		},
		Data: map[string][]byte{
			mapKey: []byte(value),
		},
	}
}

func kubeSecretReponse(ns string, name string) *corev1.Secret {
	return &corev1.Secret{
		Type: corev1.SecretTypeOpaque,
	}
}

func getSecretName(ns string) string {
	log.Info("getSecretName:Start:" + "demo-" + ns + "-key")
	return "demo-" + ns + "-key"
}

func updateSecretsVolume(pod *corev1.Pod, secretName string) {
	log.V(1).Info("updateSecretsVolume", "secretName", secretName)
	found := false
	index := -1
	for i, element := range pod.Spec.Volumes {
		if element.Name == "google-auth-token" {
			log.V(1).Info("updateSecretsVolume:volumeFound", "element.Name", element.Name)
			found = true
			index = i
		}
	}
	element := &corev1.Volume{
		Name: "google-auth-token",
		VolumeSource: corev1.VolumeSource{
			Secret: &corev1.SecretVolumeSource{
				SecretName: secretName,
			},
		},
	}
	if !found {
		index = len(pod.Spec.Volumes)
		pod.Spec.Volumes = append(pod.Spec.Volumes, *element)
	} else {
		pod.Spec.Volumes[index] = *element
	}

}

func updateContainers(pod *corev1.Pod, containerName string, mountName string, mountPath string, envName string) {
	log.V(1).Info("updateContainers",
		"containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath,
	)

	for index, element := range pod.Spec.Containers {
		if element.Name == containerName {
			log.V(1).Info("updateContainers:found",
				"containerName", containerName,
				"mountName", mountName,
				"mountPath", mountPath)
			updateContainerMounts(&element, containerName, mountName, mountPath)
			updateContainerEnv(&element, containerName, envName, mountPath+"/key.json")
			pod.Spec.Containers[index] = element
			return
		}
	}

	//TODO - Decide how to fail or just warning
	log.Info("updateContainers:containerNotFound", "containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath)
}

func updateContainerMounts(container *corev1.Container, containerName string, mountName string, mountPath string) {
	log.V(1).Info("updateContainerMounts",
		"containerName", containerName,
		"mountName", mountName,
		"mountPath", mountPath,
	)
	found := false
	index := -1
	for i, element := range container.VolumeMounts {
		if element.Name == mountName {
			found = true
			index = i
			log.V(1).Info("updateContainerMounts:found",
				"containerName", containerName,
				"mountName", mountName,
				"mountPath", mountPath,
			)
		}
	}
	element := &corev1.VolumeMount{
		Name:      mountName,
		MountPath: mountPath,
	}

	if !found {
		// Append The Mount
		log.V(1).Info("updateContainerMounts:addMount",
			"containerName", containerName,
			"mountName", mountName,
			"mountPath", mountPath,
		)
		index = len(container.VolumeMounts)
		container.VolumeMounts = append(container.VolumeMounts, *element)
	} else {
		container.VolumeMounts[index] = *element
	}

	log.V(1).Info("updateContainerMounts:exit",
		"containerInfo", container,
	)

}

func updateContainerEnv(container *corev1.Container, containerName string, envName string, mountPath string) {
	log.V(1).Info("updateContainerEnv",
		"containerName", containerName,
		"envName", envName,
		"mountPath", mountPath,
	)
	found := false
	index := -1
	for i, element := range container.Env {
		if element.Name == envName {
			found = true
			index = i
			log.V(1).Info("updateContainerEnv:found",
				"containerName", containerName,
				"envName", envName,
				"mountPath", mountPath,
			)
		}
	}
	element := &corev1.EnvVar{
		Name:  envName,
		Value: mountPath,
	}

	if !found {
		// Append The Mount
		log.V(1).Info("updateContainerEnv:addMount",
			"containerName", containerName,
			"envName", envName,
			"mountPath", mountPath,
		)
		index = len(container.Env)
		container.Env = append(container.Env, *element)
	} else {
		container.Env[index] = *element
	}

	log.V(1).Info("updateContainerEnv:exit",
		"containerInfo", container,
	)

}

func (a *podAnnotator) handleGCPSecretIfNeeded(ctx context.Context, pod *corev1.Pod, app *appconfig.AppEnvConfigTemplateV2) error {

	log.Info("podAnnotator:handleGCPSecretIfNeeded")
	if app.Spec.Auth == nil ||
		app.Spec.Auth.GCPAccess == nil ||
		app.Spec.Auth.GCPAccess.AccessType != "secret" {

		return nil
	}

	secretName := app.Spec.Auth.GCPAccess.SecretInfo.Name
	secretNamespace := TODO_FIND_NAMESPACE
	secret := &corev1.Secret{}

	cl := localMgr.GetClient()
	err := cl.Get(ctx, types.NamespacedName{Name: secretName, Namespace: secretNamespace}, secret)
	if err != nil {
		log.Error(err, "Get Google Key from Secret to generate token")
		return errors.New("Secret Not Found")
		//	Try Create
		//err = cl.Create(ctx, kubeSecretFromTemplate(req.Namespace, "google-cloud-key"))
		//if err != nil {
		//	log.Error(err, "Secret:Create")
		//	return admission.Errored(http.StatusBadRequest, err)
		//}
	}
	log.Info("HandleUpdate:Secret", "secret", secret.Name)
	token := string(secret.Data["key.json"])

	appSecret := &corev1.Secret{}
	err = cl.Get(ctx, types.NamespacedName{Name: "google-cloud-token", Namespace: app.Namespace}, appSecret)
	if err != nil {
		// avoid using ! in compound statement due to readability
		if k8sapierrors.IsNotFound(err) {
			err = cl.Create(ctx, kubeSecretFromTemplate(app.Namespace, "google-cloud-token", "key.json", token))
			if err != nil {
				return err
			}
		} else {
			return err
		}
	} else {
		appSecret.Data["key.json"] = []byte(token)
		err = cl.Update(ctx, appSecret)
		if err != nil {
			return err
		}
	}

	updateSecretsVolume(pod, "google-cloud-token")
	updateContainers(pod, "hello-app-pubsub", "google-auth-token",
		"/var/run/secrets/google/token", "GOOGLE_APPLICATION_CREDENTIALS")

	return nil

}

func getApplicationName(pod *corev1.Pod) (string, error) {
	if pod.Annotations == nil {
		return "", errors.New("Annotation not found, empty annotations")
	}

	if val, ok := pod.Annotations["appconfigmgr.cft.dev/application"]; ok {
		return val, nil
	}

	return "", errors.New("Annotation not found, empty annotations")
}

// podAnnotator adds an annotation to every incoming pods.
func (a *podAnnotator) Handle(ctx context.Context, req admission.Request) admission.Response {

	pod := &corev1.Pod{}

	log.Info("HandleUpdate:Start", req.Name, req.Namespace)

	err := a.decoder.Decode(req, pod)
	if err != nil {
		return admission.Errored(http.StatusBadRequest, err)
	}

	app := &appconfig.AppEnvConfigTemplateV2{}

	applicationName, err := getApplicationName(pod)
	if err != nil {
		log.Error(err, "Application annotation not found")

	}

	log.Info("HandleUpdate:applicationName", "applicationName", applicationName,
		"req.Namespace", req.Namespace, "req.Operation", req.Operation)

	err = localMgr.GetClient().Get(ctx, types.NamespacedName{Name: applicationName, Namespace: req.Namespace}, app)
	if err != nil {
		log.Error(err, "Application Does not Exist - working to see why it is not in scheme, hardcoded app to pubsub")
		//return admission.Errored(http.StatusBadRequest, err)
	}

	if req.Operation == "CREATE" {
		err := a.handleGCPSecretIfNeeded(ctx, pod, app)
		if err != nil {
			log.Error(err, "Application GCP Secret could not be handled see error")
			return admission.Errored(http.StatusBadRequest, err)
		}
	}

	if pod.Annotations == nil {
		pod.Annotations = map[string]string{}
	}
	pod.Annotations["example-mutating-admission-webhook"] = "foo"

	marshaledPod, err := json.Marshal(pod)
	if err != nil {
		return admission.Errored(http.StatusInternalServerError, err)
	}

	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
}

// podAnnotator implements inject.Client.
// A client will be automatically injected.

// InjectClient injects the client.
func (a *podAnnotator) InjectClient(c client.Client) error {
	a.client = c
	return nil
}

// podAnnotator implements admission.DecoderInjector.
// A decoder will be automatically injected.

// InjectDecoder injects the decoder.
func (a *podAnnotator) InjectDecoder(d *admission.Decoder) error {
	a.decoder = d
	return nil
}

// +kubebuilder:webhook:path=/validate-v1-pod,mutating=false,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod.appconfigmgr.cft.dev

// podValidator validates Pods
type podValidator struct {
	client  client.Client
	decoder *admission.Decoder
}

// podValidator admits a pod iff a specific annotation exists.
func (v *podValidator) Handle(ctx context.Context, req admission.Request) admission.Response {
	pod := &corev1.Pod{}

	err := v.decoder.Decode(req, pod)
	if err != nil {
		return admission.Errored(http.StatusBadRequest, err)
	}

	key := "example-mutating-admission-webhook"
	anno, found := pod.Annotations[key]
	if !found {
		return admission.Denied(fmt.Sprintf("missing annotation %s", key))
	}
	if anno != "foo" {
		return admission.Denied(fmt.Sprintf("annotation %s did not have value %q", key, "foo"))
	}

	return admission.Allowed("")
}

// podValidator implements inject.Client.
// A client will be automatically injected.

// InjectClient injects the client.
func (v *podValidator) InjectClient(c client.Client) error {
	v.client = c
	return nil
}

// podValidator implements admission.DecoderInjector.
// A decoder will be automatically injected.

// InjectDecoder injects the decoder.
func (v *podValidator) InjectDecoder(d *admission.Decoder) error {
	v.decoder = d
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Richard Hawkins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ball TODO doc

package ball

import (
	"math"
	"runtime"

	"github.com/go-gl/mathgl/mgl32"
	"github.com/hurricanerix/shade/entity"
	"github.com/hurricanerix/shade/shapes"
	"github.com/hurricanerix/shade/sprite"
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
}

// Ball TODO doc
type Ball struct {
	Pos    mgl32.Vec3
	Sprite *sprite.Context
	Bounds *shapes.Shape
	dx     float32
	dy     float32
}

// New TODO doc
func New(x, y, speed, angle float32, s *sprite.Context, group *[]entity.Entity) (*Ball, error) {
	// TODO should take a group in as a argument
	b := Ball{
		Pos:    mgl32.Vec3{x, y, 1.0},
		Sprite: s,
		Bounds: shapes.NewCircle(mgl32.Vec2{float32(s.Width) / 2, float32(s.Height) / 2}, float32(s.Width)/2),
	}

	b.dx = float32(math.Cos(float64(angle))) * speed
	b.dy = float32(math.Sin(float64(angle))) * speed

	// TODO: this should probably be added outside of ball
	*group = append(*group, &b)
	return &b, nil
}

func (b Ball) Type() string {
	return "ball"
}

func (b Ball) Label() string {
	return ""
}

// Bind TODO doc
func (b *Ball) Bind(program uint32) error {
	return b.Sprite.Bind(program)
}

// Update TODO doc
func (b *Ball) Update(dt float32, g []entity.Entity) {
	lastPos := mgl32.Vec3{b.Pos[0], b.Pos[1], b.Pos[2]}

	b.Pos[0] += b.dx * dt
	b.Pos[1] += b.dy * dt

	newPos := &b.Pos

	switchDx := false
	switchDy := false

	for _, cell := range sprite.Collide(b, g, false) {
		println(cell)
		/*
			for cb := range cell.Bounds() {
				if lastR.Left() <= cb.Right() && lastR.Right() >= cb.Left() {
					switchDx = true
				}
				if lastR.Bottom() <= cb.Top() && lastR.Top() >= cb.Bottom() {
					switchDy = true
				}
			}
		*/
	}
	if switchDx {
		newPos[0] = lastPos[0]
		b.dx *= -1
	}
	if switchDy {
		newPos[1] = lastPos[1]
		b.dy *= -1
	}
}

// Draw TODO doc
func (b Ball) Draw() {
	b.Sprite.Draw(b.Pos, nil)
}
<commit_msg>Fix 03-collisions example<commit_after>// Copyright 2016 Richard Hawkins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ball TODO doc

package ball

import (
	"math"
	"runtime"

	"github.com/go-gl/mathgl/mgl32"
	"github.com/hurricanerix/shade/entity"
	"github.com/hurricanerix/shade/shapes"
	"github.com/hurricanerix/shade/sprite"
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
}

// Ball TODO doc
type Ball struct {
	Pos    mgl32.Vec3
	Sprite *sprite.Context
	Bounds *shapes.Shape
	dx     float32
	dy     float32
}

// New TODO doc
func New(x, y, speed, angle float32, s *sprite.Context, group *[]entity.Entity) (*Ball, error) {
	// TODO should take a group in as a argument
	b := Ball{
		Pos:    mgl32.Vec3{x, y, 1.0},
		Sprite: s,
		Bounds: shapes.NewCircle(mgl32.Vec2{float32(s.Width) / 2, float32(s.Height) / 2}, float32(s.Width)/2),
	}

	b.dx = float32(math.Cos(float64(angle))) * speed
	b.dy = float32(math.Sin(float64(angle))) * speed

	// TODO: this should probably be added outside of ball
	*group = append(*group, &b)
	return &b, nil
}

func (b Ball) Type() string {
	return "ball"
}

func (b Ball) Label() string {
	return ""
}

// Bind TODO doc
func (b *Ball) Bind(program uint32) error {
	return b.Sprite.Bind(program)
}

// Update TODO doc
func (b *Ball) Update(dt float32, g []entity.Entity) {
	lastPos := mgl32.Vec3{b.Pos[0], b.Pos[1], b.Pos[2]}

	b.Pos[0] += b.dx * dt
	b.Pos[1] += b.dy * dt

	newPos := &b.Pos

	switchDx := false
	switchDy := false

	for _, cell := range sprite.Collide(b, &g, false) {
		println(cell)
		/*
			for cb := range cell.Bounds() {
				if lastR.Left() <= cb.Right() && lastR.Right() >= cb.Left() {
					switchDx = true
				}
				if lastR.Bottom() <= cb.Top() && lastR.Top() >= cb.Bottom() {
					switchDy = true
				}
			}
		*/
	}
	if switchDx {
		newPos[0] = lastPos[0]
		b.dx *= -1
	}
	if switchDy {
		newPos[1] = lastPos[1]
		b.dy *= -1
	}
}

// Draw TODO doc
func (b Ball) Draw() {
	b.Sprite.Draw(b.Pos, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
)

// This server watches a list of git repos for changes and syncs the meta data of all commits
// to a BigTable backed datastore.

func main() {
	// Define the flags and parse them.
	var (
		btInstanceID    = flag.String("bt_instance", "production", "Big Table instance")
		btTableID       = flag.String("bt_table", "git-repos", "BigTable table ID")
		httpPort        = flag.String("http_port", ":9091", "The http port where ready-ness endpoints are served.")
		local           = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
		projectID       = flag.String("project", "skia-public", "ID of the GCP project")
		repoURLs        = common.NewMultiStringFlag("repo_url", []string{}, "Repo url")
		runInit         = flag.Bool("init", false, "Initialize the BigTable instance and quit. This should be run with a different different user who has admin rights.")
		refreshInterval = flag.Duration("refresh", 10*time.Minute, "Interval in which to poll git and refresh the GitStore.")
		workDir         = flag.String("workdir", "", "Working directory where repos are cached. Use the same directory between calls to speed up checkout time.")
	)
	common.Init()

	// Make sure we have a data directory and it exists or can be created.
	if *workDir == "" {
		sklog.Fatal("No workdir specified.")
	}
	useWorkDir := fileutil.Must(fileutil.EnsureDirExists(*workDir))

	// Make sure we have at least one repo configured.
	if len(*repoURLs) == 0 {
		sklog.Fatalf("At least one repository URL must be configured.")
	}

	// TODO(stephana): Pass the token source explicitly to the BigTable related functions below.

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Set up Git authentication if a service account email was set.
	if !*local {
		user, err := user.Current()
		if err != nil {
			sklog.Fatal(err)
		}

		// Use the gitcookie created by gitauth package if .gitcookies does not already exist.
		gitcookiesPath := filepath.Join(user.HomeDir, ".gitcookies")
		if _, err := gitauth.New(ts, gitcookiesPath, true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	// Configure the bigtable instance.
	config := &gitstore.BTConfig{
		ProjectID:  *projectID,
		InstanceID: *btInstanceID,
		TableID:    *btTableID,
	}

	// Initialize bigtable if invoked with --init and quit.
	// This should be invoked with a user that has admin privileges, so that the production user that
	// wants to write to the instance does not need admin privileges.
	if *runInit {
		if err := gitstore.InitBT(config); err != nil {
			sklog.Fatalf("Error initializing BT: %s", err)
		}
		sklog.Infof("BigTable instance %s and table %s in project %s initialized.", *btInstanceID, *btTableID, *projectID)
		return
	}

	// Start all repo watchers in the background.
	ctx := context.Background()
	for _, repoURL := range *repoURLs {
		go func(repoURL string) {
			repoDir, err := git.NormalizeURL(repoURL)
			if err != nil {
				sklog.Fatalf("Error getting normalized URL for %q:  %s", repoURL, err)
			}
			repoDir = strings.Replace(repoDir, "/", "_", -1)
			repoDir = filepath.Join(useWorkDir, repoDir)
			sklog.Infof("Checking out %s into %s", repoURL, repoDir)

			watcher, err := NewRepoWatcher(ctx, config, repoURL, repoDir)
			if err != nil {
				sklog.Fatalf("Error initializing repo watcher: %s", err)
			}
			watcher.Start(ctx, *refreshInterval)
		}(repoURL)
	}

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	log.Fatal(http.ListenAndServe(*httpPort, nil))
}
<commit_msg>[gitsync] Small fixes to make it work in K8s and more logging<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/gitstore"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/sklog"
)

// This server watches a list of git repos for changes and syncs the meta data of all commits
// to a BigTable backed datastore.

func main() {
	// Define the flags and parse them.
	var (
		btInstanceID    = flag.String("bt_instance", "production", "Big Table instance")
		btTableID       = flag.String("bt_table", "git-repos", "BigTable table ID")
		httpPort        = flag.String("http_port", ":9091", "The http port where ready-ness endpoints are served.")
		local           = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
		projectID       = flag.String("project", "skia-public", "ID of the GCP project")
		repoURLs        = common.NewMultiStringFlag("repo_url", []string{}, "Repo url")
		runInit         = flag.Bool("init", false, "Initialize the BigTable instance and quit. This should be run with a different different user who has admin rights.")
		refreshInterval = flag.Duration("refresh", 10*time.Minute, "Interval in which to poll git and refresh the GitStore.")
		workDir         = flag.String("workdir", "", "Working directory where repos are cached. Use the same directory between calls to speed up checkout time.")
	)
	common.Init()

	// Make sure we have a data directory and it exists or can be created.
	if *workDir == "" {
		sklog.Fatal("No workdir specified.")
	}
	useWorkDir := fileutil.Must(fileutil.EnsureDirExists(*workDir))

	// Make sure we have at least one repo configured.
	if len(*repoURLs) == 0 {
		sklog.Fatalf("At least one repository URL must be configured.")
	}

	// TODO(stephana): Pass the token source explicitly to the BigTable related functions below.

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Set up Git authentication if a service account email was set.
	if !*local {
		// Use the gitcookie created by the gitauth package.
		gitcookiesPath := "/tmp/gitcookies"
		sklog.Infof("Writing gitcookies to %s", gitcookiesPath)
		if _, err := gitauth.New(ts, gitcookiesPath, true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	// Configure the bigtable instance.
	config := &gitstore.BTConfig{
		ProjectID:  *projectID,
		InstanceID: *btInstanceID,
		TableID:    *btTableID,
	}

	// Initialize bigtable if invoked with --init and quit.
	// This should be invoked with a user that has admin privileges, so that the production user that
	// wants to write to the instance does not need admin privileges.
	if *runInit {
		if err := gitstore.InitBT(config); err != nil {
			sklog.Fatalf("Error initializing BT: %s", err)
		}
		sklog.Infof("BigTable instance %s and table %s in project %s initialized.", *btInstanceID, *btTableID, *projectID)
		return
	}

	// Start all repo watchers in the background.
	ctx := context.Background()
	for _, repoURL := range *repoURLs {
		go func(repoURL string) {
			repoDir, err := git.NormalizeURL(repoURL)
			if err != nil {
				sklog.Fatalf("Error getting normalized URL for %q:  %s", repoURL, err)
			}
			repoDir = strings.Replace(repoDir, "/", "_", -1)
			repoDir = filepath.Join(useWorkDir, repoDir)
			sklog.Infof("Checking out %s into %s", repoURL, repoDir)

			watcher, err := NewRepoWatcher(ctx, config, repoURL, repoDir)
			if err != nil {
				sklog.Fatalf("Error initializing repo watcher: %s", err)
			}
			watcher.Start(ctx, *refreshInterval)
		}(repoURL)
	}

	// Set up the http handler to indicate ready-ness and start serving.
	http.HandleFunc("/healthz", httputils.ReadyHandleFunc)
	sklog.Infof("Listening on port: %s", *httpPort)
	log.Fatal(http.ListenAndServe(*httpPort, nil))
}
<|endoftext|>


<commit_before>package elasticthought

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/logg"
)

// Worker job that splits a dataset into training/test set
type DatasetSplitter struct {
	Configuration Configuration
	Dataset       Dataset
}

// Run this job
func (d DatasetSplitter) Run() {

	switch d.Dataset.isSplittable() {
	case true:
		d.SplitDatafile()
	default:
		d.DownloadDatafiles()
	}

}

func (d DatasetSplitter) SplitDatafile() {

	// Find the datafile object associated with dataset
	db := d.Configuration.DbConnection()
	datafile, err := d.Dataset.GetSplittableDatafile(db)
	if err != nil {
		errMsg := fmt.Errorf("Error looking up datafile: %+v.  Error: %v", d.Dataset, err)
		d.recordProcessingError(errMsg)
		return
	}

	// Open the url -- content type should be application/x-gzip
	tr, err := openTarGzStream(datafile.Url)
	if err != nil {
		errMsg := fmt.Errorf("Error opening tar.gz streams: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// Create pipes
	prTrain, pwTrain := io.Pipe()
	prTest, pwTest := io.Pipe()

	// Wrap in gzip writers
	pwGzTest := gzip.NewWriter(pwTest)
	pwGzTrain := gzip.NewWriter(pwTrain)

	// Create tar writers on the write end of the pipes
	tarWriterTesting := tar.NewWriter(pwGzTest)
	tarWriterTraining := tar.NewWriter(pwGzTrain)

	// Create a cbfs client
	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// Figure out where to store these on cbfs
	destTraining := d.Dataset.TrainingArtifactPath()
	destTesting := d.Dataset.TestingArtifactPath()

	// Spawn a goroutine that will read from tar.gz reader coming from url data
	// and write to the training and test tar writers (which are on write ends of pipe)
	transformDoneChan := make(chan error, 1)
	go func() {

		// Must close _underlying_ piped writers, or the piped readers will
		// never get an EOF.  Closing just the tar writers that wrap the underlying
		// piped writers is not enough.
		defer pwTest.Close()
		defer pwTrain.Close()
		defer pwGzTest.Close()
		defer pwGzTrain.Close()

		logg.LogTo("DATASET_SPLITTER", "Calling transform")
		err = d.transform(tr, tarWriterTraining, tarWriterTesting)
		if err != nil {
			errMsg := fmt.Errorf("Error transforming tar stream: %v", err)
			logg.LogError(errMsg)
			transformDoneChan <- errMsg
			return
		}

		transformDoneChan <- nil

	}()

	// Spawn goroutines to read off the read ends of the pipe and store in cbfs
	cbfsTrainDoneChan := make(chan error, 1)
	cbfsTestDoneChan := make(chan error, 1)
	go func() {
		if err := cbfs.Put("", destTesting, prTest, options); err != nil {
			errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTesting, err)
			logg.LogError(errMsg)
			cbfsTestDoneChan <- errMsg
			return

		}
		logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTesting)
		cbfsTestDoneChan <- nil
	}()
	go func() {
		if err := cbfs.Put("", destTraining, prTrain, options); err != nil {
			errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTraining, err)
			logg.LogError(errMsg)
			cbfsTrainDoneChan <- errMsg
			return
		}
		logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTraining)
		cbfsTrainDoneChan <- nil
	}()

	// Wait for the results from all the goroutines
	cbfsTrainResult := <-cbfsTrainDoneChan
	cbfsTestResult := <-cbfsTestDoneChan
	transformResult := <-transformDoneChan

	// If any results had an error, log it and return
	results := []error{transformResult, cbfsTestResult, cbfsTrainResult}
	for _, result := range results {
		if result != nil {
			logg.LogTo("DATASET_SPLITTER", "Setting dataset to failed: %v", result)
			d.Dataset.Failed(db, fmt.Errorf("%v", result))
			return
		}
	}

	// Update the state of the dataset to be finished
	d.Dataset.FinishedSuccessfully(db)
	if err := d.Dataset.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking dataset %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

func (d DatasetSplitter) DownloadDatafiles() {

	// Create a cbfs client
	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	db := d.Configuration.DbConnection()

	source2destEntries := []struct {
		Url      string
		DestPath string
	}{
		{
			Url:      d.Dataset.GetTrainingDatafileUrl(db),
			DestPath: d.Dataset.TrainingArtifactPath(),
		},
		{
			Url:      d.Dataset.GetTestingDatafileUrl(db),
			DestPath: d.Dataset.TestingArtifactPath(),
		},
	}

	for _, source2destEntry := range source2destEntries {

		// open tar.gz stream to source
		// Open the url -- content type should be application/x-gzip

		func() {

			resp, err := http.Get(source2destEntry.Url)
			defer resp.Body.Close()

			if err != nil {
				errMsg := fmt.Errorf("Error opening stream to: %v. Err %v", source2destEntry.Url, err)
				d.recordProcessingError(errMsg)
				return
			}

			if err := cbfs.Put("", source2destEntry.DestPath, resp.Body, options); err != nil {
				errMsg := fmt.Errorf("Error writing %v to cbfs: %v", source2destEntry.DestPath, err)
				d.recordProcessingError(errMsg)
				return
			}

			logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", source2destEntry.DestPath)

		}()

	}

	// Update the state of the dataset to be finished
	if err := d.Dataset.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking dataset %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

// Codereview: de-dupe
func (d DatasetSplitter) recordProcessingError(err error) {
	logg.LogError(err)
	db := d.Configuration.DbConnection()
	if err := d.Dataset.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting dataset as failed: %v", err)
		logg.LogError(errMsg)
	}
}

// Read from source tar stream and write training and test to given tar writers
func (d DatasetSplitter) transform(source *tar.Reader, train, test *tar.Writer) error {

	splitter := d.splitter(train, test)

	for {
		hdr, err := source.Next()

		if err == io.EOF {
			// end of tar archive
			break
		}

		if err != nil {
			return err
		}

		tw := splitter(hdr.Name)

		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		_, err = io.Copy(tw, source)
		if err != nil {
			return err
		}

	}

	// close writers
	if err := train.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar writer: %v", err)
		logg.LogError(errMsg)
		return err
	}
	if err := test.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar reader: %v", err)
		logg.LogError(errMsg)
		return err
	}

	return nil
}

func (d DatasetSplitter) splitter(train, test *tar.Writer) func(string) *tar.Writer {

	trainingRatio := int(d.Dataset.TrainingDataset.SplitPercentage * 100)
	testRatio := int(d.Dataset.TestDataset.SplitPercentage * 100)

	ratio := [2]int{trainingRatio, testRatio}

	dircounts := make(map[string][2]int)
	return func(file string) *tar.Writer {
		dir := path.Dir(file)
		counts := dircounts[dir]
		count0ratio1 := counts[0] * ratio[1]
		count1ratio0 := counts[1] * ratio[0]
		if count0ratio1 <= count1ratio0 {
			counts[0]++
			dircounts[dir] = counts
			return train
		} else {
			counts[1]++
			dircounts[dir] = counts
			return test
		}
	}
}

// Validate that the source tar stream conforms to expected specs
func (d DatasetSplitter) validate(source *tar.Reader) (bool, error) {

	// validation rules:
	// 1. has at least 2 files
	// 2. the depth of each file is 2 (folder/filename.xxx)

	numFiles := 0
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return false, err
		}
		numFiles += 1

		pathComponents := strings.Split(hdr.Name, "/")
		if len(pathComponents) != 2 {
			return false, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

	}

	if numFiles < 2 {
		return false, fmt.Errorf("Archive must contain at least 2 files")
	}

	return true, nil
}
<commit_msg>only close body if no error<commit_after>package elasticthought

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/logg"
)

// Worker job that splits a dataset into training/test set
type DatasetSplitter struct {
	Configuration Configuration
	Dataset       Dataset
}

// Run this job
func (d DatasetSplitter) Run() {

	switch d.Dataset.isSplittable() {
	case true:
		d.SplitDatafile()
	default:
		d.DownloadDatafiles()
	}

}

func (d DatasetSplitter) SplitDatafile() {

	// Find the datafile object associated with dataset
	db := d.Configuration.DbConnection()
	datafile, err := d.Dataset.GetSplittableDatafile(db)
	if err != nil {
		errMsg := fmt.Errorf("Error looking up datafile: %+v.  Error: %v", d.Dataset, err)
		d.recordProcessingError(errMsg)
		return
	}

	// Open the url -- content type should be application/x-gzip
	tr, err := openTarGzStream(datafile.Url)
	if err != nil {
		errMsg := fmt.Errorf("Error opening tar.gz streams: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// Create pipes
	prTrain, pwTrain := io.Pipe()
	prTest, pwTest := io.Pipe()

	// Wrap in gzip writers
	pwGzTest := gzip.NewWriter(pwTest)
	pwGzTrain := gzip.NewWriter(pwTrain)

	// Create tar writers on the write end of the pipes
	tarWriterTesting := tar.NewWriter(pwGzTest)
	tarWriterTraining := tar.NewWriter(pwGzTrain)

	// Create a cbfs client
	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// Figure out where to store these on cbfs
	destTraining := d.Dataset.TrainingArtifactPath()
	destTesting := d.Dataset.TestingArtifactPath()

	// Spawn a goroutine that will read from tar.gz reader coming from url data
	// and write to the training and test tar writers (which are on write ends of pipe)
	transformDoneChan := make(chan error, 1)
	go func() {

		// Must close _underlying_ piped writers, or the piped readers will
		// never get an EOF.  Closing just the tar writers that wrap the underlying
		// piped writers is not enough.
		defer pwTest.Close()
		defer pwTrain.Close()
		defer pwGzTest.Close()
		defer pwGzTrain.Close()

		logg.LogTo("DATASET_SPLITTER", "Calling transform")
		err = d.transform(tr, tarWriterTraining, tarWriterTesting)
		if err != nil {
			errMsg := fmt.Errorf("Error transforming tar stream: %v", err)
			logg.LogError(errMsg)
			transformDoneChan <- errMsg
			return
		}

		transformDoneChan <- nil

	}()

	// Spawn goroutines to read off the read ends of the pipe and store in cbfs
	cbfsTrainDoneChan := make(chan error, 1)
	cbfsTestDoneChan := make(chan error, 1)
	go func() {
		if err := cbfs.Put("", destTesting, prTest, options); err != nil {
			errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTesting, err)
			logg.LogError(errMsg)
			cbfsTestDoneChan <- errMsg
			return

		}
		logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTesting)
		cbfsTestDoneChan <- nil
	}()
	go func() {
		if err := cbfs.Put("", destTraining, prTrain, options); err != nil {
			errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTraining, err)
			logg.LogError(errMsg)
			cbfsTrainDoneChan <- errMsg
			return
		}
		logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTraining)
		cbfsTrainDoneChan <- nil
	}()

	// Wait for the results from all the goroutines
	cbfsTrainResult := <-cbfsTrainDoneChan
	cbfsTestResult := <-cbfsTestDoneChan
	transformResult := <-transformDoneChan

	// If any results had an error, log it and return
	results := []error{transformResult, cbfsTestResult, cbfsTrainResult}
	for _, result := range results {
		if result != nil {
			logg.LogTo("DATASET_SPLITTER", "Setting dataset to failed: %v", result)
			d.Dataset.Failed(db, fmt.Errorf("%v", result))
			return
		}
	}

	// Update the state of the dataset to be finished
	d.Dataset.FinishedSuccessfully(db)
	if err := d.Dataset.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking dataset %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

func (d DatasetSplitter) DownloadDatafiles() {

	// Create a cbfs client
	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	db := d.Configuration.DbConnection()

	source2destEntries := []struct {
		Url      string
		DestPath string
	}{
		{
			Url:      d.Dataset.GetTrainingDatafileUrl(db),
			DestPath: d.Dataset.TrainingArtifactPath(),
		},
		{
			Url:      d.Dataset.GetTestingDatafileUrl(db),
			DestPath: d.Dataset.TestingArtifactPath(),
		},
	}

	for _, source2destEntry := range source2destEntries {

		// open tar.gz stream to source
		// Open the url -- content type should be application/x-gzip

		func() {

			resp, err := http.Get(source2destEntry.Url)

			if err != nil {
				errMsg := fmt.Errorf("Error opening stream to: %v. Err %v", source2destEntry.Url, err)
				d.recordProcessingError(errMsg)
				return
			}
			defer resp.Body.Close()

			if err := cbfs.Put("", source2destEntry.DestPath, resp.Body, options); err != nil {
				errMsg := fmt.Errorf("Error writing %v to cbfs: %v", source2destEntry.DestPath, err)
				d.recordProcessingError(errMsg)
				return
			}

			logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", source2destEntry.DestPath)

		}()

	}

	// Update the state of the dataset to be finished
	if err := d.Dataset.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking dataset %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

// Codereview: de-dupe
func (d DatasetSplitter) recordProcessingError(err error) {
	logg.LogError(err)
	db := d.Configuration.DbConnection()
	if err := d.Dataset.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting dataset as failed: %v", err)
		logg.LogError(errMsg)
	}
}

// Read from source tar stream and write training and test to given tar writers
func (d DatasetSplitter) transform(source *tar.Reader, train, test *tar.Writer) error {

	splitter := d.splitter(train, test)

	for {
		hdr, err := source.Next()

		if err == io.EOF {
			// end of tar archive
			break
		}

		if err != nil {
			return err
		}

		tw := splitter(hdr.Name)

		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		_, err = io.Copy(tw, source)
		if err != nil {
			return err
		}

	}

	// close writers
	if err := train.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar writer: %v", err)
		logg.LogError(errMsg)
		return err
	}
	if err := test.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar reader: %v", err)
		logg.LogError(errMsg)
		return err
	}

	return nil
}

func (d DatasetSplitter) splitter(train, test *tar.Writer) func(string) *tar.Writer {

	trainingRatio := int(d.Dataset.TrainingDataset.SplitPercentage * 100)
	testRatio := int(d.Dataset.TestDataset.SplitPercentage * 100)

	ratio := [2]int{trainingRatio, testRatio}

	dircounts := make(map[string][2]int)
	return func(file string) *tar.Writer {
		dir := path.Dir(file)
		counts := dircounts[dir]
		count0ratio1 := counts[0] * ratio[1]
		count1ratio0 := counts[1] * ratio[0]
		if count0ratio1 <= count1ratio0 {
			counts[0]++
			dircounts[dir] = counts
			return train
		} else {
			counts[1]++
			dircounts[dir] = counts
			return test
		}
	}
}

// Validate that the source tar stream conforms to expected specs
func (d DatasetSplitter) validate(source *tar.Reader) (bool, error) {

	// validation rules:
	// 1. has at least 2 files
	// 2. the depth of each file is 2 (folder/filename.xxx)

	numFiles := 0
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return false, err
		}
		numFiles += 1

		pathComponents := strings.Split(hdr.Name, "/")
		if len(pathComponents) != 2 {
			return false, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

	}

	if numFiles < 2 {
		return false, fmt.Errorf("Archive must contain at least 2 files")
	}

	return true, nil
}
<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/kirves/go-form-it"
	"github.com/kirves/go-form-it/fields"
	"sort"
	"strconv"
	"strings"
	"time"
)

type Bag struct {
	gorm.Model       `form_options:"skip"`
	Name             string
	Size             int64
	Files            []File    `form_options:"skip"`
	StorageURL       string    `form_options:"skip"`
	MetadataURL      string    `form_options:"skip"`
	RemoteIdentifier string    `form_options:"skip"`
	StoredAt         time.Time `form_options:"skip"`
}

type File struct {
	gorm.Model        `form_options:"skip"`
	BagID             int64
	Name              string
	Size              int64
	Md5               string    `form_options:"skip"`
	Sha256            string    `form_options:"skip"`
	StorageURL        string    `form_options:"skip"`
	StoredAsPartOfBag bool      `form_options:"skip"`
	ETag              string    `form_options:"skip"`
	StoredAt          time.Time `form_options:"skip"`
}

type BagItProfile struct {
	gorm.Model       `form_options:"skip"`
	Name             string
	Description      string
	JSON             string            `form_widget:"textarea"`
	DefaultTagValues []DefaultTagValue `form_options:"skip"`
}

func (profile *BagItProfile) Profile() (*bagit.BagItProfile, error) {
	bagItProfile := &bagit.BagItProfile{}
	err := json.Unmarshal([]byte(profile.JSON), bagItProfile)
	return bagItProfile, err
}

func (profile *BagItProfile) GetForm() (*forms.Form, error) {
	postUrl := fmt.Sprintf("/profile/new")
	if profile.ID > uint(0) {
		postUrl = fmt.Sprintf("/profile/%d/edit", profile.ID)
	}
	form := forms.BootstrapFormFromModel(*profile, forms.POST, postUrl)
	form.Field("JSON").AddCss("height", "300px")
	if profile.JSON == "" {
		return form, nil
	}
	profileDef, err := profile.Profile()
	if err != nil {
		return nil, err
	}

	// TODO: Move sort into BagItProfile object.
	sortedFileNames := make([]string, len(profileDef.TagFilesRequired))
	i := 0
	for relFilePath, _ := range profileDef.TagFilesRequired {
		sortedFileNames[i] = relFilePath
		i++
	}
	sort.Strings(sortedFileNames)

	// Remove the submit button from the end of the form,
	// add our new elements, and then replace the submit button
	// at the end.
	submitButton := form.Field("submit")
	form.RemoveElement("submit")

	for _, relFilePath := range sortedFileNames {
		fieldsInSet := make([]fields.FieldInterface, 0)
		mapOfRequiredTags := profileDef.TagFilesRequired[relFilePath]
		// TODO: Move sort into BagItProfile object.
		sortedTagNames := make([]string, len(mapOfRequiredTags))
		i := 0
		for tagname, _ := range mapOfRequiredTags {
			sortedTagNames[i] = tagname
			i++
		}
		sort.Strings(sortedTagNames)
		for _, tagname := range sortedTagNames {
			tagdef := mapOfRequiredTags[tagname]
			defaultTags := profile.GetDefaultTagValues(relFilePath, tagname)
			defaultValue := ""
			defaultTagId := uint(0)
			if len(defaultTags) > 0 {
				defaultValue = defaultTags[0].TagValue
				defaultTagId = defaultTags[0].ID
			}
			fieldName := fmt.Sprintf("%s|%s|%d", relFilePath, tagname, defaultTagId)
			fieldLabel := tagname

			formField := fields.TextField(fieldName)
			if len(tagdef.Values) > 0 {
				options := make(map[string][]fields.InputChoice)
				options[""] = make([]fields.InputChoice, len(tagdef.Values))
				for i, val := range tagdef.Values {
					options[""][i] = fields.InputChoice{Id: val, Val: val}
				}
				formField = fields.SelectField(fieldName, options)
			}
			formField.SetLabel(fieldLabel)
			formField.SetValue(defaultValue)
			fieldsInSet = append(fieldsInSet, formField)
		}
		// Unfortunately, go-form-it does not support fieldset legends
		fieldSetLabel := fields.StaticField("", fmt.Sprintf("Default values for %s", relFilePath))
		fieldSetLabel.AddClass("fieldset-header")
		form.Elements(fieldSetLabel)
		fieldSet := forms.FieldSet(relFilePath, fieldsInSet...)
		form.Elements(fieldSet)
	}
	form.Elements(submitButton)

	return form, nil
}

func (profile *BagItProfile) GetDefaultTagValues(tagFile, tagName string) []DefaultTagValue {
	defaults := make([]DefaultTagValue, 0)
	for _, dtv := range profile.DefaultTagValues {
		if dtv.TagFile == tagFile && dtv.TagName == tagName {
			defaults = append(defaults, dtv)
		}
	}
	return defaults
}

func (profile *BagItProfile) DecodeDefaultTagValues(data map[string][]string) []DefaultTagValue {
	values := make([]DefaultTagValue, 0)
	for key, value := range data {
		if !strings.Contains(key, "|") {
			continue
		}
		keyParts := strings.Split(key, "|")
		//tagFile, tagName, tagId
		dtv := DefaultTagValue{
			BagItProfileID: int64(profile.ID), // TODO: int64 or uint??
			TagFile:        keyParts[0],
			TagName:        keyParts[1],
			TagValue:       value[0],
		}
		id, _ := strconv.Atoi(keyParts[2])
		dtv.ID = uint(id)
		values = append(values, dtv)
	}
	return values
}

type DefaultTagValue struct {
	gorm.Model     `form_options:"skip"`
	BagItProfileID int64
	TagFile        string
	TagName        string
	TagValue       string
}

type AppSetting struct {
	gorm.Model `form_options:"skip"`
	Name       string
	Value      string
}

type Job struct {
	gorm.Model         `form_options:"skip"`
	BagID              int64     `form_options:"skip"`
	FileID             int64     `form_options:"skip"`
	WorkflowID         int64     `form_widget:"select"`
	WorkflowSnapshot   string    `form_options:"skip"`
	ScheduledStartTime time.Time `form_options:"skip"`
	StartedAt          time.Time `form_options:"skip"`
	FinishedAt         time.Time `form_options:"skip"`
	Pid                int       `form_options:"skip"`
	Outcome            string    `form_options:"skip"`
	CapturedOutput     string    `form_options:"skip"`
}

type StorageService struct {
	gorm.Model     `form_options:"skip"`
	Name           string
	Description    string
	Protocol       string `form_widget:"select"`
	URL            string
	BucketOrFolder string
	LoginName      string
	LoginPassword  string `form_widget:"password"`
	LoginExtra     string `form_widget:"password"`
}

type Tag struct {
	gorm.Model  `form_options:"skip"`
	BagID       int64 `form_widget:"select"`
	RelFilePath string
	Name        string
	Value       string
}

type Workflow struct {
	gorm.Model          `form_options:"skip"`
	Name                string
	Description         string
	SerializationFormat string `form_widget:"select"`
	BagItProfileID      int64  `form_widget:"select"`
	StorageServiceID    int64  `form_widget:"select"`
}
<commit_msg>Use new sort methods from BagItProfile<commit_after>package models

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/kirves/go-form-it"
	"github.com/kirves/go-form-it/fields"
	"strconv"
	"strings"
	"time"
)

type Bag struct {
	gorm.Model       `form_options:"skip"`
	Name             string
	Size             int64
	Files            []File    `form_options:"skip"`
	StorageURL       string    `form_options:"skip"`
	MetadataURL      string    `form_options:"skip"`
	RemoteIdentifier string    `form_options:"skip"`
	StoredAt         time.Time `form_options:"skip"`
}

type File struct {
	gorm.Model        `form_options:"skip"`
	BagID             int64
	Name              string
	Size              int64
	Md5               string    `form_options:"skip"`
	Sha256            string    `form_options:"skip"`
	StorageURL        string    `form_options:"skip"`
	StoredAsPartOfBag bool      `form_options:"skip"`
	ETag              string    `form_options:"skip"`
	StoredAt          time.Time `form_options:"skip"`
}

type BagItProfile struct {
	gorm.Model       `form_options:"skip"`
	Name             string
	Description      string
	JSON             string            `form_widget:"textarea"`
	DefaultTagValues []DefaultTagValue `form_options:"skip"`
}

func (profile *BagItProfile) Profile() (*bagit.BagItProfile, error) {
	bagItProfile := &bagit.BagItProfile{}
	err := json.Unmarshal([]byte(profile.JSON), bagItProfile)
	return bagItProfile, err
}

func (profile *BagItProfile) GetForm() (*forms.Form, error) {
	postUrl := fmt.Sprintf("/profile/new")
	if profile.ID > uint(0) {
		postUrl = fmt.Sprintf("/profile/%d/edit", profile.ID)
	}
	form := forms.BootstrapFormFromModel(*profile, forms.POST, postUrl)
	form.Field("JSON").AddCss("height", "300px")
	if profile.JSON == "" {
		return form, nil
	}
	profileDef, err := profile.Profile()
	if err != nil {
		return nil, err
	}

	// Remove the submit button from the end of the form,
	// add our new elements, and then replace the submit button
	// at the end.
	submitButton := form.Field("submit")
	form.RemoveElement("submit")

	for _, relFilePath := range profileDef.SortedTagFilesRequired() {
		fieldsInSet := make([]fields.FieldInterface, 0)
		mapOfRequiredTags := profileDef.TagFilesRequired[relFilePath]
		for _, tagname := range profileDef.SortedTagNames(relFilePath) {
			tagdef := mapOfRequiredTags[tagname]
			defaultTags := profile.GetDefaultTagValues(relFilePath, tagname)
			defaultValue := ""
			defaultTagId := uint(0)
			if len(defaultTags) > 0 {
				defaultValue = defaultTags[0].TagValue
				defaultTagId = defaultTags[0].ID
			}
			fieldName := fmt.Sprintf("%s|%s|%d", relFilePath, tagname, defaultTagId)
			fieldLabel := tagname

			formField := fields.TextField(fieldName)
			if len(tagdef.Values) > 0 {
				options := make(map[string][]fields.InputChoice)
				options[""] = make([]fields.InputChoice, len(tagdef.Values))
				for i, val := range tagdef.Values {
					options[""][i] = fields.InputChoice{Id: val, Val: val}
				}
				formField = fields.SelectField(fieldName, options)
			}
			formField.SetLabel(fieldLabel)
			formField.SetValue(defaultValue)
			fieldsInSet = append(fieldsInSet, formField)
		}
		// Unfortunately, go-form-it does not support fieldset legends
		fieldSetLabel := fields.StaticField("", fmt.Sprintf("Default values for %s", relFilePath))
		fieldSetLabel.AddClass("fieldset-header")
		form.Elements(fieldSetLabel)
		fieldSet := forms.FieldSet(relFilePath, fieldsInSet...)
		form.Elements(fieldSet)
	}
	form.Elements(submitButton)

	return form, nil
}

func (profile *BagItProfile) GetDefaultTagValues(tagFile, tagName string) []DefaultTagValue {
	defaults := make([]DefaultTagValue, 0)
	for _, dtv := range profile.DefaultTagValues {
		if dtv.TagFile == tagFile && dtv.TagName == tagName {
			defaults = append(defaults, dtv)
		}
	}
	return defaults
}

func (profile *BagItProfile) DecodeDefaultTagValues(data map[string][]string) []DefaultTagValue {
	values := make([]DefaultTagValue, 0)
	for key, value := range data {
		if !strings.Contains(key, "|") {
			continue
		}
		keyParts := strings.Split(key, "|")
		//tagFile, tagName, tagId
		dtv := DefaultTagValue{
			BagItProfileID: int64(profile.ID), // TODO: int64 or uint??
			TagFile:        keyParts[0],
			TagName:        keyParts[1],
			TagValue:       value[0],
		}
		id, _ := strconv.Atoi(keyParts[2])
		dtv.ID = uint(id)
		values = append(values, dtv)
	}
	return values
}

type DefaultTagValue struct {
	gorm.Model     `form_options:"skip"`
	BagItProfileID int64
	TagFile        string
	TagName        string
	TagValue       string
}

type AppSetting struct {
	gorm.Model `form_options:"skip"`
	Name       string
	Value      string
}

type Job struct {
	gorm.Model         `form_options:"skip"`
	BagID              int64     `form_options:"skip"`
	FileID             int64     `form_options:"skip"`
	WorkflowID         int64     `form_widget:"select"`
	WorkflowSnapshot   string    `form_options:"skip"`
	ScheduledStartTime time.Time `form_options:"skip"`
	StartedAt          time.Time `form_options:"skip"`
	FinishedAt         time.Time `form_options:"skip"`
	Pid                int       `form_options:"skip"`
	Outcome            string    `form_options:"skip"`
	CapturedOutput     string    `form_options:"skip"`
}

type StorageService struct {
	gorm.Model     `form_options:"skip"`
	Name           string
	Description    string
	Protocol       string `form_widget:"select"`
	URL            string
	BucketOrFolder string
	LoginName      string
	LoginPassword  string `form_widget:"password"`
	LoginExtra     string `form_widget:"password"`
}

type Tag struct {
	gorm.Model  `form_options:"skip"`
	BagID       int64 `form_widget:"select"`
	RelFilePath string
	Name        string
	Value       string
}

type Workflow struct {
	gorm.Model          `form_options:"skip"`
	Name                string
	Description         string
	SerializationFormat string `form_widget:"select"`
	BagItProfileID      int64  `form_widget:"select"`
	StorageServiceID    int64  `form_widget:"select"`
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"
)

func (db *SQLDB) InsertVolume(data Volume) error {
	tx, err := db.conn.Begin()
	if err != nil {
		return err
	}

	defer tx.Rollback()

	var resourceVersion []byte

	columns := []string{"worker_name", "ttl", "handle"}
	params := []interface{}{data.WorkerName, data.TTL, data.Handle}
	values := []string{"$1", "$2", "$3"}

	if data.TTL == 0 {
		columns = append(columns, "expires_at")
		values = append(values, "NULL")
	} else {
		columns = append(columns, "expires_at")
		params = append(params, fmt.Sprintf("%d second", int(data.TTL.Seconds())))
		values = append(values, fmt.Sprintf("NOW() + $%d::INTERVAL", len(params)))
	}

	switch {
	case data.Identifier.ResourceCache != nil:
		resourceVersion, err = json.Marshal(data.Identifier.ResourceCache.ResourceVersion)
		if err != nil {
			return err
		}

		columns = append(columns, "resource_version")
		params = append(params, resourceVersion)
		values = append(values, fmt.Sprintf("$%d", len(params)))

		columns = append(columns, "resource_hash")
		params = append(params, data.Identifier.ResourceCache.ResourceHash)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.COW != nil:
		columns = append(columns, "original_volume_handle")
		params = append(params, data.Identifier.COW.ParentVolumeHandle)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.Output != nil:
		columns = append(columns, "output_name")
		params = append(params, data.Identifier.Output.Name)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.Import != nil:
		columns = append(columns, "path")
		params = append(params, data.Identifier.Import.Path)
		values = append(values, fmt.Sprintf("$%d", len(params)))

		columns = append(columns, "host_path_version")
		params = append(params, data.Identifier.Import.Version)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	}

	_, err = tx.Exec(
		fmt.Sprintf(
			`
				INSERT INTO volumes(
					%s
				) VALUES (
					%s
				)
			`,
			strings.Join(columns, ", "),
			strings.Join(values, ", "),
		), params...)
	if err != nil {
		if strings.Contains(err.Error(), `duplicate key value violates unique constraint "volumes_worker_name_handle_key"`) {
			return nil
		}

		return err
	}

	return tx.Commit()
}

func (db *SQLDB) ReapVolume(handle string) error {
	_, err := db.conn.Exec(`
		DELETE FROM volumes
		WHERE handle = $1
	`, handle)
	return err
}

func (db *SQLDB) GetVolumes() ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	rows, err := db.conn.Query(`
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
	`)
	if err != nil {
		return nil, err
	}

	volumes, err := scanVolumes(rows)
	return volumes, err
}

func (db *SQLDB) GetVolumesByIdentifier(id VolumeIdentifier) ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	conditions := []string{}
	params := []interface{}{}

	addParam := func(column string, param interface{}) {
		conditions = append(conditions, fmt.Sprintf("%s = $%d", column, len(params)+1))
		params = append(params, param)
	}

	switch {
	case id.ResourceCache != nil:
		resourceVersion, err := json.Marshal(id.ResourceCache.ResourceVersion)
		if err != nil {
			return nil, err
		}
		addParam("resource_version", resourceVersion)
		addParam("resource_hash", id.ResourceCache.ResourceHash)
	case id.COW != nil:
		addParam("original_volume_handle", id.COW.ParentVolumeHandle)
	case id.Output != nil:
		addParam("output_name", id.Output.Name)
	case id.Import != nil:
		addParam("path", id.Import.Path)
		addParam("worker_name", id.Import.WorkerName)
		if id.Import.Version != nil {
			addParam("host_path_version", id.Import.Version)
		}
	}

	statement := `
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
		`

	statement += "WHERE " + strings.Join(conditions, " AND ")
	statement += "ORDER BY id ASC"
	rows, err := db.conn.Query(statement, params...)
	if err != nil {
		return nil, err
	}

	savedVolumes, err := scanVolumes(rows)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return savedVolumes, nil
}

func (db *SQLDB) GetVolumesForOneOffBuildImageResources() ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	rows, err := db.conn.Query(`
		SELECT DISTINCT
			v.worker_name,
			v.ttl,
			EXTRACT(epoch FROM v.expires_at - NOW()),
			v.handle,
			v.resource_version,
			v.resource_hash,
			v.id,
			v.original_volume_handle,
			v.output_name,
			v.path,
			v.host_path_version
		FROM volumes v
			INNER JOIN image_resource_versions i
				ON i.version = v.resource_version
				AND i.resource_hash = v.resource_hash
			INNER JOIN builds b
				ON b.id = i.build_id
		WHERE b.job_id IS NULL
	`)
	if err != nil {
		return nil, err
	}

	volumes, err := scanVolumes(rows)
	return volumes, err
}

func (db *SQLDB) SetVolumeTTL(handle string, ttl time.Duration) error {
	if ttl == 0 {
		_, err := db.conn.Exec(`
			UPDATE volumes
			SET expires_at = null, ttl = 0
			WHERE handle = $1
		`, handle)

		return err
	}

	interval := fmt.Sprintf("%d second", int(ttl.Seconds()))

	_, err := db.conn.Exec(`
		UPDATE volumes
		SET expires_at = NOW() + $1::INTERVAL,
		ttl = $2
		WHERE handle = $3
	`, interval, ttl, handle)

	return err
}

func (db *SQLDB) GetVolumeTTL(handle string) (time.Duration, bool, error) {
	var ttl time.Duration

	err := db.conn.QueryRow(`
		SELECT ttl
		FROM volumes
		WHERE handle = $1
	`, handle).Scan(&ttl)
	if err == sql.ErrNoRows {
		return 0, false, nil
	} else if err != nil {
		return 0, false, err
	}

	return ttl, true, nil
}

func (db *SQLDB) getVolume(originalVolumeHandle string) (SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return SavedVolume{}, err
	}

	rows, err := db.conn.Query(`
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
		WHERE handle = $1
	`, originalVolumeHandle)
	if err != nil {
		return SavedVolume{}, err
	}

	volumes, err := scanVolumes(rows)
	if err != nil {
		return SavedVolume{}, err
	}

	switch len(volumes) {
	case 0:
		return SavedVolume{}, errors.New(fmt.Sprintf("unable to find volume handle %s", originalVolumeHandle))
	case 1:
		return volumes[0], nil
	default:
		return SavedVolume{}, errors.New(fmt.Sprintf("%d volumes found for handle %s", len(volumes), originalVolumeHandle))
	}
}

func (db *SQLDB) expireVolumes() error {
	_, err := db.conn.Exec(`
		DELETE FROM volumes
		WHERE expires_at IS NOT NULL
		AND expires_at < NOW()
	`)
	return err
}

func scanVolumes(rows *sql.Rows) ([]SavedVolume, error) {
	defer rows.Close()

	var (
		volume               SavedVolume
		ttlSeconds           *float64
		versionJSON          sql.NullString
		resourceHash         sql.NullString
		originalVolumeHandle sql.NullString
		outputName           sql.NullString
		path                 sql.NullString
		hostPathVersion      sql.NullString
	)

	volumes := []SavedVolume{}

	for rows.Next() {
		err := rows.Scan(
			&volume.WorkerName,
			&volume.TTL,
			&ttlSeconds,
			&volume.Handle,
			&versionJSON,
			&resourceHash,
			&volume.ID,
			&originalVolumeHandle,
			&outputName,
			&path,
			&hostPathVersion,
		)
		if err != nil {
			return []SavedVolume{}, err
		}

		if ttlSeconds != nil {
			volume.ExpiresIn = time.Duration(*ttlSeconds) * time.Second
		}

		switch {
		case versionJSON.Valid && resourceHash.Valid:
			var cacheID ResourceCacheIdentifier

			err = json.Unmarshal([]byte(versionJSON.String), &cacheID.ResourceVersion)
			if err != nil {
				return []SavedVolume{}, err
			}

			cacheID.ResourceHash = resourceHash.String

			volume.Volume.Identifier.ResourceCache = &cacheID
		case originalVolumeHandle.Valid:
			volume.Volume.Identifier.COW = &COWIdentifier{
				ParentVolumeHandle: originalVolumeHandle.String,
			}
		case outputName.Valid:
			volume.Volume.Identifier.Output = &OutputIdentifier{
				Name: outputName.String,
			}
		case path.Valid:
			volume.Volume.Identifier.Import = &ImportIdentifier{
				Path:       path.String,
				WorkerName: volume.WorkerName,
				Version:    &hostPathVersion.String,
			}
		}

		volumes = append(volumes, volume)
	}

	return volumes, nil
}
<commit_msg>fix variable sharing when getting volumes<commit_after>package db

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"
)

func (db *SQLDB) InsertVolume(data Volume) error {
	tx, err := db.conn.Begin()
	if err != nil {
		return err
	}

	defer tx.Rollback()

	var resourceVersion []byte

	columns := []string{"worker_name", "ttl", "handle"}
	params := []interface{}{data.WorkerName, data.TTL, data.Handle}
	values := []string{"$1", "$2", "$3"}

	if data.TTL == 0 {
		columns = append(columns, "expires_at")
		values = append(values, "NULL")
	} else {
		columns = append(columns, "expires_at")
		params = append(params, fmt.Sprintf("%d second", int(data.TTL.Seconds())))
		values = append(values, fmt.Sprintf("NOW() + $%d::INTERVAL", len(params)))
	}

	switch {
	case data.Identifier.ResourceCache != nil:
		resourceVersion, err = json.Marshal(data.Identifier.ResourceCache.ResourceVersion)
		if err != nil {
			return err
		}

		columns = append(columns, "resource_version")
		params = append(params, resourceVersion)
		values = append(values, fmt.Sprintf("$%d", len(params)))

		columns = append(columns, "resource_hash")
		params = append(params, data.Identifier.ResourceCache.ResourceHash)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.COW != nil:
		columns = append(columns, "original_volume_handle")
		params = append(params, data.Identifier.COW.ParentVolumeHandle)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.Output != nil:
		columns = append(columns, "output_name")
		params = append(params, data.Identifier.Output.Name)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	case data.Identifier.Import != nil:
		columns = append(columns, "path")
		params = append(params, data.Identifier.Import.Path)
		values = append(values, fmt.Sprintf("$%d", len(params)))

		columns = append(columns, "host_path_version")
		params = append(params, data.Identifier.Import.Version)
		values = append(values, fmt.Sprintf("$%d", len(params)))
	}

	_, err = tx.Exec(
		fmt.Sprintf(
			`
				INSERT INTO volumes(
					%s
				) VALUES (
					%s
				)
			`,
			strings.Join(columns, ", "),
			strings.Join(values, ", "),
		), params...)
	if err != nil {
		if strings.Contains(err.Error(), `duplicate key value violates unique constraint "volumes_worker_name_handle_key"`) {
			return nil
		}

		return err
	}

	return tx.Commit()
}

func (db *SQLDB) ReapVolume(handle string) error {
	_, err := db.conn.Exec(`
		DELETE FROM volumes
		WHERE handle = $1
	`, handle)
	return err
}

func (db *SQLDB) GetVolumes() ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	rows, err := db.conn.Query(`
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
	`)
	if err != nil {
		return nil, err
	}

	volumes, err := scanVolumes(rows)
	return volumes, err
}

func (db *SQLDB) GetVolumesByIdentifier(id VolumeIdentifier) ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	conditions := []string{}
	params := []interface{}{}

	addParam := func(column string, param interface{}) {
		conditions = append(conditions, fmt.Sprintf("%s = $%d", column, len(params)+1))
		params = append(params, param)
	}

	switch {
	case id.ResourceCache != nil:
		resourceVersion, err := json.Marshal(id.ResourceCache.ResourceVersion)
		if err != nil {
			return nil, err
		}
		addParam("resource_version", resourceVersion)
		addParam("resource_hash", id.ResourceCache.ResourceHash)
	case id.COW != nil:
		addParam("original_volume_handle", id.COW.ParentVolumeHandle)
	case id.Output != nil:
		addParam("output_name", id.Output.Name)
	case id.Import != nil:
		addParam("path", id.Import.Path)
		addParam("worker_name", id.Import.WorkerName)
		if id.Import.Version != nil {
			addParam("host_path_version", id.Import.Version)
		}
	}

	statement := `
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
		`

	statement += "WHERE " + strings.Join(conditions, " AND ")
	statement += "ORDER BY id ASC"
	rows, err := db.conn.Query(statement, params...)
	if err != nil {
		return nil, err
	}

	savedVolumes, err := scanVolumes(rows)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return savedVolumes, nil
}

func (db *SQLDB) GetVolumesForOneOffBuildImageResources() ([]SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return nil, err
	}

	rows, err := db.conn.Query(`
		SELECT DISTINCT
			v.worker_name,
			v.ttl,
			EXTRACT(epoch FROM v.expires_at - NOW()),
			v.handle,
			v.resource_version,
			v.resource_hash,
			v.id,
			v.original_volume_handle,
			v.output_name,
			v.path,
			v.host_path_version
		FROM volumes v
			INNER JOIN image_resource_versions i
				ON i.version = v.resource_version
				AND i.resource_hash = v.resource_hash
			INNER JOIN builds b
				ON b.id = i.build_id
		WHERE b.job_id IS NULL
	`)
	if err != nil {
		return nil, err
	}

	volumes, err := scanVolumes(rows)
	return volumes, err
}

func (db *SQLDB) SetVolumeTTL(handle string, ttl time.Duration) error {
	if ttl == 0 {
		_, err := db.conn.Exec(`
			UPDATE volumes
			SET expires_at = null, ttl = 0
			WHERE handle = $1
		`, handle)

		return err
	}

	interval := fmt.Sprintf("%d second", int(ttl.Seconds()))

	_, err := db.conn.Exec(`
		UPDATE volumes
		SET expires_at = NOW() + $1::INTERVAL,
		ttl = $2
		WHERE handle = $3
	`, interval, ttl, handle)

	return err
}

func (db *SQLDB) GetVolumeTTL(handle string) (time.Duration, bool, error) {
	var ttl time.Duration

	err := db.conn.QueryRow(`
		SELECT ttl
		FROM volumes
		WHERE handle = $1
	`, handle).Scan(&ttl)
	if err == sql.ErrNoRows {
		return 0, false, nil
	} else if err != nil {
		return 0, false, err
	}

	return ttl, true, nil
}

func (db *SQLDB) getVolume(originalVolumeHandle string) (SavedVolume, error) {
	err := db.expireVolumes()
	if err != nil {
		return SavedVolume{}, err
	}

	rows, err := db.conn.Query(`
		SELECT
			worker_name,
			ttl,
			EXTRACT(epoch FROM expires_at - NOW()),
			handle,
			resource_version,
			resource_hash,
			id,
			original_volume_handle,
			output_name,
			path,
			host_path_version
		FROM volumes
		WHERE handle = $1
	`, originalVolumeHandle)
	if err != nil {
		return SavedVolume{}, err
	}

	volumes, err := scanVolumes(rows)
	if err != nil {
		return SavedVolume{}, err
	}

	switch len(volumes) {
	case 0:
		return SavedVolume{}, errors.New(fmt.Sprintf("unable to find volume handle %s", originalVolumeHandle))
	case 1:
		return volumes[0], nil
	default:
		return SavedVolume{}, errors.New(fmt.Sprintf("%d volumes found for handle %s", len(volumes), originalVolumeHandle))
	}
}

func (db *SQLDB) expireVolumes() error {
	_, err := db.conn.Exec(`
		DELETE FROM volumes
		WHERE expires_at IS NOT NULL
		AND expires_at < NOW()
	`)
	return err
}

func scanVolumes(rows *sql.Rows) ([]SavedVolume, error) {
	defer rows.Close()

	volumes := []SavedVolume{}

	for rows.Next() {
		var (
			volume               SavedVolume
			ttlSeconds           *float64
			versionJSON          sql.NullString
			resourceHash         sql.NullString
			originalVolumeHandle sql.NullString
			outputName           sql.NullString
			path                 sql.NullString
			hostPathVersion      sql.NullString
		)

		err := rows.Scan(
			&volume.WorkerName,
			&volume.TTL,
			&ttlSeconds,
			&volume.Handle,
			&versionJSON,
			&resourceHash,
			&volume.ID,
			&originalVolumeHandle,
			&outputName,
			&path,
			&hostPathVersion,
		)
		if err != nil {
			return []SavedVolume{}, err
		}

		if ttlSeconds != nil {
			volume.ExpiresIn = time.Duration(*ttlSeconds) * time.Second
		}

		switch {
		case versionJSON.Valid && resourceHash.Valid:
			var cacheID ResourceCacheIdentifier

			err = json.Unmarshal([]byte(versionJSON.String), &cacheID.ResourceVersion)
			if err != nil {
				return []SavedVolume{}, err
			}

			cacheID.ResourceHash = resourceHash.String

			volume.Volume.Identifier.ResourceCache = &cacheID
		case originalVolumeHandle.Valid:
			volume.Volume.Identifier.COW = &COWIdentifier{
				ParentVolumeHandle: originalVolumeHandle.String,
			}
		case outputName.Valid:
			volume.Volume.Identifier.Output = &OutputIdentifier{
				Name: outputName.String,
			}
		case path.Valid:
			volume.Volume.Identifier.Import = &ImportIdentifier{
				Path:       path.String,
				WorkerName: volume.WorkerName,
				Version:    &hostPathVersion.String,
			}
		}

		volumes = append(volumes, volume)
	}

	return volumes, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Ka-Hing Cheung
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
)

// Set up custom help text for goofys; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.7",
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-west-2",
				Usage: "The region to connect to." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.BoolFlag{
				Name: "use-path-request",
				Usage: "Use a path-style request instead of virtual host-style." +
					" (deprecated, always on)",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3)in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          uint32
	Gid          uint32

	// S3
	Endpoint       string
	Region         string
	StorageClass   string
	UsePathRequest bool
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (flags *FlagStorage) {
	flags = &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		StorageClass:   c.String("storage-class"),
		UsePathRequest: c.Bool("use-path-request"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}
	return
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<commit_msg>v0.0.8<commit_after>// Copyright 2015 Ka-Hing Cheung
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
)

// Set up custom help text for goofys; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.8",
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-west-2",
				Usage: "The region to connect to." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.BoolFlag{
				Name: "use-path-request",
				Usage: "Use a path-style request instead of virtual host-style." +
					" (deprecated, always on)",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3)in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions map[string]string
	DirMode      os.FileMode
	FileMode     os.FileMode
	Uid          uint32
	Gid          uint32

	// S3
	Endpoint       string
	Region         string
	StorageClass   string
	UsePathRequest bool
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string

	// Tuning
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (flags *FlagStorage) {
	flags = &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		StorageClass:   c.String("storage-class"),
		UsePathRequest: c.Bool("use-path-request"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}
	return
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<|endoftext|>