<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrade

import (
	"os"
	"path/filepath"

	"github.com/pkg/errors"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	errorsutil "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/version"
	clientset "k8s.io/client-go/kubernetes"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
)

// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
	errs := []error{}

	// Upload currently used configuration to the cluster
	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
	// depend on centralized information from this source in the future
	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
		errs = append(errs, err)
	}

	// Create the new, version-branched kubelet ComponentConfig ConfigMap
	if err := kubeletphase.CreateConfigMap(cfg.ClusterConfiguration.ComponentConfigs.Kubelet, cfg.KubernetesVersion, client); err != nil {
		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
	}

	// Write the new kubelet config down to disk and the env file if needed
	if err := writeKubeletConfigFiles(client, cfg, newK8sVer, dryRun); err != nil {
		errs = append(errs, err)
	}

	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
	// --cri-socket.
	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
		errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
		errs = append(errs, err)
	}

	// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
	// Create the cluster-info ConfigMap with the associated RBAC rules
	// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
	// 	return err
	//}
	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
	if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
		errs = append(errs, err)
	}

	// Upgrade kube-dns/CoreDNS and kube-proxy
	if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
		errs = append(errs, err)
	}
	// Remove the old DNS deployment if a new DNS service is now used (kube-dns to CoreDNS or vice versa)
	if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(&cfg.ClusterConfiguration, client, dryRun); err != nil {
		errs = append(errs, err)
	}

	if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client); err != nil {
		errs = append(errs, err)
	}
	return errorsutil.NewAggregate(errs)
}

func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, dryRun bool) error {
	return apiclient.TryRunCommand(func() error {
		installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName
		deploymentToDelete := kubeadmconstants.CoreDNSDeploymentName

		if cfg.DNS.Type == kubeadmapi.CoreDNS {
			installedDeploymentName = kubeadmconstants.CoreDNSDeploymentName
			deploymentToDelete = kubeadmconstants.KubeDNSDeploymentName
		}

		// If we're dry-running, we don't need to wait for the new DNS addon to become ready
		if !dryRun {
			dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(installedDeploymentName, metav1.GetOptions{})
			if err != nil {
				return err
			}
			if dnsDeployment.Status.ReadyReplicas == 0 {
				return errors.New("the DNS deployment isn't ready yet")
			}
		}

		// We don't want to wait for the DNS deployment above to become ready when dryrunning (as it never will)
		// but here we should execute the DELETE command against the dryrun clientset, as it will only be logged
		err := apiclient.DeleteDeploymentForeground(client, metav1.NamespaceSystem, deploymentToDelete)
		if err != nil && !apierrors.IsNotFound(err) {
			return err
		}
		return nil
	}, 10)
}

func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
	kubeletDir, err := GetKubeletDir(dryRun)
	if err != nil {
		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
		return err
	}
	errs := []error{}
	// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
	if err := kubeletphase.DownloadConfig(client, newK8sVer, kubeletDir); err != nil {
		// Tolerate the error being NotFound when dryrunning, as there is a pretty common scenario: the dryrun process
		// *would* post the new kubelet-config-1.X configmap that doesn't exist now when we're trying to download it
		// again.
		if !(apierrors.IsNotFound(err) && dryRun) {
			errs = append(errs, errors.Wrap(err, "error downloading kubelet configuration from the ConfigMap"))
		}
	}

	if dryRun { // Print what contents would be written
		dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
	}

	envFilePath := filepath.Join(kubeadmconstants.KubeletRunDirectory, kubeadmconstants.KubeletEnvFileName)
	if _, err := os.Stat(envFilePath); os.IsNotExist(err) {
		// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the control-plane,
		// as we handle that ourselves in the mark-control-plane phase
		// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the mark-control-plane phase?
		if err := kubeletphase.WriteKubeletDynamicEnvFile(&cfg.ClusterConfiguration, &cfg.NodeRegistration, false, kubeletDir); err != nil {
			errs = append(errs, errors.Wrap(err, "error writing a dynamic environment file for the kubelet"))
		}

		if dryRun { // Print what contents would be written
			dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletEnvFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
		}
	}
	return errorsutil.NewAggregate(errs)
}

// GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
func GetKubeletDir(dryRun bool) (string, error) {
	if dryRun {
		return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
	}
	return kubeadmconstants.KubeletRunDirectory, nil
}

// moveFiles moves files from one directory to another.
func moveFiles(files map[string]string) error {
	filesToRecover := map[string]string{}
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			return rollbackFiles(filesToRecover, err)
		}
		filesToRecover[to] = from
	}
	return nil
}

// rollbackFiles moves the files back to the original directory.
func rollbackFiles(files map[string]string, originalErr error) error {
	errs := []error{originalErr}
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
}
<commit_msg>remove Kubeadm-env file overwrite from apply and node command<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrade

import (
	"os"

	"github.com/pkg/errors"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	errorsutil "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/version"
	clientset "k8s.io/client-go/kubernetes"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
)

// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
	errs := []error{}

	// Upload currently used configuration to the cluster
	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
	// depend on centralized information from this source in the future
	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
		errs = append(errs, err)
	}

	// Create the new, version-branched kubelet ComponentConfig ConfigMap
	if err := kubeletphase.CreateConfigMap(cfg.ClusterConfiguration.ComponentConfigs.Kubelet, cfg.KubernetesVersion, client); err != nil {
		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
	}

	// Write the new kubelet config down to disk and the env file if needed
	if err := writeKubeletConfigFiles(client, cfg, newK8sVer, dryRun); err != nil {
		errs = append(errs, err)
	}

	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
	// --cri-socket.
	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
		errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
		errs = append(errs, err)
	}

	// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
	// Create the cluster-info ConfigMap with the associated RBAC rules
	// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
	// 	return err
	//}
	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
	if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
		errs = append(errs, err)
	}

	// Upgrade kube-dns/CoreDNS and kube-proxy
	if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
		errs = append(errs, err)
	}
	// Remove the old DNS deployment if a new DNS service is now used (kube-dns to CoreDNS or vice versa)
	if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(&cfg.ClusterConfiguration, client, dryRun); err != nil {
		errs = append(errs, err)
	}

	if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client); err != nil {
		errs = append(errs, err)
	}
	return errorsutil.NewAggregate(errs)
}

func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, dryRun bool) error {
	return apiclient.TryRunCommand(func() error {
		installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName
		deploymentToDelete := kubeadmconstants.CoreDNSDeploymentName

		if cfg.DNS.Type == kubeadmapi.CoreDNS {
			installedDeploymentName = kubeadmconstants.CoreDNSDeploymentName
			deploymentToDelete = kubeadmconstants.KubeDNSDeploymentName
		}

		// If we're dry-running, we don't need to wait for the new DNS addon to become ready
		if !dryRun {
			dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(installedDeploymentName, metav1.GetOptions{})
			if err != nil {
				return err
			}
			if dnsDeployment.Status.ReadyReplicas == 0 {
				return errors.New("the DNS deployment isn't ready yet")
			}
		}

		// We don't want to wait for the DNS deployment above to become ready when dryrunning (as it never will)
		// but here we should execute the DELETE command against the dryrun clientset, as it will only be logged
		err := apiclient.DeleteDeploymentForeground(client, metav1.NamespaceSystem, deploymentToDelete)
		if err != nil && !apierrors.IsNotFound(err) {
			return err
		}
		return nil
	}, 10)
}

func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
	kubeletDir, err := GetKubeletDir(dryRun)
	if err != nil {
		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
		return err
	}
	errs := []error{}
	// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
	if err := kubeletphase.DownloadConfig(client, newK8sVer, kubeletDir); err != nil {
		// Tolerate the error being NotFound when dryrunning, as there is a pretty common scenario: the dryrun process
		// *would* post the new kubelet-config-1.X configmap that doesn't exist now when we're trying to download it
		// again.
		if !(apierrors.IsNotFound(err) && dryRun) {
			errs = append(errs, errors.Wrap(err, "error downloading kubelet configuration from the ConfigMap"))
		}
	}

	if dryRun { // Print what contents would be written
		dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
	}
	return errorsutil.NewAggregate(errs)
}

// GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
func GetKubeletDir(dryRun bool) (string, error) {
	if dryRun {
		return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
	}
	return kubeadmconstants.KubeletRunDirectory, nil
}

// moveFiles moves files from one directory to another.
func moveFiles(files map[string]string) error {
	filesToRecover := map[string]string{}
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			return rollbackFiles(filesToRecover, err)
		}
		filesToRecover[to] = from
	}
	return nil
}

// rollbackFiles moves the files back to the original directory.
func rollbackFiles(files map[string]string, originalErr error) error {
	errs := []error{originalErr}
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
}
<|endoftext|>


<commit_before>package paymentapi

import (
	"encoding/json"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net/http"
	"socialapi/workers/payment"
)

var (
	DefaultPlanUrl = "http://localhost:7000/payments/subscriptions"
	FreePlanName   = "free"
)

type Client struct {
	PlanUrl       string
	DefaultToPaid bool
}

func New(planUrl string) *Client {
	if planUrl == "" {
		planUrl = DefaultPlanUrl
	}

	return &Client{
		PlanUrl:       planUrl,
		DefaultToPaid: true,
	}
}

func (c *Client) IsPaidAccount(account *models.Account) (bool, error) {
	sub, err := c.GetByAccount(account)
	if err != nil {
		return false, err
	}

	return sub.PlanTitle != FreePlanName, nil
}

func (c *Client) GetByUsername(username string) (*payment.SubscriptionsResponse, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	return c.GetByAccount(account)
}

func (c *Client) GetByAccountId(accountId string) (*payment.SubscriptionsResponse, error) {
	account, err := modelhelper.GetAccountById(accountId)
	if err != nil {
		return nil, err
	}

	return c.GetByAccount(account)
}

func (c *Client) GetByAccount(account *models.Account) (*payment.SubscriptionsResponse, error) {
	url := fmt.Sprintf("%s?account_id=%s", c.PlanUrl, account.Id.Hex())
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	var subscription *payment.SubscriptionsResponse
	e := json.NewDecoder(resp.Body)
	if err := e.Decode(&subscription); err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response status not 200, was %d", resp.StatusCode)
	}

	return subscription, nil
}
<commit_msg>janitor: close http response body<commit_after>package paymentapi

import (
	"encoding/json"
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net/http"
	"socialapi/workers/payment"
)

var (
	DefaultPlanUrl = "http://localhost:7000/payments/subscriptions"
	FreePlanName   = "free"
)

type Client struct {
	PlanUrl       string
	DefaultToPaid bool
}

func New(planUrl string) *Client {
	if planUrl == "" {
		planUrl = DefaultPlanUrl
	}

	return &Client{
		PlanUrl:       planUrl,
		DefaultToPaid: true,
	}
}

func (c *Client) IsPaidAccount(account *models.Account) (bool, error) {
	sub, err := c.GetByAccount(account)
	if err != nil {
		return false, err
	}

	return sub.PlanTitle != FreePlanName, nil
}

func (c *Client) GetByUsername(username string) (*payment.SubscriptionsResponse, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return nil, err
	}

	return c.GetByAccount(account)
}

func (c *Client) GetByAccountId(accountId string) (*payment.SubscriptionsResponse, error) {
	account, err := modelhelper.GetAccountById(accountId)
	if err != nil {
		return nil, err
	}

	return c.GetByAccount(account)
}

func (c *Client) GetByAccount(account *models.Account) (*payment.SubscriptionsResponse, error) {
	url := fmt.Sprintf("%s?account_id=%s", c.PlanUrl, account.Id.Hex())
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	var subscription *payment.SubscriptionsResponse
	if err := json.NewDecoder(resp.Body).Decode(&subscription); err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response status not 200, was %d", resp.StatusCode)
	}

	return subscription, nil
}
<|endoftext|>


<commit_before>package trollmode

import (
	"encoding/json"
	"socialapi/models"

	"github.com/koding/logging"
	"github.com/koding/worker"
	"github.com/streadway/amqp"
)

type Action func(*TrollModeController, *models.Account) error

type TrollModeController struct {
	routes map[string]Action
	log    logging.Logger
}

func (t *TrollModeController) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func NewTrollModeController(log logging.Logger) *TrollModeController {
	ffc := &TrollModeController{
		log: log,
	}

	routes := map[string]Action{
		"api.marked_as_troll":   (*TrollModeController).MarkedAsTroll,
		"api.unmarked_as_troll": (*TrollModeController).UnMarkedAsTroll,
	}

	ffc.routes = routes

	return ffc
}

func (t *TrollModeController) HandleEvent(event string, data []byte) error {
	t.log.Debug("New Event Received %s", event)
	handler, ok := t.routes[event]
	if !ok {
		return worker.HandlerNotFoundErr
	}

	acc, err := mapMessage(data)
	if err != nil {
		return err
	}

	return handler(t, acc)
}

func (t *TrollModeController) MarkedAsTroll(data *models.Account) error {
	t.log.Critical("marked as troll ehehe %v", data)
	return nil
}

func (t *TrollModeController) UnMarkedAsTroll(data *models.Account) error {
	t.log.Critical("un marked as troll ehehe %v", data)
	return nil
}

func mapMessage(data []byte) (*models.Account, error) {
	cm := models.NewAccount()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}
<commit_msg>Social: fix eventname<commit_after>package trollmode

import (
	"encoding/json"
	"socialapi/models"

	"github.com/koding/logging"
	"github.com/koding/worker"
	"github.com/streadway/amqp"
)

type Action func(*TrollModeController, *models.Account) error

type TrollModeController struct {
	routes map[string]Action
	log    logging.Logger
}

func (t *TrollModeController) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func NewTrollModeController(log logging.Logger) *TrollModeController {
	ffc := &TrollModeController{
		log: log,
	}

	routes := map[string]Action{
		"api.account_marked_as_troll":   (*TrollModeController).MarkedAsTroll,
		"api.account_unmarked_as_troll": (*TrollModeController).UnMarkedAsTroll,
	}

	ffc.routes = routes

	return ffc
}

func (t *TrollModeController) HandleEvent(event string, data []byte) error {
	t.log.Debug("New Event Received %s", event)
	handler, ok := t.routes[event]
	if !ok {
		return worker.HandlerNotFoundErr
	}

	acc, err := mapMessage(data)
	if err != nil {
		return err
	}

	return handler(t, acc)
}

func (t *TrollModeController) MarkedAsTroll(data *models.Account) error {
	t.log.Critical("marked as troll ehehe %v", data)
	return nil
}

func (t *TrollModeController) UnMarkedAsTroll(data *models.Account) error {
	t.log.Critical("un marked as troll ehehe %v", data)
	return nil
}

func mapMessage(data []byte) (*models.Account, error) {
	cm := models.NewAccount()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}
<|endoftext|>


<commit_before>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	c.Charset = NormaliseCharset(cs)
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		attr, exists := s.Attr("http-equiv")
		if exists && attr == "Content-Type" {
			return false
		}
		return true
	})

	if selection != nil {
		attr, _ := selection.Attr("content")
		attr = strings.Replace(attr, " ", "", -1)

		if strings.HasPrefix(attr, "text/html;charset=") {
			cs := strings.TrimPrefix(attr, "text/html;charset=")
			return NormaliseCharset(cs)
		}
	}

	// <meta charset="utf-8">
	selection = document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	if c.RawHTML == "" {
		c.RawHTML = c.fetchHTML(c.url, c.config.timeout)
	}
	if c.RawHTML == "" {
		return nil, nil
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		document, err = goquery.NewDocumentFromReader(reader)

		if nil != err {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() *Article {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		panic(err.Error())
	}
	if nil == document {
		return article
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, _ = document.Html()
	article.FinalURL = c.url
	article.Doc = document
	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}
	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) string {
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Println(err.Error())
		return ""
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		c.RawHTML = string(contents)
	} else {
		log.Println(err.Error())
	}
	resp.Body.Close()

	return c.RawHTML
}
<commit_msg>better charset/content-type handling<commit_after>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	cs = strings.Replace(cs, " ", "", -1)
	if strings.HasPrefix(cs, "text/html;charset=") {
		cs = strings.TrimPrefix(cs, "text/html;charset=")
	}
	c.Charset = NormaliseCharset(cs)
}

// GetContentType returns the Content-Type string extracted from the meta tags
func (c Crawler) GetContentType(document *goquery.Document) string {
	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		attr, exists := s.Attr("http-equiv")
		if exists && attr == "Content-Type" {
			return false
		}
		return true
	})

	if selection != nil {
		attr, _ := selection.Attr("content")
		return attr
	}

	return ""
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	ct := c.GetContentType(document)
	if "" != ct {
		ct = strings.Replace(ct, " ", "", -1)
		if strings.HasPrefix(ct, "text/html;charset=") {
			cs := strings.TrimPrefix(ct, "text/html;charset=")
			return NormaliseCharset(cs)
		}
	}

	// <meta charset="utf-8">
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	if c.RawHTML == "" {
		c.RawHTML = c.fetchHTML(c.url, c.config.timeout)
	}
	if c.RawHTML == "" {
		return nil, nil
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		document, err = goquery.NewDocumentFromReader(reader)

		if nil != err {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() *Article {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		panic(err.Error())
	}
	if nil == document {
		return article
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, _ = document.Html()
	article.FinalURL = c.url
	article.Doc = document
	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}
	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) string {
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Println(err.Error())
		return ""
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		c.RawHTML = string(contents)
	} else {
		log.Println(err.Error())
	}
	resp.Body.Close()

	return c.RawHTML
}
<|endoftext|>


<commit_before>package chevalier

import (
	zmq "github.com/pebbe/zmq4"
)

// GetContents list for origin from a Vaultaire
// readerd listening on endpoint, returning it as a DataSourceBurst.
func GetContents(endpoint, origin string) (*DataSourceBurst, error) {
	sources := make([]*DataSourceResponse, 0)
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		return nil, err
	}
	err = sock.Connect(endpoint)
	if err != nil {
		return nil, err
	}
	request := make([][]byte, 2)
	request[0] = make([]byte, len(origin))
	for idx, chr := range origin {
		request[0][idx] = byte(chr)
	}
	request[1] = make([]byte, 1)
	request[1][0] = byte(ContentsListRequest)
	_, err = sock.SendMessage(request[0], request[1])
	if err != nil {
		return nil, err
	}
	for res := new(ContentsResponse); notStopResponse(res); res, err = recvContentsMessage(sock) {
		if err != nil {
			return nil, err
		}
		res_ := unpackSourceResponse(res)
		sources = append(sources, res_)
	}
	return nil, nil
}

func notStopResponse(res *ContentsResponse) bool {
	if res == nil {
		return false // first iteration or error
	}
	if res.opCode == ContentsListEntry {
		return false // data response, continue
	}
	return true
}

func recvContentsMessage(sock *zmq.Socket) (*ContentsResponse, error) {
	bs, err := sock.RecvBytes(0)
	if err != nil {
		return nil, err
	}
	return unpackContentsResponse(bs)
}

func unpackSourceResponse(res *ContentsResponse) *DataSourceResponse {
	return nil
}
<commit_msg>Unpack contents responses to ElasticsearchSources<commit_after>package chevalier

import (
	zmq "github.com/pebbe/zmq4"
)

// GetContents list for origin from a Vaultaire
// readerd listening on endpoint, returning it as a DataSourceBurst.
func GetContents(endpoint, origin string) ([]*ElasticsearchSource, error) {
	sources := make([]*ElasticsearchSource, 0)
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		return nil, err
	}
	err = sock.Connect(endpoint)
	if err != nil {
		return nil, err
	}
	request := make([][]byte, 2)
	request[0] = make([]byte, len(origin))
	for idx, chr := range origin {
		request[0][idx] = byte(chr)
	}
	request[1] = make([]byte, 1)
	request[1][0] = byte(ContentsListRequest)
	_, err = sock.SendMessage(request[0], request[1])
	if err != nil {
		return nil, err
	}
	for res := new(ContentsResponse); notStopResponse(res); res, err = recvContentsMessage(sock) {
		if err != nil {
			return nil, err
		}
		res_ := unpackSourceResponse(origin, res)
		sources = append(sources, res_)
	}
	return sources, nil
}

func notStopResponse(res *ContentsResponse) bool {
	if res == nil {
		return false // first iteration or error
	}
	if res.opCode == ContentsListEntry {
		return false // data response, continue
	}
	return true
}

func recvContentsMessage(sock *zmq.Socket) (*ContentsResponse, error) {
	bs, err := sock.RecvBytes(0)
	if err != nil {
		return nil, err
	}
	return unpackContentsResponse(bs)
}

func unpackSourceResponse(origin string, res *ContentsResponse) *ElasticsearchSource {
	source := new(ElasticsearchSource)
	source.Address = res.entry.address
	for k, v := range res.entry.tags {
		source.Source[k] = v
	}
	source.Origin = origin
	return source
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

var manifestURL = "https://github.com/docker-library/official-images/raw/master/library/caddy"

var maintainers = []string{"Ambrose Chua <ambrose@makerforce.io> (@serverwentdown)"}
var gitRepo = "https://github.com/productionwentdown/caddy.git"

var readmeTemplate = template.Must(template.New("readme").Parse(`
# [caddy](https://hub.docker.com/r/productionwentdown/caddy/) ![Docker Pulls](https://img.shields.io/docker/pulls/productionwentdown/caddy) ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/productionwentdown/caddy/alpine?style=flat-square)

A tiny <10MB Caddy image compressed with [UPX](https://github.com/upx/upx).

# Shared Tags

New versions are tracked within 4 hours. Currently available versions:
{{range $entry := .Entries}}
* {{range $index, $tag := $entry.Tags}}{{if $index}}, {{end}}` + "`{{$tag}}`" + `{{end}}
{{end}}
Legacy tags available that should not be used:

* ` + "`2.0.0`" + `
* ` + "`1.0.0`, `1.0.1`, `1.0.2`, `1.0.3`, `1.0.4`, `1.0.5`, `1`" + `
* ` + "`0.10.*`, `0.11.*`" + `

# Usage

See the [official image](https://hub.docker.com/_/caddy) for documentation. This image behaves the same way, except that it is much slimmer.
`))

var dockerfileTemplate = template.Must(template.New("name").Parse(`FROM caddy:{{.Tag}} as build

RUN apk add --no-cache upx ca-certificates \
    && upx --ultra-brute /usr/bin/caddy \
    && caddy version

FROM scratch

COPY --from=build /etc/caddy/Caddyfile /etc/caddy/Caddyfile
COPY --from=build /usr/share/caddy/index.html /usr/share/caddy/index.html
COPY --from=build /usr/bin/caddy /usr/bin/caddy
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build /etc/nsswitch.conf /etc/nsswitch.conf

ENV XDG_CONFIG_HOME=/config
ENV XDG_DATA_HOME=/data
ENV HOME=/

VOLUME /config
VOLUME /data

LABEL org.opencontainers.image.version=v{{.Version}}
LABEL org.opencontainers.image.title="Caddy (productionwentdown build)"
LABEL org.opencontainers.image.description="a powerful, enterprise-ready, open source web server with automatic HTTPS written in Go"
LABEL org.opencontainers.image.url=https://productionwentdown.makerforce.io
LABEL org.opencontainers.image.documentation=https://caddyserver.com/docs
LABEL org.opencontainers.image.vendor="productionwentdown"
LABEL org.opencontainers.image.licenses=Apache-2.0
LABEL org.opencontainers.image.source="https://github.com/productionwentdown/caddy"

EXPOSE 80
EXPOSE 443
EXPOSE 2019

WORKDIR /srv

CMD ["/usr/bin/caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
`))

type dockerfileData struct {
	Tag     string
	Version string
	Arch    string
}

func main() {
	doDockerfiles := flag.Bool("dockerfiles", false, "Update Dockerfiles")
	doManifest := flag.Bool("manifest", false, "Update manifest")
	doReadme := flag.Bool("readme", false, "Update README.md")
	commit := flag.String("commit", "", "Current commit hash")
	flag.Parse()

	_, _, man, err := manifest.Fetch("nonexistent", manifestURL)
	if err != nil {
		panic(err)
	}

	man = subsetManifest(man)

	if *doDockerfiles {
		err = updateDockerfiles(man)
		if err != nil {
			panic(err)
		}
	}
	if *doManifest {
		err = updateManifest(man, *commit)
		if err != nil {
			panic(err)
		}
	}
	if *doReadme {
		err = updateReadme(man)
		if err != nil {
			panic(err)
		}
	}
}

// subsetManifest returns the subset of the manifest that can be built
func subsetManifest(man *manifest.Manifest2822) *manifest.Manifest2822 {
	entries := make([]manifest.Manifest2822Entry, 0, 1)
	for _, entry := range man.Entries {

		isWindows := false
		for _, arch := range entry.Architectures {
			if strings.Contains(arch, "windows") {
				isWindows = true
			}
		}

		isBuilder := false
		for _, tag := range entry.Tags {
			if strings.Contains(tag, "builder") {
				isBuilder = true
			}
		}

		if isWindows || isBuilder {
			continue
		}
		entries = append(entries, entry)
	}
	return &manifest.Manifest2822{
		Global:  man.Global,
		Entries: entries,
	}
}

// updateDockerfiles generates a new set of Dockerfiles based on the manifest
func updateDockerfiles(man *manifest.Manifest2822) error {
	for _, entry := range man.Entries {
		tag := entry.Tags[0]
		version := strings.Split(entry.Tags[0], "-")[0]
		for _, arch := range entry.Architectures {
			data := dockerfileData{Tag: tag, Version: version, Arch: arch}
			err := writeDockerfile(entry, data)
			if err != nil {
				return err
			}
		}
	}
	// TODO: Remove old Dockerfiles
	return nil
}

func writeDockerfile(entry manifest.Manifest2822Entry, data dockerfileData) error {
	dockerfilePath := path.Join(entry.ArchDirectory(data.Arch), entry.ArchFile(data.Arch))
	err := os.MkdirAll(entry.ArchDirectory(data.Arch), 0755)
	if err != nil {
		return err
	}
	dockerfileFile, err := os.OpenFile(dockerfilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer dockerfileFile.Close()

	err = dockerfileTemplate.Execute(dockerfileFile, data)
	if err != nil {
		return err
	}

	return nil
}

// updateManifest replaces upstream maintainer and Git info with this repo
func updateManifest(man *manifest.Manifest2822, gitCommit string) error {
	man.Global.Maintainers = maintainers
	for i := range man.Entries {
		entry := &man.Entries[i]
		entry.Maintainers = maintainers
		entry.GitRepo = gitRepo
		entry.GitCommit = gitCommit
	}
	manifestPath := path.Join("library", "caddy")
	err := os.MkdirAll("library", 0755)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(manifestPath, []byte(man.String()), 0644)
}

// updateReadme generates a new README.md
func updateReadme(man *manifest.Manifest2822) error {
	readmeFile, err := os.OpenFile("README.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer readmeFile.Close()

	err = readmeTemplate.Execute(readmeFile, man)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Update README to link to Docker Hub and GitHub<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

var manifestURL = "https://github.com/docker-library/official-images/raw/master/library/caddy"

var maintainers = []string{"Ambrose Chua <ambrose@makerforce.io> (@serverwentdown)"}
var gitRepo = "https://github.com/productionwentdown/caddy.git"

var readmeTemplate = template.Must(template.New("readme").Parse(`
# [caddy](https://hub.docker.com/r/productionwentdown/caddy) ![Docker Pulls](https://img.shields.io/docker/pulls/productionwentdown/caddy?style=flat-square) ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/productionwentdown/caddy/alpine?style=flat-square)

A tiny <10MB Caddy image compressed with [UPX](https://github.com/upx/upx).

- [Docker Hub](https://hub.docker.com/r/productionwentdown/caddy)
- [GitHub](https://github.com/productionwentdown/caddy)

# Shared Tags

New versions are tracked within 4 hours. Currently available versions:
{{range $entry := .Entries}}
* {{range $index, $tag := $entry.Tags}}{{if $index}}, {{end}}` + "`{{$tag}}`" + `{{end}}
{{end}}
Legacy tags available that should not be used:

* ` + "`2.0.0`" + `
* ` + "`1.0.0`, `1.0.1`, `1.0.2`, `1.0.3`, `1.0.4`, `1.0.5`, `1`" + `
* ` + "`0.10.*`, `0.11.*`" + `

# Usage

See the [official image](https://hub.docker.com/_/caddy) for documentation. This image behaves the same way, except that it is slightly slimmer.
`))

var dockerfileTemplate = template.Must(template.New("name").Parse(`FROM caddy:{{.Tag}} as build

RUN apk add --no-cache upx ca-certificates \
    && upx --ultra-brute /usr/bin/caddy \
    && caddy version

FROM scratch

COPY --from=build /etc/caddy/Caddyfile /etc/caddy/Caddyfile
COPY --from=build /usr/share/caddy/index.html /usr/share/caddy/index.html
COPY --from=build /usr/bin/caddy /usr/bin/caddy
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build /etc/nsswitch.conf /etc/nsswitch.conf

ENV XDG_CONFIG_HOME=/config
ENV XDG_DATA_HOME=/data
ENV HOME=/

VOLUME /config
VOLUME /data

LABEL org.opencontainers.image.version=v{{.Version}}
LABEL org.opencontainers.image.title="Caddy (productionwentdown build)"
LABEL org.opencontainers.image.description="a powerful, enterprise-ready, open source web server with automatic HTTPS written in Go"
LABEL org.opencontainers.image.url=https://productionwentdown.makerforce.io
LABEL org.opencontainers.image.documentation=https://caddyserver.com/docs
LABEL org.opencontainers.image.vendor="productionwentdown"
LABEL org.opencontainers.image.licenses=Apache-2.0
LABEL org.opencontainers.image.source="https://github.com/productionwentdown/caddy"

EXPOSE 80
EXPOSE 443
EXPOSE 2019

WORKDIR /srv

CMD ["/usr/bin/caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
`))

type dockerfileData struct {
	Tag     string
	Version string
	Arch    string
}

func main() {
	doDockerfiles := flag.Bool("dockerfiles", false, "Update Dockerfiles")
	doManifest := flag.Bool("manifest", false, "Update manifest")
	doReadme := flag.Bool("readme", false, "Update README.md")
	commit := flag.String("commit", "", "Current commit hash")
	flag.Parse()

	_, _, man, err := manifest.Fetch("nonexistent", manifestURL)
	if err != nil {
		panic(err)
	}

	man = subsetManifest(man)

	if *doDockerfiles {
		err = updateDockerfiles(man)
		if err != nil {
			panic(err)
		}
	}
	if *doManifest {
		err = updateManifest(man, *commit)
		if err != nil {
			panic(err)
		}
	}
	if *doReadme {
		err = updateReadme(man)
		if err != nil {
			panic(err)
		}
	}
}

// subsetManifest returns the subset of the manifest that can be built
func subsetManifest(man *manifest.Manifest2822) *manifest.Manifest2822 {
	entries := make([]manifest.Manifest2822Entry, 0, 1)
	for _, entry := range man.Entries {

		isWindows := false
		for _, arch := range entry.Architectures {
			if strings.Contains(arch, "windows") {
				isWindows = true
			}
		}

		isBuilder := false
		for _, tag := range entry.Tags {
			if strings.Contains(tag, "builder") {
				isBuilder = true
			}
		}

		if isWindows || isBuilder {
			continue
		}
		entries = append(entries, entry)
	}
	return &manifest.Manifest2822{
		Global:  man.Global,
		Entries: entries,
	}
}

// updateDockerfiles generates a new set of Dockerfiles based on the manifest
func updateDockerfiles(man *manifest.Manifest2822) error {
	for _, entry := range man.Entries {
		tag := entry.Tags[0]
		version := strings.Split(entry.Tags[0], "-")[0]
		for _, arch := range entry.Architectures {
			data := dockerfileData{Tag: tag, Version: version, Arch: arch}
			err := writeDockerfile(entry, data)
			if err != nil {
				return err
			}
		}
	}
	// TODO: Remove old Dockerfiles
	return nil
}

func writeDockerfile(entry manifest.Manifest2822Entry, data dockerfileData) error {
	dockerfilePath := path.Join(entry.ArchDirectory(data.Arch), entry.ArchFile(data.Arch))
	err := os.MkdirAll(entry.ArchDirectory(data.Arch), 0755)
	if err != nil {
		return err
	}
	dockerfileFile, err := os.OpenFile(dockerfilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer dockerfileFile.Close()

	err = dockerfileTemplate.Execute(dockerfileFile, data)
	if err != nil {
		return err
	}

	return nil
}

// updateManifest replaces upstream maintainer and Git info with this repo
func updateManifest(man *manifest.Manifest2822, gitCommit string) error {
	man.Global.Maintainers = maintainers
	for i := range man.Entries {
		entry := &man.Entries[i]
		entry.Maintainers = maintainers
		entry.GitRepo = gitRepo
		entry.GitCommit = gitCommit
	}
	manifestPath := path.Join("library", "caddy")
	err := os.MkdirAll("library", 0755)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(manifestPath, []byte(man.String()), 0644)
}

// updateReadme generates a new README.md
func updateReadme(man *manifest.Manifest2822) error {
	readmeFile, err := os.OpenFile("README.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer readmeFile.Close()

	err = readmeTemplate.Execute(readmeFile, man)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	Logln("updating")
	if err := golock.Lock(updateLockPath); err != nil {
		panic(err)
	}
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
		Logln("Timed out while updating")
	case <-done:
		touchAutoupdateFile()
	}
	Logln("done updating")
}

func updatePlugins() {
	Logln("updating plugins")
	b, _ := node.UpdatePackages()
	if len(b) > 0 {
		Logln("clearing plugins cache")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		return
	}
	Logln("updating from %s to %s (%s)", Version, manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 1*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command("./heroku-cli", "update").Start(); err != nil {
		panic(err)
	}
}
<commit_msg>added note for updates<commit_after>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	Errln("updating cli...")
	Logln("updating")
	if err := golock.Lock(updateLockPath); err != nil {
		panic(err)
	}
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
		Logln("Timed out while updating")
	case <-done:
		touchAutoupdateFile()
	}
	Logln("done updating")
}

func updatePlugins() {
	Logln("updating plugins")
	b, _ := node.UpdatePackages()
	if len(b) > 0 {
		Logln("clearing plugins cache")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		return
	}
	Logln("updating from %s to %s (%s)", Version, manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 1*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command("./heroku-cli", "update").Start(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package dns

// Implements wrapper functions for dealing with dynamic update packets.
// Dynamic update packets are identical to normal DNS messages, but the
// names are redefined. See RFC 2136 for the details.

type Update struct{ Msg }

// Not sure if I want to keep these functions, but they
// may help a programmer

func (u *Update) Zone() []Question {
	return u.Msg.Question
}

func (u *Update) Prereq() []RR {
	return u.Msg.Answer
}

func (u *Update) Update() []RR {
	return u.Msg.Ns
}

func (u *Update) Additional() []RR {
	return u.Msg.Extra
}

// NewUpdate creats a new DNS update packet.
func NewUpdate(zone string, class uint16) *Update {
        u := new(Update)
        u.MsgHdr.Opcode = OpcodeUpdate
        u.Question = make([]Question, 1)
        u.Question[0] = Question{zone, TypeSOA, class}
        return u
}

// 3.2.4 - Table Of Metavalues Used In Prerequisite Section
//
//   CLASS    TYPE     RDATA    Meaning
//   ------------------------------------------------------------
//   ANY      ANY      empty    Name is in use
//   ANY      rrset    empty    RRset exists (value independent)
//   NONE     ANY      empty    Name is not in use
//   NONE     rrset    empty    RRset does not exist
//   zone     rrset    rr       RRset exists (value dependent)

// NameUsed sets the RRs in the prereq section to
// "Name is in use" RRs. RFC 2136 section 2.4.4.
func (u *Update) NameUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
		u.Answer[i] = &RR_ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassANY}}
	}
}

// NameNotUsed sets the RRs in the prereq section to
// "Name is in not use" RRs. RFC 2136 section 2.4.5.
func (u *Update) NameNotUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = &RR_ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassNONE}}
	}
}

// RRsetUsedFull sets the RRs in the prereq section to
// "RRset exists (value dependent -- with rdata)" RRs. RFC 2136 section 2.4.2.
func (u *Update) RRsetUsedFull(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = u.Msg.Question[0].Qclass       // TODO crashes if question is zero
	}
}

// RRsetUsed sets the RRs in the prereq section to
// "RRset exists (value independent -- no rdata)" RRs. RFC 2136 section 2.4.1.
func (u *Update) RRsetUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = ClassANY
                /* rdata should be cleared */
	}
}

// RRsetNotUsed sets the RRs in the prereq section to
// "RRset does not exist" RRs. RFC 2136 section 2.4.3.
func (u *Update) RRsetNotUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = ClassNONE
                /* rdata should be cleared */
	}
}
<commit_msg>Need to think about naming<commit_after>package dns

// Implements wrapper functions for dealing with dynamic update packets.
// Dynamic update packets are identical to normal DNS messages, but the
// names are redefined. See RFC 2136 for the details.

type Update struct{ Msg }

// Not sure if I want to keep these functions, but they
// may help a programmer

func (u *Update) Zone() []Question {
	return u.Msg.Question
}

func (u *Update) Prereq() []RR {
	return u.Msg.Answer
}

func (u *Update) Update() []RR {
	return u.Msg.Ns
}

func (u *Update) Additional() []RR {
	return u.Msg.Extra
}

// NewUpdate creats a new DNS update packet.
func NewUpdate(zone string, class uint16) *Update {
        u := new(Update)
        u.MsgHdr.Opcode = OpcodeUpdate
        u.Question = make([]Question, 1)
        u.Question[0] = Question{zone, TypeSOA, class}
        return u
}

// 3.2.4 - Table Of Metavalues Used In Prerequisite Section
//
//   CLASS    TYPE     RDATA    Meaning
//   ------------------------------------------------------------
//   ANY      ANY      empty    Name is in use
//   ANY      rrset    empty    RRset exists (value independent)
//   NONE     ANY      empty    Name is not in use
//   NONE     rrset    empty    RRset does not exist
//   zone     rrset    rr       RRset exists (value dependent)

// NameUsed sets the RRs in the prereq section to
// "Name is in use" RRs. RFC 2136 section 2.4.4.
func (u *Update) NameUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
		u.Answer[i] = &RR_ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassANY}}
	}
}

// NameNotUsed sets the RRs in the prereq section to
// "Name is in not use" RRs. RFC 2136 section 2.4.5.
func (u *Update) NameNotUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = &RR_ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassNONE}}
	}
}

// RRsetUsedFull sets the RRs in the prereq section to
// "RRset exists (value dependent -- with rdata)" RRs. RFC 2136 section 2.4.2.
func (u *Update) RRsetUsedFull(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = u.Msg.Question[0].Qclass       // TODO crashes if question is zero
	}
}

// RRsetUsed sets the RRs in the prereq section to
// "RRset exists (value independent -- no rdata)" RRs. RFC 2136 section 2.4.1.
func (u *Update) RRsetUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = ClassANY
                u.Answer[i].Header().Rdlength = 0
	}
}

// RRsetNotUsed sets the RRs in the prereq section to
// "RRset does not exist" RRs. RFC 2136 section 2.4.3.
func (u *Update) RRsetNotUsed(rr []RR) {
	u.Answer = make([]RR, len(rr))
	for i, r := range rr {
                u.Answer[i] = r
                u.Answer[i].Header().Class = ClassNONE
                u.Answer[i].Header().Rdlength = 0
	}
}

// 3.4.2.6 - Table Of Metavalues Used In Update Section
//
//   CLASS    TYPE     RDATA    Meaning
//   ---------------------------------------------------------
//   ANY      ANY      empty    Delete all RRsets from a name
//   ANY      rrset    empty    Delete an RRset
//   NONE     rrset    rr       Delete an RR from an RRset
//   zone     rrset    rr       Add to an RRset

// RRsetAddFull adds an complete RRset, see RFC 2136 section 2.5.1
func (u *Update) RRsetAddFull(rr []RR) {
        u.Ns = make([]RR, len(rr))
        for i, r := range rr {
                u.Ns[i] = r
                u.Ns[i].Header().Class = u.Msg.Question[0].Qclass       // TODO crashes if question is zero
        }
}

// RRsetDeleteFull delete the full RR, see RFC 2136 section 2.5.2
func (u *Update) RRsetDeleteFull(rr []RR) {
        u.Ns = make([]RR, len(rr))
        for i, r := range rr {
                u.Ns[i] = r
                u.Ns[i].Header().Class = ClassNONE
        }
}

// RRsetDelete delete the RRset, see RFC 2136 section 2.5.2
func (u *Update) DeleteFull(rr []RR) {
        u.Ns = make([]RR, len(rr))
        for i, r := range rr {
                u.Ns[i] = r
                u.Ns[i].Header().Class = ClassNONE
        }
}
<|endoftext|>


<commit_before>package experimental

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

func expectHelpText(session *Session) {
	Eventually(session).Should(Say(`NAME:`))
	Eventually(session).Should(Say(`create-space - Create a space\n`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`USAGE:`))
	Eventually(session).Should(Say(`cf create-space SPACE \[-o ORG\] \[-q SPACE_QUOTA\]\n`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`OPTIONS:`))
	Eventually(session).Should(Say(`-o\s+Organization`))
	Eventually(session).Should(Say(`-q\s+Quota to assign to the newly created space`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`SEE ALSO:`))
	Eventually(session).Should(Say(`set-space-isolation-segment, space-quotas, spaces, target`))
}

func expectSuccessTextAndExitCode(session *Session, user, orgName, spaceName string) {
	Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
	Eventually(session).Should(Say(`OK\n`))
	Eventually(session).Should(Say(`Assigning role SpaceManager to user %s in org %s / space %s as %s\.\.\.`, user, orgName, spaceName, user))
	Eventually(session).Should(Say(`OK\n`))
	Eventually(session).Should(Say(`Assigning role SpaceDeveloper to user %s in org %s / space %s as %s\.\.\.`, user, orgName, spaceName, user))
	Eventually(session).Should(Say(`OK\n\n`))
	Eventually(session).Should(Say(`TIP: Use 'cf target -o "%s" -s "%s"' to target new space`, orgName, spaceName))
	Eventually(session).Should(Exit(0))
}

var _ = Describe("create-space", func() {
	var spaceName string

	When("invoked with --help", func() {
		It("displays the help information", func() {
			session := helpers.CF("create-space", "--help")
			expectHelpText(session)
			Eventually(session).Should(Exit(0))
		})
	})

	When("invoked with no arguments", func() {
		It("shows an error and the help text", func() {
			session := helpers.CF("create-space")
			Eventually(session.Err).Should(Say("Incorrect Usage: the required argument `SPACE` was not provided"))
			expectHelpText(session)
			Eventually(session).Should(Exit(1))
		})
	})

	When("the environment is not set up correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, "", "create-space", "some-space")
		})
	})

	BeforeEach(func() {
		spaceName = helpers.NewSpaceName()
	})

	When("logged in as a client", func() {
		var client, orgName string

		BeforeEach(func() {
			client = helpers.LoginCFWithClientCredentials()
			orgName = helpers.CreateAndTargetOrg()
		})

		It("successfully creates a space", func() {
			session := helpers.CF("create-space", spaceName)
			expectSuccessTextAndExitCode(session, client, orgName, spaceName)
		})
	})

	When("logged in as a user", func() {
		var user, orgName string

		BeforeEach(func() {
			user = helpers.LoginCF()
			orgName = helpers.CreateAndTargetOrg()
		})

		When("the space already exists", func() {
			BeforeEach(func() {
				session := helpers.CF("create-space", spaceName)
				Eventually(session).Should(Exit(0))
			})

			It("warns the user that the space already exists", func() {
				session := helpers.CF("create-space", spaceName)
				Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
				Eventually(session.Err).Should(Say(`Space %s already exists`, spaceName))
				Eventually(session).Should(Say(`OK\n`))
				Eventually(session).Should(Exit(0))
			})
		})

		When("the space does not exist yet", func() {
			When("a quota is not specified", func() {
				It("creates the space in the targeted org", func() {
					session := helpers.CF("create-space", spaceName)
					expectSuccessTextAndExitCode(session, user, orgName, spaceName)

					session = helpers.CF("space", spaceName)
					Eventually(session).Should(Say(`name:\s+%s`, spaceName))
					Eventually(session).Should(Say(`org:\s+%s`, orgName))
					Eventually(session).Should(Exit(0))
				})

				It("makes the user a space manager and space developer", func() {
					session := helpers.CF("create-space", spaceName)
					expectSuccessTextAndExitCode(session, user, orgName, spaceName)

					session = helpers.CF("space-users", orgName, spaceName)
					Eventually(session).Should(Say(`SPACE MANAGER\n\s+%s`, user))
					Eventually(session).Should(Say(`SPACE DEVELOPER\n\s+%s`, user))
					Eventually(session).Should(Exit(0))
				})
			})

			When("quota is specified", func() {
				var (
					quotaName string
					session   *Session
				)

				When("the quota exists", func() {
					BeforeEach(func() {
						quotaName = helpers.QuotaName()
						quotaSession := helpers.CF("create-space-quota", quotaName)
						Eventually(quotaSession).Should(Exit(0))
					})

					It("creates the space with the provided quota", func() {
						session = helpers.CF("create-space", spaceName, "-q", quotaName)
						expectSuccessTextAndExitCode(session, user, orgName, spaceName)
						session = helpers.CF("space", spaceName)
						Eventually(session).Should(Say(`name:\s+%s`, spaceName))
						Eventually(session).Should(Say(`space quota:\s+%s`, quotaName))
						Eventually(session).Should(Exit(0))
					})
				})

				When("the quota does not exist", func() {
					BeforeEach(func() {
						quotaName = "no-such-quota"
						session = helpers.CF("create-space", spaceName, "-q", quotaName)
					})

					It("fails with an error", func() {
						Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
						Eventually(session.Err).Should(Say(`Quota no-such-quota not found`))
						Eventually(session).Should(Say(`FAILED\n`))
						Eventually(session).Should(Exit(1))
					})

					It("does not create the space", func() {
						Eventually(helpers.CF("space", spaceName)).Should(Exit(1))
					})
				})
			})

			When("org is specified", func() {
				var session *Session

				When("the org exists", func() {
					BeforeEach(func() {
						orgName = helpers.NewOrgName()
						orgSession := helpers.CF("create-org", orgName)
						Eventually(orgSession).Should(Exit(0))
					})

					It("creates the space in the specified org", func() {
						session = helpers.CF("create-space", spaceName, "-o", orgName)
						expectSuccessTextAndExitCode(session, user, orgName, spaceName)

						helpers.TargetOrg(orgName)
						session = helpers.CF("space", spaceName)
						Eventually(session).Should(Say(`name:\s+%s`, spaceName))
						Eventually(session).Should(Say(`org:\s+%s`, orgName))
						Eventually(session).Should(Exit(0))
					})
				})

				When("the org does not exist", func() {
					BeforeEach(func() {
						orgName = "no-such-org"
					})

					It("fails with an error and does not create the space", func() {
						session = helpers.CF("create-space", spaceName, "-o", orgName)
						Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
						Eventually(session.Err).Should(Say(`Organization '%s' not found`, orgName))
						Eventually(session).Should(Say(`FAILED\n`))
						Eventually(session).Should(Exit(1))

						Eventually(helpers.CF("space", spaceName)).Should(Exit(1))
					})
				})
			})

			When("the user is not authorized to create a space", func() {
				var user string

				BeforeEach(func() {
					user = helpers.SwitchToOrgRole(orgName, "OrgAuditor")
				})

				AfterEach(func() {
					helpers.ClearTarget()
					Expect(user).To(MatchRegexp(`^INTEGRATION-USER-[\da-f-]+$`))
					helpers.DeleteUser(user)
				})

				It("fails with an error telling the user that they are not authorized", func() {
					session := helpers.CF("create-space", spaceName, "-o", orgName)
					Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
					Eventually(session.Err).Should(Say(`You are not authorized to perform the requested action`))
					Eventually(session).Should(Say(`FAILED\n`))
					Eventually(session).Should(Exit(1))
				})
			})
		})
	})
})
<commit_msg>Pend create-space test for client credentials workflow<commit_after>package experimental

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

func expectHelpText(session *Session) {
	Eventually(session).Should(Say(`NAME:`))
	Eventually(session).Should(Say(`create-space - Create a space\n`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`USAGE:`))
	Eventually(session).Should(Say(`cf create-space SPACE \[-o ORG\] \[-q SPACE_QUOTA\]\n`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`OPTIONS:`))
	Eventually(session).Should(Say(`-o\s+Organization`))
	Eventually(session).Should(Say(`-q\s+Quota to assign to the newly created space`))
	Eventually(session).Should(Say(`\n`))

	Eventually(session).Should(Say(`SEE ALSO:`))
	Eventually(session).Should(Say(`set-space-isolation-segment, space-quotas, spaces, target`))
}

func expectSuccessTextAndExitCode(session *Session, user, orgName, spaceName string) {
	Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
	Eventually(session).Should(Say(`OK\n`))
	Eventually(session).Should(Say(`Assigning role SpaceManager to user %s in org %s / space %s as %s\.\.\.`, user, orgName, spaceName, user))
	Eventually(session).Should(Say(`OK\n`))
	Eventually(session).Should(Say(`Assigning role SpaceDeveloper to user %s in org %s / space %s as %s\.\.\.`, user, orgName, spaceName, user))
	Eventually(session).Should(Say(`OK\n\n`))
	Eventually(session).Should(Say(`TIP: Use 'cf target -o "%s" -s "%s"' to target new space`, orgName, spaceName))
	Eventually(session).Should(Exit(0))
}

var _ = Describe("create-space", func() {
	var spaceName string

	When("invoked with --help", func() {
		It("displays the help information", func() {
			session := helpers.CF("create-space", "--help")
			expectHelpText(session)
			Eventually(session).Should(Exit(0))
		})
	})

	When("invoked with no arguments", func() {
		It("shows an error and the help text", func() {
			session := helpers.CF("create-space")
			Eventually(session.Err).Should(Say("Incorrect Usage: the required argument `SPACE` was not provided"))
			expectHelpText(session)
			Eventually(session).Should(Exit(1))
		})
	})

	When("the environment is not set up correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, "", "create-space", "some-space")
		})
	})

	BeforeEach(func() {
		spaceName = helpers.NewSpaceName()
	})

	PWhen("logged in as a client", func() {
		var client, orgName string

		BeforeEach(func() {
			client = helpers.LoginCFWithClientCredentials()
			orgName = helpers.CreateAndTargetOrg()
		})

		It("successfully creates a space", func() {
			session := helpers.CF("create-space", spaceName)
			expectSuccessTextAndExitCode(session, client, orgName, spaceName)
		})
	})

	When("logged in as a user", func() {
		var user, orgName string

		BeforeEach(func() {
			user = helpers.LoginCF()
			orgName = helpers.CreateAndTargetOrg()
		})

		When("the space already exists", func() {
			BeforeEach(func() {
				session := helpers.CF("create-space", spaceName)
				Eventually(session).Should(Exit(0))
			})

			It("warns the user that the space already exists", func() {
				session := helpers.CF("create-space", spaceName)
				Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
				Eventually(session.Err).Should(Say(`Space %s already exists`, spaceName))
				Eventually(session).Should(Say(`OK\n`))
				Eventually(session).Should(Exit(0))
			})
		})

		When("the space does not exist yet", func() {
			When("a quota is not specified", func() {
				It("creates the space in the targeted org", func() {
					session := helpers.CF("create-space", spaceName)
					expectSuccessTextAndExitCode(session, user, orgName, spaceName)

					session = helpers.CF("space", spaceName)
					Eventually(session).Should(Say(`name:\s+%s`, spaceName))
					Eventually(session).Should(Say(`org:\s+%s`, orgName))
					Eventually(session).Should(Exit(0))
				})

				It("makes the user a space manager and space developer", func() {
					session := helpers.CF("create-space", spaceName)
					expectSuccessTextAndExitCode(session, user, orgName, spaceName)

					session = helpers.CF("space-users", orgName, spaceName)
					Eventually(session).Should(Say(`SPACE MANAGER\n\s+%s`, user))
					Eventually(session).Should(Say(`SPACE DEVELOPER\n\s+%s`, user))
					Eventually(session).Should(Exit(0))
				})
			})

			When("quota is specified", func() {
				var (
					quotaName string
					session   *Session
				)

				When("the quota exists", func() {
					BeforeEach(func() {
						quotaName = helpers.QuotaName()
						quotaSession := helpers.CF("create-space-quota", quotaName)
						Eventually(quotaSession).Should(Exit(0))
					})

					It("creates the space with the provided quota", func() {
						session = helpers.CF("create-space", spaceName, "-q", quotaName)
						expectSuccessTextAndExitCode(session, user, orgName, spaceName)
						session = helpers.CF("space", spaceName)
						Eventually(session).Should(Say(`name:\s+%s`, spaceName))
						Eventually(session).Should(Say(`space quota:\s+%s`, quotaName))
						Eventually(session).Should(Exit(0))
					})
				})

				When("the quota does not exist", func() {
					BeforeEach(func() {
						quotaName = "no-such-quota"
						session = helpers.CF("create-space", spaceName, "-q", quotaName)
					})

					It("fails with an error", func() {
						Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
						Eventually(session.Err).Should(Say(`Quota no-such-quota not found`))
						Eventually(session).Should(Say(`FAILED\n`))
						Eventually(session).Should(Exit(1))
					})

					It("does not create the space", func() {
						Eventually(helpers.CF("space", spaceName)).Should(Exit(1))
					})
				})
			})

			When("org is specified", func() {
				var session *Session

				When("the org exists", func() {
					BeforeEach(func() {
						orgName = helpers.NewOrgName()
						orgSession := helpers.CF("create-org", orgName)
						Eventually(orgSession).Should(Exit(0))
					})

					It("creates the space in the specified org", func() {
						session = helpers.CF("create-space", spaceName, "-o", orgName)
						expectSuccessTextAndExitCode(session, user, orgName, spaceName)

						helpers.TargetOrg(orgName)
						session = helpers.CF("space", spaceName)
						Eventually(session).Should(Say(`name:\s+%s`, spaceName))
						Eventually(session).Should(Say(`org:\s+%s`, orgName))
						Eventually(session).Should(Exit(0))
					})
				})

				When("the org does not exist", func() {
					BeforeEach(func() {
						orgName = "no-such-org"
					})

					It("fails with an error and does not create the space", func() {
						session = helpers.CF("create-space", spaceName, "-o", orgName)
						Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
						Eventually(session.Err).Should(Say(`Organization '%s' not found`, orgName))
						Eventually(session).Should(Say(`FAILED\n`))
						Eventually(session).Should(Exit(1))

						Eventually(helpers.CF("space", spaceName)).Should(Exit(1))
					})
				})
			})

			When("the user is not authorized to create a space", func() {
				var user string

				BeforeEach(func() {
					user = helpers.SwitchToOrgRole(orgName, "OrgAuditor")
				})

				AfterEach(func() {
					helpers.ClearTarget()
					Expect(user).To(MatchRegexp(`^INTEGRATION-USER-[\da-f-]+$`))
					helpers.DeleteUser(user)
				})

				It("fails with an error telling the user that they are not authorized", func() {
					session := helpers.CF("create-space", spaceName, "-o", orgName)
					Eventually(session).Should(Say(`Creating space %s in org %s as %s\.\.\.`, spaceName, orgName, user))
					Eventually(session.Err).Should(Say(`You are not authorized to perform the requested action`))
					Eventually(session).Should(Say(`FAILED\n`))
					Eventually(session).Should(Exit(1))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrei Matei (andrei@cockroachlabs.com)

// Package crdb provides helpers for using CockroachDB in client
// applications.
package crdb

import (
	"database/sql"

	"github.com/cockroachdb/pq"
)

// AmbiguousCommitError represents an error that left a transaction in an
// ambiguous state: unclear if it committed or not.
type AmbiguousCommitError struct {
	error
}

// ExecuteTx runs fn inside a transaction and retries it as needed.
// On non-retryable failures, the transaction is aborted and rolled
// back; on success, the transaction is committed.
// There are cases where the state of a transaction is inherently ambiguous: if
// we err on RELEASE with a communication error it's unclear if the transaction
// has been committed or not (similar to erroring on COMMIT in other databases).
// In that case, we return AmbiguousCommitError.
//
// For more information about CockroachDB's transaction model see
// https://cockroachlabs.com/docs/transactions.html.
//
// NOTE: the supplied exec closure should not have external side
// effects beyond changes to the database.
func ExecuteTx(db *sql.DB, fn func(*sql.Tx) error) (err error) {
	// Start a transaction.
	var tx *sql.Tx
	tx, err = db.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if err == nil {
			// Ignore commit errors. The tx has already been committed by RELEASE.
			_ = tx.Commit()
		} else {
			// We always need to execute a Rollback() so sql.DB releases the
			// connection.
			_ = tx.Rollback()
		}
	}()
	// Specify that we intend to retry this txn in case of CockroachDB retryable
	// errors.
	if _, err = tx.Exec("SAVEPOINT cockroach_restart"); err != nil {
		return err
	}

	for {
		released := false
		err = fn(tx)
		if err == nil {
			// RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an
			// opportunity to react to retryable errors, whereas tx.Commit() doesn't.
			released = true
			if _, err = tx.Exec("RELEASE SAVEPOINT cockroach_restart"); err == nil {
				return nil
			}
		}
		// We got an error; let's see if it's a retryable one and, if so, restart.
		pqErr, ok := err.(*pq.Error)
		if retryable := ok && pqErr.Code == "CR000"; !retryable {
			if released {
				err = &AmbiguousCommitError{err}
			}
			return err
		}
		if _, err = tx.Exec("ROLLBACK TO SAVEPOINT cockroach_restart"); err != nil {
			return err
		}
	}
}
<commit_msg>Handle SerializationFailureError for retryable errors<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrei Matei (andrei@cockroachlabs.com)

// Package crdb provides helpers for using CockroachDB in client
// applications.
package crdb

import (
	"database/sql"

	"github.com/cockroachdb/pq"
)

// AmbiguousCommitError represents an error that left a transaction in an
// ambiguous state: unclear if it committed or not.
type AmbiguousCommitError struct {
	error
}

// ExecuteTx runs fn inside a transaction and retries it as needed.
// On non-retryable failures, the transaction is aborted and rolled
// back; on success, the transaction is committed.
// There are cases where the state of a transaction is inherently ambiguous: if
// we err on RELEASE with a communication error it's unclear if the transaction
// has been committed or not (similar to erroring on COMMIT in other databases).
// In that case, we return AmbiguousCommitError.
//
// For more information about CockroachDB's transaction model see
// https://cockroachlabs.com/docs/transactions.html.
//
// NOTE: the supplied exec closure should not have external side
// effects beyond changes to the database.
func ExecuteTx(db *sql.DB, fn func(*sql.Tx) error) (err error) {
	// Start a transaction.
	var tx *sql.Tx
	tx, err = db.Begin()
	if err != nil {
		return err
	}
	defer func() {
		if err == nil {
			// Ignore commit errors. The tx has already been committed by RELEASE.
			_ = tx.Commit()
		} else {
			// We always need to execute a Rollback() so sql.DB releases the
			// connection.
			_ = tx.Rollback()
		}
	}()
	// Specify that we intend to retry this txn in case of CockroachDB retryable
	// errors.
	if _, err = tx.Exec("SAVEPOINT cockroach_restart"); err != nil {
		return err
	}

	for {
		released := false
		err = fn(tx)
		if err == nil {
			// RELEASE acts like COMMIT in CockroachDB. We use it since it gives us an
			// opportunity to react to retryable errors, whereas tx.Commit() doesn't.
			released = true
			if _, err = tx.Exec("RELEASE SAVEPOINT cockroach_restart"); err == nil {
				return nil
			}
		}
		// We got an error; let's see if it's a retryable one and, if so, restart. We look
		// for either the standard PG errcode SerializationFailureError:40001 or the Cockroach extension
		// errcode RetriableError:CR000. The Cockroach extension has been removed server-side, but support
		// for it has been left here for now to maintain backwards compatibility.
		pqErr, ok := err.(*pq.Error)
		if retryable := ok && (pqErr.Code == "CR000" || pqErr.Code == "40001"); !retryable {
			if released {
				err = &AmbiguousCommitError{err}
			}
			return err
		}
		if _, err = tx.Exec("ROLLBACK TO SAVEPOINT cockroach_restart"); err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>package s32cs

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudsearchdomain"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/pkg/errors"
)

const MaxUploadSize = 5 * 1024 * 1024

var (
	openBracket  = []byte{'['}
	closeBracket = []byte{']'}
	comma        = []byte{','}
	DEBUG        = false
)

type buffer struct {
	bytes.Buffer
}

func (b *buffer) init() {
	b.Reset()
	b.Write(openBracket)
}

func (b *buffer) close() {
	b.Write(closeBracket)
}

func (b *buffer) allowAppend(bs []byte) bool {
	return b.Len()+len(bs)+2 < MaxUploadSize
}

func (b *buffer) append(bs []byte) {
	if b.Len() > 1 {
		b.Write(comma)
	}
	b.Write(bs)
}

type Domain struct {
	domain *cloudsearchdomain.CloudSearchDomain
	s3     *s3manager.Downloader
	queue  *sqs.SQS
	buf    *buffer
}

func NewDomain(csSess *session.Session, sess *session.Session) *Domain {
	buf := &buffer{}
	buf.Grow(MaxUploadSize)
	buf.init()
	return &Domain{
		domain: cloudsearchdomain.New(csSess),
		s3:     s3manager.NewDownloader(sess),
		queue:  sqs.New(sess),
		buf:    buf,
	}
}

func (d *Domain) Process(event S3Event) error {
	for _, record := range event.Records {
		name, key := record.S3.Bucket.Name, record.S3.Object.Key
		r, err := d.fetch(name, key)
		if err != nil {
			return errors.Wrap(err, "fetch failed")
		}
		defer r.Close()
		if strings.HasSuffix(key, ".gz") {
			r, err = gzip.NewReader(r)
			if err != nil {
				return errors.Wrap(err, "new gzip reader failed")
			}
		}

		if err = d.Upload(r); err != nil {
			return errors.Wrap(err, "upload failed")
		}
	}
	return nil
}

func (d *Domain) fetch(bucket, key string) (io.ReadCloser, error) {
	tmp, err := ioutil.TempFile(os.TempDir(), "s32cs")
	if err != nil {
		return nil, errors.Wrap(err, "create tempfile failed")
	}
	defer os.Remove(tmp.Name())

	log.Printf("downloading s3://%s/%s", bucket, key)
	n, err := d.s3.Download(tmp, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, errors.Wrap(err, "download failed")
	}
	log.Printf("%d bytes fetched", n)
	tmp.Seek(0, os.SEEK_SET)
	return tmp, nil
}

func (d *Domain) Upload(src io.Reader) error {
	dec := json.NewDecoder(src)
	for {
		var record SDFRecord
		if err := dec.Decode(&record); err != nil {
			if err == io.EOF {
				break
			}
			log.Printf("decode json failed %s", err)
			continue
		}
		if err := record.Validate(); err != nil {
			log.Printf("SDF record validation failed %s %#v", err, record)
			continue
		}
		bs, err := json.Marshal(record)
		if err != nil {
			return err
		}
		if !d.buf.allowAppend(bs) {
			err := d.flush()
			if err != nil {
				return err
			}
		}
		d.buf.append(bs)
	}
	return d.flush()
}

func (d *Domain) flush() error {
	defer d.buf.init()
	d.buf.close()
	log.Printf("starting upload %d bytes", d.buf.Len())
	if DEBUG {
		log.Println(string(d.buf.Bytes()))
	}
	out, err := d.domain.UploadDocuments(
		&cloudsearchdomain.UploadDocumentsInput{
			ContentType: aws.String("application/json"),
			Documents:   bytes.NewReader(d.buf.Bytes()),
		},
	)
	if err != nil {
		return errors.Wrap(err, "UploadDocuments failed")
	}
	log.Println("upload completed", out.String())
	return nil
}
<commit_msg>refactor<commit_after>package s32cs

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudsearchdomain"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/pkg/errors"
)

const MaxUploadSize = 5 * 1024 * 1024

var (
	openBracket  = []byte{'['}
	closeBracket = []byte{']'}
	comma        = []byte{','}
	DEBUG        = false
)

type buffer struct {
	bytes.Buffer
}

func (b *buffer) init() {
	b.Reset()
	b.Write(openBracket)
}

func (b *buffer) close() {
	b.Write(closeBracket)
}

func (b *buffer) allowAppend(bs []byte) bool {
	return b.Len()+len(bs)+2 < MaxUploadSize
}

func (b *buffer) append(bs []byte) {
	if b.Len() > 1 {
		b.Write(comma)
	}
	b.Write(bs)
}

type Domain struct {
	domain *cloudsearchdomain.CloudSearchDomain
	s3     *s3manager.Downloader
	queue  *sqs.SQS
	buf    *buffer
}

func NewDomain(csSess *session.Session, sess *session.Session) *Domain {
	buf := &buffer{}
	buf.Grow(MaxUploadSize)
	buf.init()
	return &Domain{
		domain: cloudsearchdomain.New(csSess),
		s3:     s3manager.NewDownloader(sess),
		queue:  sqs.New(sess),
		buf:    buf,
	}
}

func (d *Domain) Process(event S3Event) error {
	for _, record := range event.Records {
		name, key := record.S3.Bucket.Name, record.S3.Object.Key
		r, err := d.fetch(name, key)
		if err != nil {
			return errors.Wrap(err, "fetch failed")
		}
		defer r.Close()

		if err = d.Upload(r); err != nil {
			return errors.Wrap(err, "upload failed")
		}
	}
	return nil
}

func (d *Domain) fetch(bucket, key string) (io.ReadCloser, error) {
	tmp, err := ioutil.TempFile(os.TempDir(), "s32cs")
	if err != nil {
		return nil, errors.Wrap(err, "create tempfile failed")
	}
	defer os.Remove(tmp.Name())

	log.Printf("downloading s3://%s/%s", bucket, key)
	n, err := d.s3.Download(tmp, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, errors.Wrap(err, "download failed")
	}
	log.Printf("%d bytes fetched", n)
	tmp.Seek(0, os.SEEK_SET)

	if strings.HasSuffix(key, ".gz") {
		return gzip.NewReader(tmp)
	}
	return tmp, nil
}

func (d *Domain) Upload(src io.Reader) error {
	dec := json.NewDecoder(src)
	for {
		var record SDFRecord
		if err := dec.Decode(&record); err != nil {
			if err == io.EOF {
				break
			}
			log.Printf("decode json failed %s", err)
			continue
		}
		if err := record.Validate(); err != nil {
			log.Printf("SDF record validation failed %s %#v", err, record)
			continue
		}
		bs, err := json.Marshal(record)
		if err != nil {
			return err
		}
		if !d.buf.allowAppend(bs) {
			err := d.flush()
			if err != nil {
				return err
			}
		}
		d.buf.append(bs)
	}
	return d.flush()
}

func (d *Domain) flush() error {
	defer d.buf.init()
	d.buf.close()
	log.Printf("starting upload %d bytes", d.buf.Len())
	if DEBUG {
		log.Println(string(d.buf.Bytes()))
	}
	out, err := d.domain.UploadDocuments(
		&cloudsearchdomain.UploadDocumentsInput{
			ContentType: aws.String("application/json"),
			Documents:   bytes.NewReader(d.buf.Bytes()),
		},
	)
	if err != nil {
		return errors.Wrap(err, "UploadDocuments failed")
	}
	log.Println("upload completed", out.String())
	return nil
}
<|endoftext|>