<commit_before>/*
Copyright 2017 by the contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/kubernetes-sigs/aws-iam-authenticator/pkg/arn"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientauthv1alpha1 "k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
)

// Identity is returned on successful Verify() results. It contains a parsed
// version of the AWS identity used to create the token.
type Identity struct {
	// ARN is the raw Amazon Resource Name returned by sts:GetCallerIdentity
	ARN string

	// CanonicalARN is the Amazon Resource Name converted to a more canonical
	// representation. In particular, STS assumed role ARNs like
	// "arn:aws:sts::ACCOUNTID:assumed-role/ROLENAME/SESSIONNAME" are converted
	// to their IAM ARN equivalent "arn:aws:iam::ACCOUNTID:role/NAME"
	CanonicalARN string

	// AccountID is the 12 digit AWS account number.
	AccountID string

	// UserID is the unique user/role ID (e.g., "AROAAAAAAAAAAAAAAAAAA").
	UserID string

	// SessionName is the STS session name (or "" if this is not a
	// session-based identity). For EC2 instance roles, this will be the EC2
	// instance ID (e.g., "i-0123456789abcdef0"). You should only rely on it
	// if you trust that _only_ EC2 is allowed to assume the IAM Role. If IAM
	// users or other roles are allowed to assume the role, they can provide
	// (nearly) arbitrary strings here.
	SessionName string
}

const (
	v1Prefix         = "k8s-aws-v1."
	maxTokenLenBytes = 1024 * 4
	clusterIDHeader  = "x-k8s-aws-id"
)

// FormatError is returned when there is a problem with token that is
// an encoded sts request.  This can include the url, data, action or anything
// else that prevents the sts call from being made.
type FormatError struct {
	message string
}

func (e FormatError) Error() string {
	return "input token was not properly formatted: " + e.message
}

// STSError is returned when there was either an error calling STS or a problem
// processing the data returned from STS.
type STSError struct {
	message string
}

func (e STSError) Error() string {
	return "sts getCallerIdentity failed: " + e.message
}

// NewSTSError creates a error of type STS.
func NewSTSError(m string) STSError {
	return STSError{message: m}
}

var parameterWhitelist = map[string]bool{
	"action":               true,
	"version":              true,
	"x-amz-algorithm":      true,
	"x-amz-credential":     true,
	"x-amz-date":           true,
	"x-amz-expires":        true,
	"x-amz-security-token": true,
	"x-amz-signature":      true,
	"x-amz-signedheaders":  true,
}

// this is the result type from the GetCallerIdentity endpoint
type getCallerIdentityWrapper struct {
	GetCallerIdentityResponse struct {
		GetCallerIdentityResult struct {
			Account string `json:"Account"`
			Arn     string `json:"Arn"`
			UserID  string `json:"UserId"`
		} `json:"GetCallerIdentityResult"`
		ResponseMetadata struct {
			RequestID string `json:"RequestId"`
		} `json:"ResponseMetadata"`
	} `json:"GetCallerIdentityResponse"`
}

// Generator provides new tokens for the heptio authenticator.
type Generator interface {
	// Get a token using credentials in the default credentials chain.
	Get(string) (string, error)
	// GetWithRole creates a token by assuming the provided role, using the credentials in the default chain.
	GetWithRole(clusterID, roleARN string) (string, error)
	// GetWithRoleForSession creates a token by assuming the provided role, using the provided session.
	GetWithRoleForSession(clusterID string, roleARN string, sess *session.Session) (string, error)
	// GetWithSTS assumes returns a token valid for clusterID using the given STS client.
	GetWithSTS(clusterID string, stsAPI *sts.STS) (string, error)
	// FormatJSON returns the client auth formatted json for the ExecCredential auth
	FormatJSON(string) string
}

type generator struct {
}

// NewGenerator creates a Generator and returns it.
func NewGenerator() (Generator, error) {
	return generator{}, nil
}

// Get uses the directly available AWS credentials to return a token valid for
// clusterID. It follows the default AWS credential handling behavior.
func (g generator) Get(clusterID string) (string, error) {
	return g.GetWithRole(clusterID, "")
}

func StdinStderrTokenProvider() (string, error) {
	var v string
	fmt.Fprint(os.Stderr, "Assume Role MFA token code: ")
	_, err := fmt.Scanln(&v)
	return v, err
}

// GetWithRole assumes the given AWS IAM role and returns a token valid for
// clusterID. If roleARN is empty, behaves like Get (does not assume a role).
func (g generator) GetWithRole(clusterID string, roleARN string) (string, error) {
	// create a session with the "base" credentials available
	// (from environment variable, profile files, EC2 metadata, etc)
	sess, err := session.NewSessionWithOptions(session.Options{
		AssumeRoleTokenProvider: StdinStderrTokenProvider,
		SharedConfigState:       session.SharedConfigEnable,
	})
	if err != nil {
		return "", fmt.Errorf("could not create session: %v", err)
	}

	return g.GetWithRoleForSession(clusterID, roleARN, sess)
}

// GetWithRole assumes the given AWS IAM role for the given session and behaves
// like GetWithRole.
func (g generator) GetWithRoleForSession(clusterID string, roleARN string, sess *session.Session) (string, error) {
	// use an STS client based on the direct credentials
	stsAPI := sts.New(sess)

	// if a roleARN was specified, replace the STS client with one that uses
	// temporary credentials from that role.
	if roleARN != "" {
		// create STS-based credentials that will assume the given role
		creds := stscreds.NewCredentials(sess, roleARN)

		// create an STS API interface that uses the assumed role's temporary credentials
		stsAPI = sts.New(sess, &aws.Config{Credentials: creds})
	}

	return g.GetWithSTS(clusterID, stsAPI)
}

// GetWithSTS assumes returns a token valid for clusterID using the given STS client.
func (g generator) GetWithSTS(clusterID string, stsAPI *sts.STS) (string, error) {
	// generate an sts:GetCallerIdentity request and add our custom cluster ID header
	request, _ := stsAPI.GetCallerIdentityRequest(&sts.GetCallerIdentityInput{})
	request.HTTPRequest.Header.Add(clusterIDHeader, clusterID)

	// sign the request
	presignedURLString, err := request.Presign(60 * time.Second)
	if err != nil {
		return "", err
	}

	// TODO: this may need to be a constant-time base64 encoding
	return v1Prefix + base64.RawURLEncoding.EncodeToString([]byte(presignedURLString)), nil
}

// FormatJSON formats the json to support ExecCredential authentication
func (g generator) FormatJSON(token string) string {
	execInput := &clientauthv1alpha1.ExecCredential{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "client.authentication.k8s.io/v1alpha1",
			Kind:       "ExecCredential",
		},
		Status: &clientauthv1alpha1.ExecCredentialStatus{
			Token: token,
		},
	}
	enc, _ := json.Marshal(execInput)
	return string(enc)
}

// Verifier validates tokens by calling STS and returning the associated identity.
type Verifier interface {
	Verify(token string) (*Identity, error)
}

type tokenVerifier struct {
	client    *http.Client
	clusterID string
}

// NewVerifier creates a Verifier that is bound to the clusterID and uses the default http client.
func NewVerifier(clusterID string) Verifier {
	return tokenVerifier{
		client:    http.DefaultClient,
		clusterID: clusterID,
	}
}

// Verify a token is valid for the specified clusterID. On success, returns an
// Identity that contains information about the AWS principal that created the
// token. On failure, returns nil and a non-nil error.
func (v tokenVerifier) Verify(token string) (*Identity, error) {
	if len(token) > maxTokenLenBytes {
		return nil, FormatError{"token is too large"}
	}

	if !strings.HasPrefix(token, v1Prefix) {
		return nil, FormatError{fmt.Sprintf("token is missing expected %q prefix", v1Prefix)}
	}

	// TODO: this may need to be a constant-time base64 decoding
	tokenBytes, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(token, v1Prefix))
	if err != nil {
		return nil, FormatError{err.Error()}
	}

	parsedURL, err := url.Parse(string(tokenBytes))
	if err != nil {
		return nil, FormatError{err.Error()}
	}

	if parsedURL.Scheme != "https" {
		return nil, FormatError{fmt.Sprintf("unexpected scheme %q in pre-signed URL", parsedURL.Scheme)}
	}

	if parsedURL.Host != "sts.amazonaws.com" {
		return nil, FormatError{"unexpected hostname in pre-signed URL"}
	}

	if parsedURL.Path != "/" {
		return nil, FormatError{"unexpected path in pre-signed URL"}
	}

	queryParamsLower := make(url.Values)
	queryParams := parsedURL.Query()
	for key, values := range queryParams {
		if !parameterWhitelist[strings.ToLower(key)] {
			return nil, FormatError{fmt.Sprintf("non-whitelisted query parameter %q", key)}
		}
		if len(values) != 1 {
			return nil, FormatError{"query parameter with multiple values not supported"}
		}
		queryParamsLower.Set(strings.ToLower(key), values[0])
	}

	if queryParamsLower.Get("action") != "GetCallerIdentity" {
		return nil, FormatError{"unexpected action parameter in pre-signed URL"}
	}

	if !hasSignedClusterIDHeader(&queryParamsLower) {
		return nil, FormatError{fmt.Sprintf("client did not sign the %s header in the pre-signed URL", clusterIDHeader)}
	}

	expires, err := strconv.Atoi(queryParamsLower.Get("x-amz-expires"))
	if err != nil || expires < 0 || expires > 60 {
		return nil, FormatError{"invalid X-Amz-Expires parameter in pre-signed URL"}
	}

	req, err := http.NewRequest("GET", parsedURL.String(), nil)
	req.Header.Set(clusterIDHeader, v.clusterID)
	req.Header.Set("accept", "application/json")

	response, err := v.client.Do(req)
	if err != nil {
		// special case to avoid printing the full URL if possible
		if urlErr, ok := err.(*url.Error); ok {
			return nil, NewSTSError(fmt.Sprintf("error during GET: %v", urlErr.Err))
		}
		return nil, NewSTSError(fmt.Sprintf("error during GET: %v", err))
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, NewSTSError(fmt.Sprintf("error from AWS (expected 200, got %d)", response.StatusCode))
	}

	responseBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, NewSTSError(fmt.Sprintf("error reading HTTP result: %v", err))
	}

	var callerIdentity getCallerIdentityWrapper
	err = json.Unmarshal(responseBody, &callerIdentity)
	if err != nil {
		return nil, NewSTSError(err.Error())
	}

	// parse the response into an Identity
	id := &Identity{
		ARN:       callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
		AccountID: callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
	}
	id.CanonicalARN, err = arn.Canonicalize(id.ARN)
	if err != nil {
		return nil, NewSTSError(err.Error())
	}

	// The user ID is either UserID:SessionName (for assumed roles) or just
	// UserID (for IAM User principals).
	userIDParts := strings.Split(callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.UserID, ":")
	if len(userIDParts) == 2 {
		id.UserID = userIDParts[0]
		id.SessionName = userIDParts[1]
	} else if len(userIDParts) == 1 {
		id.UserID = userIDParts[0]
	} else {
		return nil, STSError{fmt.Sprintf(
			"malformed UserID %q",
			callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.UserID)}
	}

	return id, nil
}

func hasSignedClusterIDHeader(paramsLower *url.Values) bool {
	signedHeaders := strings.Split(paramsLower.Get("x-amz-signedheaders"), ";")
	for _, hdr := range signedHeaders {
		if strings.ToLower(hdr) == strings.ToLower(clusterIDHeader) {
			return true
		}
	}
	return false
}
<commit_msg>Mapped existing sts caller specified role name<commit_after>/*
Copyright 2017 by the contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/kubernetes-sigs/aws-iam-authenticator/pkg/arn"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientauthv1alpha1 "k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
)

// Identity is returned on successful Verify() results. It contains a parsed
// version of the AWS identity used to create the token.
type Identity struct {
	// ARN is the raw Amazon Resource Name returned by sts:GetCallerIdentity
	ARN string

	// CanonicalARN is the Amazon Resource Name converted to a more canonical
	// representation. In particular, STS assumed role ARNs like
	// "arn:aws:sts::ACCOUNTID:assumed-role/ROLENAME/SESSIONNAME" are converted
	// to their IAM ARN equivalent "arn:aws:iam::ACCOUNTID:role/NAME"
	CanonicalARN string

	// AccountID is the 12 digit AWS account number.
	AccountID string

	// UserID is the unique user/role ID (e.g., "AROAAAAAAAAAAAAAAAAAA").
	UserID string

	// SessionName is the STS session name (or "" if this is not a
	// session-based identity). For EC2 instance roles, this will be the EC2
	// instance ID (e.g., "i-0123456789abcdef0"). You should only rely on it
	// if you trust that _only_ EC2 is allowed to assume the IAM Role. If IAM
	// users or other roles are allowed to assume the role, they can provide
	// (nearly) arbitrary strings here.
	SessionName string
}

const (
	v1Prefix         = "k8s-aws-v1."
	maxTokenLenBytes = 1024 * 4
	clusterIDHeader  = "x-k8s-aws-id"
)

// FormatError is returned when there is a problem with token that is
// an encoded sts request.  This can include the url, data, action or anything
// else that prevents the sts call from being made.
type FormatError struct {
	message string
}

func (e FormatError) Error() string {
	return "input token was not properly formatted: " + e.message
}

// STSError is returned when there was either an error calling STS or a problem
// processing the data returned from STS.
type STSError struct {
	message string
}

func (e STSError) Error() string {
	return "sts getCallerIdentity failed: " + e.message
}

// NewSTSError creates a error of type STS.
func NewSTSError(m string) STSError {
	return STSError{message: m}
}

var parameterWhitelist = map[string]bool{
	"action":               true,
	"version":              true,
	"x-amz-algorithm":      true,
	"x-amz-credential":     true,
	"x-amz-date":           true,
	"x-amz-expires":        true,
	"x-amz-security-token": true,
	"x-amz-signature":      true,
	"x-amz-signedheaders":  true,
}

// this is the result type from the GetCallerIdentity endpoint
type getCallerIdentityWrapper struct {
	GetCallerIdentityResponse struct {
		GetCallerIdentityResult struct {
			Account string `json:"Account"`
			Arn     string `json:"Arn"`
			UserID  string `json:"UserId"`
		} `json:"GetCallerIdentityResult"`
		ResponseMetadata struct {
			RequestID string `json:"RequestId"`
		} `json:"ResponseMetadata"`
	} `json:"GetCallerIdentityResponse"`
}

// Generator provides new tokens for the heptio authenticator.
type Generator interface {
	// Get a token using credentials in the default credentials chain.
	Get(string) (string, error)
	// GetWithRole creates a token by assuming the provided role, using the credentials in the default chain.
	GetWithRole(clusterID, roleARN string) (string, error)
	// GetWithRoleForSession creates a token by assuming the provided role, using the provided session.
	GetWithRoleForSession(clusterID string, roleARN string, sess *session.Session) (string, error)
	// GetWithSTS assumes returns a token valid for clusterID using the given STS client.
	GetWithSTS(clusterID string, stsAPI *sts.STS) (string, error)
	// FormatJSON returns the client auth formatted json for the ExecCredential auth
	FormatJSON(string) string
}

type generator struct {
}

// NewGenerator creates a Generator and returns it.
func NewGenerator() (Generator, error) {
	return generator{}, nil
}

// Get uses the directly available AWS credentials to return a token valid for
// clusterID. It follows the default AWS credential handling behavior.
func (g generator) Get(clusterID string) (string, error) {
	return g.GetWithRole(clusterID, "")
}

func StdinStderrTokenProvider() (string, error) {
	var v string
	fmt.Fprint(os.Stderr, "Assume Role MFA token code: ")
	_, err := fmt.Scanln(&v)
	return v, err
}

// GetWithRole assumes the given AWS IAM role and returns a token valid for
// clusterID. If roleARN is empty, behaves like Get (does not assume a role).
func (g generator) GetWithRole(clusterID string, roleARN string) (string, error) {
	// create a session with the "base" credentials available
	// (from environment variable, profile files, EC2 metadata, etc)
	sess, err := session.NewSessionWithOptions(session.Options{
		AssumeRoleTokenProvider: StdinStderrTokenProvider,
		SharedConfigState:       session.SharedConfigEnable,
	})
	if err != nil {
		return "", fmt.Errorf("could not create session: %v", err)
	}

	return g.GetWithRoleForSession(clusterID, roleARN, sess)
}

// GetWithRole assumes the given AWS IAM role for the given session and behaves
// like GetWithRole.
func (g generator) GetWithRoleForSession(clusterID string, roleARN string, sess *session.Session) (string, error) {
	// use an STS client based on the direct credentials
	stsAPI := sts.New(sess)

	// if a roleARN was specified, replace the STS client with one that uses
	// temporary credentials from that role.
	if roleARN != "" {
		// Determine if the current session is already a federated identity.  If so,
		// we carry through this session name onto the new session to provide better
		// aduiting capabilities
		//
		// NOTE: I couldn't figure out a more `direct` way of indicating if the
		// current Caller Identity's PrincipalType is `Federated`
		// (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_variables.html#principaltable)
		// If there is a more direct approach I would be interested in understanding
		resp, err := stsAPI.GetCallerIdentity(&sts.GetCallerIdentityInput{})
		if err != nil {
			return "", err
		}

		userIDParts := strings.Split(*resp.UserId, ":")
		sessionSetter := func(provider *stscreds.AssumeRoleProvider) {
			if len(userIDParts) == 2 {
				provider.RoleSessionName = userIDParts[1]
			}
		}

		// create STS-based credentials that will assume the given role
		creds := stscreds.NewCredentials(sess, roleARN, sessionSetter)

		// create an STS API interface that uses the assumed role's temporary credentials
		stsAPI = sts.New(sess, &aws.Config{Credentials: creds})
	}

	return g.GetWithSTS(clusterID, stsAPI)
}

// GetWithSTS assumes returns a token valid for clusterID using the given STS client.
func (g generator) GetWithSTS(clusterID string, stsAPI *sts.STS) (string, error) {
	// generate an sts:GetCallerIdentity request and add our custom cluster ID header
	request, _ := stsAPI.GetCallerIdentityRequest(&sts.GetCallerIdentityInput{})
	request.HTTPRequest.Header.Add(clusterIDHeader, clusterID)

	// sign the request
	presignedURLString, err := request.Presign(60 * time.Second)
	if err != nil {
		return "", err
	}

	// TODO: this may need to be a constant-time base64 encoding
	return v1Prefix + base64.RawURLEncoding.EncodeToString([]byte(presignedURLString)), nil
}

// FormatJSON formats the json to support ExecCredential authentication
func (g generator) FormatJSON(token string) string {
	execInput := &clientauthv1alpha1.ExecCredential{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "client.authentication.k8s.io/v1alpha1",
			Kind:       "ExecCredential",
		},
		Status: &clientauthv1alpha1.ExecCredentialStatus{
			Token: token,
		},
	}
	enc, _ := json.Marshal(execInput)
	return string(enc)
}

// Verifier validates tokens by calling STS and returning the associated identity.
type Verifier interface {
	Verify(token string) (*Identity, error)
}

type tokenVerifier struct {
	client    *http.Client
	clusterID string
}

// NewVerifier creates a Verifier that is bound to the clusterID and uses the default http client.
func NewVerifier(clusterID string) Verifier {
	return tokenVerifier{
		client:    http.DefaultClient,
		clusterID: clusterID,
	}
}

// Verify a token is valid for the specified clusterID. On success, returns an
// Identity that contains information about the AWS principal that created the
// token. On failure, returns nil and a non-nil error.
func (v tokenVerifier) Verify(token string) (*Identity, error) {
	if len(token) > maxTokenLenBytes {
		return nil, FormatError{"token is too large"}
	}

	if !strings.HasPrefix(token, v1Prefix) {
		return nil, FormatError{fmt.Sprintf("token is missing expected %q prefix", v1Prefix)}
	}

	// TODO: this may need to be a constant-time base64 decoding
	tokenBytes, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(token, v1Prefix))
	if err != nil {
		return nil, FormatError{err.Error()}
	}

	parsedURL, err := url.Parse(string(tokenBytes))
	if err != nil {
		return nil, FormatError{err.Error()}
	}

	if parsedURL.Scheme != "https" {
		return nil, FormatError{fmt.Sprintf("unexpected scheme %q in pre-signed URL", parsedURL.Scheme)}
	}

	if parsedURL.Host != "sts.amazonaws.com" {
		return nil, FormatError{"unexpected hostname in pre-signed URL"}
	}

	if parsedURL.Path != "/" {
		return nil, FormatError{"unexpected path in pre-signed URL"}
	}

	queryParamsLower := make(url.Values)
	queryParams := parsedURL.Query()
	for key, values := range queryParams {
		if !parameterWhitelist[strings.ToLower(key)] {
			return nil, FormatError{fmt.Sprintf("non-whitelisted query parameter %q", key)}
		}
		if len(values) != 1 {
			return nil, FormatError{"query parameter with multiple values not supported"}
		}
		queryParamsLower.Set(strings.ToLower(key), values[0])
	}

	if queryParamsLower.Get("action") != "GetCallerIdentity" {
		return nil, FormatError{"unexpected action parameter in pre-signed URL"}
	}

	if !hasSignedClusterIDHeader(&queryParamsLower) {
		return nil, FormatError{fmt.Sprintf("client did not sign the %s header in the pre-signed URL", clusterIDHeader)}
	}

	expires, err := strconv.Atoi(queryParamsLower.Get("x-amz-expires"))
	if err != nil || expires < 0 || expires > 60 {
		return nil, FormatError{"invalid X-Amz-Expires parameter in pre-signed URL"}
	}

	req, err := http.NewRequest("GET", parsedURL.String(), nil)
	req.Header.Set(clusterIDHeader, v.clusterID)
	req.Header.Set("accept", "application/json")

	response, err := v.client.Do(req)
	if err != nil {
		// special case to avoid printing the full URL if possible
		if urlErr, ok := err.(*url.Error); ok {
			return nil, NewSTSError(fmt.Sprintf("error during GET: %v", urlErr.Err))
		}
		return nil, NewSTSError(fmt.Sprintf("error during GET: %v", err))
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, NewSTSError(fmt.Sprintf("error from AWS (expected 200, got %d)", response.StatusCode))
	}

	responseBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, NewSTSError(fmt.Sprintf("error reading HTTP result: %v", err))
	}

	var callerIdentity getCallerIdentityWrapper
	err = json.Unmarshal(responseBody, &callerIdentity)
	if err != nil {
		return nil, NewSTSError(err.Error())
	}

	// parse the response into an Identity
	id := &Identity{
		ARN:       callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
		AccountID: callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
	}
	id.CanonicalARN, err = arn.Canonicalize(id.ARN)
	if err != nil {
		return nil, NewSTSError(err.Error())
	}

	// The user ID is either UserID:SessionName (for assumed roles) or just
	// UserID (for IAM User principals).
	userIDParts := strings.Split(callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.UserID, ":")
	if len(userIDParts) == 2 {
		id.UserID = userIDParts[0]
		id.SessionName = userIDParts[1]
	} else if len(userIDParts) == 1 {
		id.UserID = userIDParts[0]
	} else {
		return nil, STSError{fmt.Sprintf(
			"malformed UserID %q",
			callerIdentity.GetCallerIdentityResponse.GetCallerIdentityResult.UserID)}
	}

	return id, nil
}

func hasSignedClusterIDHeader(paramsLower *url.Values) bool {
	signedHeaders := strings.Split(paramsLower.Get("x-amz-signedheaders"), ";")
	for _, hdr := range signedHeaders {
		if strings.ToLower(hdr) == strings.ToLower(clusterIDHeader) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package prep

import (
	"bytes"
	//"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/craigmonson/colonize/config"
	"github.com/craigmonson/colonize/util"
	//"github.com/craigmonson/colonize/variables"
)

func Run(c *config.ColonizeConfig) error {
	err := BuildCombinedValuesFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedVarsFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedTfFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedDerivedFile(c)
	if err != nil {
		return err
	}

	return nil
}

func BuildCombinedValuesFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableValPaths)
	if err != nil {
		return err
	}
	complete := append(getConfigAsValues(c), combined...)
	return writeCombinedFile(c.CombinedValsFilePath, complete)
}

func BuildCombinedVarsFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableVarPaths)
	if err != nil {
		return err
	}
	complete := append(getConfigAsVariables(c), combined...)
	return writeCombinedFile(c.CombinedVarsFilePath, complete)
}

func BuildCombinedTfFile(c *config.ColonizeConfig) error {
	// get list of files to combine  they can be any tf files
	tfFiles := findTfFilesToCombine(
		c.WalkableTfPaths,
		c.Variable_Tf_File,
		c.Derived_File,
	)

	combined, err := combineFiles(tfFiles)
	if err != nil {
		return err
	}
	return writeCombinedFile(c.CombinedTfFilePath, combined)
}

func BuildCombinedDerivedFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableDerivedPaths)
	if err != nil {
		return err
	}
	//complete := append(getDerivedFromConfig(c), combined...)
	substituted := subDerivedWithVariables(c, combined)
	return writeCombinedFile(c.CombinedDerivedFilePath, substituted)
}

func findTfFilesToCombine(dirPaths []string, vFile, dFile string) []string {
	combineable := []string{}
	for _, path := range dirPaths {
		fList, _ := ioutil.ReadDir(path)
		for _, fPath := range fList {
			fullPath := util.PathJoin(path, fPath.Name())
			if isValidTfFile(fullPath, vFile, dFile) {
				combineable = append(combineable, fullPath)
			}
		}
	}

	return combineable
}

func isValidTfFile(path, vFile, dFile string) bool {
	// skip variable.tf, and files that don't end in '.tf'
	isVarFile, _ := regexp.MatchString("/"+vFile+"$", path)
	isDerivedFile, _ := regexp.MatchString("/"+dFile+"$", path)
	isTfFile, _ := regexp.MatchString("\\.tf$", path)
	return !isVarFile && !isDerivedFile && isTfFile
}

func combineFiles(paths []string) ([]byte, error) {
	combined := []byte{}
	for _, path := range paths {
		contents, err := ioutil.ReadFile(path)
		if err != nil {
			if os.IsPermission(err) {
				return nil, err
			}

			// it's ok if it doesn't exist
			continue
		}
		combined = append(combined, contents...)
	}
	return combined, nil
}

func writeCombinedFile(path string, content []byte) error {
	return ioutil.WriteFile(path, content, 0666)
}

func getConfigAsVariables(c *config.ColonizeConfig) []byte {
	output := ""
	for _, ary := range getDerivedVarList(c) {
		// variable "foo" {}"
		output = output + "variable \"" + ary[0] + "\" {}\n"
	}

	return []byte(output)
}

func getConfigAsValues(c *config.ColonizeConfig) []byte {
	output := ""
	for _, ary := range getDerivedVarList(c) {
		// foo = "bar"
		output = output + ary[0] + " = \"" + ary[1] + "\"\n"
	}

	return []byte(output)
}

// we're returning a slice because we want the lists to stay in order when
// we print it out... with a map, there's no guarantee of order.
func getDerivedVarList(c *config.ColonizeConfig) [][2]string {
	return [][2]string{
		[2]string{"environment", c.Environment},
		[2]string{"origin_path", c.OriginPath},
		[2]string{"tmpl_name", c.TmplName},
		[2]string{"tmpl_path_dashed", strings.Replace(c.TmplName, "/", "-", -1)},
		[2]string{"tmpl_path_underscored", strings.Replace(c.TmplName, "/", "_", -1)},
		[2]string{"root_path", c.RootPath},
	}
}

func subDerivedWithVariables(c *config.ColonizeConfig, derived []byte) []byte {
	for k, v := range getVariableMap(c) {
		derived = bytes.Replace(derived, []byte("${var."+k+"}"), []byte(v), -1)
	}

	return derived
}

func getVariableMap(c *config.ColonizeConfig) map[string]string {
	varMap := map[string]string{}
	content, _ := ioutil.ReadFile(c.CombinedValsFilePath)
	for _, line := range strings.Split(string(content), "\n") {
		// skip if the line doesn't match
		if matched, _ := regexp.MatchString("^.*=.*\".*\"$", line); !matched {
			continue
		}
		KV := strings.Split(line, "=")
		// if, for some reason it split to more or less than 2, skip it
		if len(KV) != 2 {
			continue
		}
		key := strings.TrimSpace(KV[0])
		val := strings.TrimSpace(KV[1])
		// remove first and last double quotes
		val = strings.TrimSuffix(strings.TrimPrefix(val, `"`), `"`)
		varMap[key] = val
	}

	return varMap
}
<commit_msg>Adding better comments<commit_after>package prep

import (
	"bytes"
	//"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/craigmonson/colonize/config"
	"github.com/craigmonson/colonize/util"
	//"github.com/craigmonson/colonize/variables"
)

func Run(c *config.ColonizeConfig) error {
	err := BuildCombinedValuesFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedVarsFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedTfFile(c)
	if err != nil {
		return err
	}

	err = BuildCombinedDerivedFile(c)
	if err != nil {
		return err
	}

	return nil
}

func BuildCombinedValuesFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableValPaths)
	if err != nil {
		return err
	}
	complete := append(getConfigAsValues(c), combined...)
	return writeCombinedFile(c.CombinedValsFilePath, complete)
}

func BuildCombinedVarsFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableVarPaths)
	if err != nil {
		return err
	}
	complete := append(getConfigAsVariables(c), combined...)
	return writeCombinedFile(c.CombinedVarsFilePath, complete)
}

func BuildCombinedTfFile(c *config.ColonizeConfig) error {
	// get list of files to combine  they can be any tf files
	tfFiles := findTfFilesToCombine(
		c.WalkableTfPaths,
		c.Variable_Tf_File,
		c.Derived_File,
	)

	combined, err := combineFiles(tfFiles)
	if err != nil {
		return err
	}
	return writeCombinedFile(c.CombinedTfFilePath, combined)
}

func BuildCombinedDerivedFile(c *config.ColonizeConfig) error {
	combined, err := combineFiles(c.WalkableDerivedPaths)
	if err != nil {
		return err
	}
	//complete := append(getDerivedFromConfig(c), combined...)
	substituted := subDerivedWithVariables(c, combined)
	return writeCombinedFile(c.CombinedDerivedFilePath, substituted)
}

func findTfFilesToCombine(dirPaths []string, vFile, dFile string) []string {
	combineable := []string{}
	for _, path := range dirPaths {
		fList, _ := ioutil.ReadDir(path)
		for _, fPath := range fList {
			fullPath := util.PathJoin(path, fPath.Name())
			if isValidTfFile(fullPath, vFile, dFile) {
				combineable = append(combineable, fullPath)
			}
		}
	}

	return combineable
}

func isValidTfFile(path, vFile, dFile string) bool {
	// skip variable.tf, and files that don't end in '.tf'
	isVarFile, _ := regexp.MatchString("/"+vFile+"$", path)
	isDerivedFile, _ := regexp.MatchString("/"+dFile+"$", path)
	isTfFile, _ := regexp.MatchString("\\.tf$", path)
	return !isVarFile && !isDerivedFile && isTfFile
}

func combineFiles(paths []string) ([]byte, error) {
	combined := []byte{}
	for _, path := range paths {
		contents, err := ioutil.ReadFile(path)
		if err != nil {
			if os.IsPermission(err) {
				return nil, err
			}

			// it's ok if it doesn't exist
			continue
		}
		combined = append(combined, contents...)
	}
	return combined, nil
}

func writeCombinedFile(path string, content []byte) error {
	return ioutil.WriteFile(path, content, 0666)
}

func getConfigAsVariables(c *config.ColonizeConfig) []byte {
	output := ""
	for _, ary := range getDerivedVarList(c) {
		// variable "foo" {}"
		output = output + "variable \"" + ary[0] + "\" {}\n"
	}

	return []byte(output)
}

func getConfigAsValues(c *config.ColonizeConfig) []byte {
	output := ""
	for _, ary := range getDerivedVarList(c) {
		// foo = "bar"
		output = output + ary[0] + " = \"" + ary[1] + "\"\n"
	}

	return []byte(output)
}

// we're returning a slice because we want the lists to stay in order when
// we print it out... with a map, there's no guarantee of order.  There's no
// functional need to do this for terraform et al... just trying to keep it
// nice for the user.
func getDerivedVarList(c *config.ColonizeConfig) [][2]string {
	return [][2]string{
		[2]string{"environment", c.Environment},
		[2]string{"origin_path", c.OriginPath},
		[2]string{"tmpl_name", c.TmplName},
		[2]string{"tmpl_path_dashed", strings.Replace(c.TmplName, "/", "-", -1)},
		[2]string{"tmpl_path_underscored", strings.Replace(c.TmplName, "/", "_", -1)},
		[2]string{"root_path", c.RootPath},
	}
}

func subDerivedWithVariables(c *config.ColonizeConfig, derived []byte) []byte {
	for k, v := range getVariableMap(c) {
		derived = bytes.Replace(derived, []byte("${var."+k+"}"), []byte(v), -1)
	}

	return derived
}

func getVariableMap(c *config.ColonizeConfig) map[string]string {
	varMap := map[string]string{}
	content, _ := ioutil.ReadFile(c.CombinedValsFilePath)
	for _, line := range strings.Split(string(content), "\n") {
		// skip if the line doesn't match blah = "blah"
		if matched, _ := regexp.MatchString("^.*=.*\".*\"$", line); !matched {
			continue
		}
		KV := strings.Split(line, "=")
		// if, for some reason it split to more or less than 2, skip it
		if len(KV) != 2 {
			continue
		}
		// clean it up...
		key := strings.TrimSpace(KV[0])
		val := strings.TrimSpace(KV[1])
		// remove first and last double quotes from the value
		val = strings.TrimSuffix(strings.TrimPrefix(val, `"`), `"`)
		varMap[key] = val
	}

	return varMap
}
<|endoftext|>


<commit_before>package ec2_test

import (
	"launchpad.net/goamz/aws"
	amzec2 "launchpad.net/goamz/ec2"
	"launchpad.net/goamz/ec2/ec2test"
	"launchpad.net/goamz/s3"
	"launchpad.net/goamz/s3/s3test"
	. "launchpad.net/gocheck"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/ec2"
	"launchpad.net/juju-core/environs/jujutest"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"regexp"
	"strings"
)

func registerLocalTests() {
	// N.B. Make sure the region we use here
	// has entries in the images/query txt files.
	aws.Regions["test"] = aws.Region{
		Name: "test",
	}
	attrs := map[string]interface{}{
		"name":                 "sample",
		"type":                 "ec2",
		"region":               "test",
		"control-bucket":       "test-bucket",
		"public-bucket":        "public-tools",
		"public-bucket-region": "test",
		"admin-secret":         "local-secret",
		"access-key":           "x",
		"secret-key":           "x",
		"authorized-keys":      "foo",
		"ca-cert":              testing.CACertPEM,
		"ca-private-key":       testing.CAKeyPEM,
	}

	Suite(&localServerSuite{
		Tests: jujutest.Tests{
			Config: attrs,
		},
	})
	Suite(&localLiveSuite{
		LiveTests: LiveTests{
			LiveTests: jujutest.LiveTests{
				Config: attrs,
			},
		},
	})
}

// localLiveSuite runs tests from LiveTests using a fake
// EC2 server that runs within the test process itself.
type localLiveSuite struct {
	testing.LoggingSuite
	LiveTests
	srv localServer
	env environs.Environ
}

func (t *localLiveSuite) SetUpSuite(c *C) {
	t.LoggingSuite.SetUpSuite(c)
	ec2.UseTestImageData(true)
	t.srv.startServer(c)
	t.LiveTests.SetUpSuite(c)
	t.env = t.LiveTests.Env
	ec2.ShortTimeouts(true)
}

func (t *localLiveSuite) TearDownSuite(c *C) {
	t.LiveTests.TearDownSuite(c)
	t.srv.stopServer(c)
	t.env = nil
	ec2.ShortTimeouts(false)
	ec2.UseTestImageData(false)
	t.LoggingSuite.TearDownSuite(c)
}

func (t *localLiveSuite) SetUpTest(c *C) {
	t.LoggingSuite.SetUpTest(c)
	t.LiveTests.SetUpTest(c)
}

func (t *localLiveSuite) TearDownTest(c *C) {
	t.LiveTests.TearDownTest(c)
	t.LoggingSuite.TearDownTest(c)
}

// localServer represents a fake EC2 server running within
// the test process itself.
type localServer struct {
	ec2srv *ec2test.Server
	s3srv  *s3test.Server
}

func (srv *localServer) startServer(c *C) {
	var err error
	srv.ec2srv, err = ec2test.NewServer()
	if err != nil {
		c.Fatalf("cannot start ec2 test server: %v", err)
	}
	srv.s3srv, err = s3test.NewServer()
	if err != nil {
		c.Fatalf("cannot start s3 test server: %v", err)
	}
	aws.Regions["test"] = aws.Region{
		Name:                 "test",
		EC2Endpoint:          srv.ec2srv.URL(),
		S3Endpoint:           srv.s3srv.URL(),
		S3LocationConstraint: true,
	}
	s3inst := s3.New(aws.Auth{}, aws.Regions["test"])
	putFakeTools(c, ec2.BucketStorage(s3inst.Bucket("public-tools")))
	srv.addSpice(c)
}

// putFakeTools sets up a bucket containing something
// that looks like a tools archive so test methods
// that start an instance can succeed even though they
// do not upload tools.
func putFakeTools(c *C, s environs.StorageWriter) {
	path := environs.ToolsStoragePath(version.Current)
	c.Logf("putting fake tools at %v", path)
	toolsContents := "tools archive, honest guv"
	err := s.Put(path, strings.NewReader(toolsContents), int64(len(toolsContents)))
	c.Assert(err, IsNil)
}

// addSpice adds some "spice" to the local server
// by adding state that may cause tests to fail.
func (srv *localServer) addSpice(c *C) {
	states := []amzec2.InstanceState{
		ec2test.ShuttingDown,
		ec2test.Terminated,
		ec2test.Stopped,
	}
	for _, state := range states {
		srv.ec2srv.NewInstances(1, "m1.small", "ami-a7f539ce", state, nil)
	}
}

func (srv *localServer) stopServer(c *C) {
	srv.ec2srv.Quit()
	srv.s3srv.Quit()
	// Clear out the region because the server address is
	// no longer valid.
	delete(aws.Regions, "test")
}

// localServerSuite contains tests that run against a fake EC2 server
// running within the test process itself.  These tests can test things that
// would be unreasonably slow or expensive to test on a live Amazon server.
// It starts a new local ec2test server for each test.  The server is
// accessed by using the "test" region, which is changed to point to the
// network address of the local server.
type localServerSuite struct {
	testing.LoggingSuite
	jujutest.Tests
	srv localServer
	env environs.Environ
}

func (t *localServerSuite) SetUpSuite(c *C) {
	t.LoggingSuite.SetUpSuite(c)
	ec2.UseTestImageData(true)
	ec2.UseTestMetadata(true)
	t.Tests.SetUpSuite(c)
	ec2.ShortTimeouts(true)
}

func (t *localServerSuite) TearDownSuite(c *C) {
	t.Tests.TearDownSuite(c)
	ec2.ShortTimeouts(false)
	ec2.UseTestMetadata(false)
	ec2.UseTestImageData(false)
	t.LoggingSuite.TearDownSuite(c)
}

func (t *localServerSuite) SetUpTest(c *C) {
	t.LoggingSuite.SetUpTest(c)
	t.srv.startServer(c)
	t.Tests.SetUpTest(c)
	t.env = t.Tests.Env
}

func (t *localServerSuite) TearDownTest(c *C) {
	t.Tests.TearDownTest(c)
	t.srv.stopServer(c)
	t.LoggingSuite.TearDownTest(c)
}

func panicWrite(name string, cert, key []byte) error {
	panic("writeCertAndKey called unexpectedly")
}

func (t *localServerSuite) TestBootstrapInstanceUserDataAndState(c *C) {
	policy := t.env.AssignmentPolicy()
	c.Assert(policy, Equals, state.AssignUnused)

	err := environs.Bootstrap(t.env, true, panicWrite)
	c.Assert(err, IsNil)

	// check that the state holds the id of the bootstrap machine.
	state, err := ec2.LoadState(t.env)
	c.Assert(err, IsNil)
	c.Assert(state.StateInstances, HasLen, 1)

	insts, err := t.env.Instances(state.StateInstances)
	c.Assert(err, IsNil)
	c.Assert(insts, HasLen, 1)
	c.Check(insts[0].Id(), Equals, state.StateInstances[0])

	info, err := t.env.StateInfo()
	c.Assert(err, IsNil)
	c.Assert(info, NotNil)

	// check that the user data is configured to start zookeeper
	// and the machine and provisioning agents.
	inst := t.srv.ec2srv.Instance(string(insts[0].Id()))
	c.Assert(inst, NotNil)
	bootstrapDNS, err := insts[0].DNSName()
	c.Assert(err, IsNil)
	c.Assert(bootstrapDNS, Not(Equals), "")

	c.Logf("first instance: UserData: %q", inst.UserData)
	var x map[interface{}]interface{}
	err = goyaml.Unmarshal(inst.UserData, &x)
	c.Assert(err, IsNil)
	CheckPackage(c, x, "git", true)
	CheckScripts(c, x, "jujud bootstrap-state", true)
	// TODO check for provisioning agent
	// TODO check for machine agent

	// check that a new instance will be started without
	// zookeeper, with a machine agent, and without a
	// provisioning agent.
	info.EntityName = "machine-1"
	inst1, err := t.env.StartInstance("1", info, nil)
	c.Assert(err, IsNil)
	inst = t.srv.ec2srv.Instance(string(inst1.Id()))
	c.Assert(inst, NotNil)
	c.Logf("second instance: UserData: %q", inst.UserData)
	x = nil
	err = goyaml.Unmarshal(inst.UserData, &x)
	c.Assert(err, IsNil)
	CheckPackage(c, x, "zookeeperd", false)
	// TODO check for provisioning agent
	// TODO check for machine agent

	p := t.env.Provider()
	addr, err := p.PublicAddress()
	c.Assert(err, IsNil)
	c.Assert(addr, Equals, "public.dummy.address.example.com")
	addr, err = p.PrivateAddress()
	c.Assert(err, IsNil)
	c.Assert(addr, Equals, "private.dummy.address.example.com")

	err = t.env.Destroy(append(insts, inst1))
	c.Assert(err, IsNil)

	_, err = ec2.LoadState(t.env)
	c.Assert(err, NotNil)
}

// If match is true, CheckScripts checks that at least one script started
// by the cloudinit data matches the given regexp pattern, otherwise it
// checks that no script matches.  It's exported so it can be used by tests
// defined in ec2_test.
func CheckScripts(c *C, x map[interface{}]interface{}, pattern string, match bool) {
	scripts0 := x["runcmd"]
	if scripts0 == nil {
		c.Errorf("cloudinit has no entry for runcmd")
		return
	}
	scripts := scripts0.([]interface{})
	re := regexp.MustCompile(pattern)
	found := false
	for _, s0 := range scripts {
		s := s0.(string)
		if re.MatchString(s) {
			found = true
		}
	}
	switch {
	case match && !found:
		c.Errorf("script %q not found in %q", pattern, scripts)
	case !match && found:
		c.Errorf("script %q found but not expected in %q", pattern, scripts)
	}
}

// CheckPackage checks that the cloudinit will or won't install the given
// package, depending on the value of match.  It's exported so it can be
// used by tests defined outside the ec2 package.
func CheckPackage(c *C, x map[interface{}]interface{}, pkg string, match bool) {
	pkgs0 := x["packages"]
	if pkgs0 == nil {
		if match {
			c.Errorf("cloudinit has no entry for packages")
		}
		return
	}

	pkgs := pkgs0.([]interface{})

	found := false
	for _, p0 := range pkgs {
		p := p0.(string)
		if p == pkg {
			found = true
		}
	}
	switch {
	case match && !found:
		c.Errorf("package %q not found in %v", pkg, pkgs)
	case !match && found:
		c.Errorf("%q found but not expected in %v", pkg, pkgs)
	}
}
<commit_msg>fix merge conflict<commit_after>package ec2_test

import (
	"launchpad.net/goamz/aws"
	amzec2 "launchpad.net/goamz/ec2"
	"launchpad.net/goamz/ec2/ec2test"
	"launchpad.net/goamz/s3"
	"launchpad.net/goamz/s3/s3test"
	. "launchpad.net/gocheck"
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/ec2"
	"launchpad.net/juju-core/environs/jujutest"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"regexp"
	"strings"
)

func registerLocalTests() {
	// N.B. Make sure the region we use here
	// has entries in the images/query txt files.
	aws.Regions["test"] = aws.Region{
		Name: "test",
	}
	attrs := map[string]interface{}{
		"name":                 "sample",
		"type":                 "ec2",
		"region":               "test",
		"control-bucket":       "test-bucket",
		"public-bucket":        "public-tools",
		"public-bucket-region": "test",
		"admin-secret":         "local-secret",
		"access-key":           "x",
		"secret-key":           "x",
		"authorized-keys":      "foo",
		"ca-cert":              testing.CACert,
		"ca-private-key":       testing.CAKey,
	}

	Suite(&localServerSuite{
		Tests: jujutest.Tests{
			Config: attrs,
		},
	})
	Suite(&localLiveSuite{
		LiveTests: LiveTests{
			LiveTests: jujutest.LiveTests{
				Config: attrs,
			},
		},
	})
}

// localLiveSuite runs tests from LiveTests using a fake
// EC2 server that runs within the test process itself.
type localLiveSuite struct {
	testing.LoggingSuite
	LiveTests
	srv localServer
	env environs.Environ
}

func (t *localLiveSuite) SetUpSuite(c *C) {
	t.LoggingSuite.SetUpSuite(c)
	ec2.UseTestImageData(true)
	t.srv.startServer(c)
	t.LiveTests.SetUpSuite(c)
	t.env = t.LiveTests.Env
	ec2.ShortTimeouts(true)
}

func (t *localLiveSuite) TearDownSuite(c *C) {
	t.LiveTests.TearDownSuite(c)
	t.srv.stopServer(c)
	t.env = nil
	ec2.ShortTimeouts(false)
	ec2.UseTestImageData(false)
	t.LoggingSuite.TearDownSuite(c)
}

func (t *localLiveSuite) SetUpTest(c *C) {
	t.LoggingSuite.SetUpTest(c)
	t.LiveTests.SetUpTest(c)
}

func (t *localLiveSuite) TearDownTest(c *C) {
	t.LiveTests.TearDownTest(c)
	t.LoggingSuite.TearDownTest(c)
}

// localServer represents a fake EC2 server running within
// the test process itself.
type localServer struct {
	ec2srv *ec2test.Server
	s3srv  *s3test.Server
}

func (srv *localServer) startServer(c *C) {
	var err error
	srv.ec2srv, err = ec2test.NewServer()
	if err != nil {
		c.Fatalf("cannot start ec2 test server: %v", err)
	}
	srv.s3srv, err = s3test.NewServer()
	if err != nil {
		c.Fatalf("cannot start s3 test server: %v", err)
	}
	aws.Regions["test"] = aws.Region{
		Name:                 "test",
		EC2Endpoint:          srv.ec2srv.URL(),
		S3Endpoint:           srv.s3srv.URL(),
		S3LocationConstraint: true,
	}
	s3inst := s3.New(aws.Auth{}, aws.Regions["test"])
	putFakeTools(c, ec2.BucketStorage(s3inst.Bucket("public-tools")))
	srv.addSpice(c)
}

// putFakeTools sets up a bucket containing something
// that looks like a tools archive so test methods
// that start an instance can succeed even though they
// do not upload tools.
func putFakeTools(c *C, s environs.StorageWriter) {
	path := environs.ToolsStoragePath(version.Current)
	c.Logf("putting fake tools at %v", path)
	toolsContents := "tools archive, honest guv"
	err := s.Put(path, strings.NewReader(toolsContents), int64(len(toolsContents)))
	c.Assert(err, IsNil)
}

// addSpice adds some "spice" to the local server
// by adding state that may cause tests to fail.
func (srv *localServer) addSpice(c *C) {
	states := []amzec2.InstanceState{
		ec2test.ShuttingDown,
		ec2test.Terminated,
		ec2test.Stopped,
	}
	for _, state := range states {
		srv.ec2srv.NewInstances(1, "m1.small", "ami-a7f539ce", state, nil)
	}
}

func (srv *localServer) stopServer(c *C) {
	srv.ec2srv.Quit()
	srv.s3srv.Quit()
	// Clear out the region because the server address is
	// no longer valid.
	delete(aws.Regions, "test")
}

// localServerSuite contains tests that run against a fake EC2 server
// running within the test process itself.  These tests can test things that
// would be unreasonably slow or expensive to test on a live Amazon server.
// It starts a new local ec2test server for each test.  The server is
// accessed by using the "test" region, which is changed to point to the
// network address of the local server.
type localServerSuite struct {
	testing.LoggingSuite
	jujutest.Tests
	srv localServer
	env environs.Environ
}

func (t *localServerSuite) SetUpSuite(c *C) {
	t.LoggingSuite.SetUpSuite(c)
	ec2.UseTestImageData(true)
	ec2.UseTestMetadata(true)
	t.Tests.SetUpSuite(c)
	ec2.ShortTimeouts(true)
}

func (t *localServerSuite) TearDownSuite(c *C) {
	t.Tests.TearDownSuite(c)
	ec2.ShortTimeouts(false)
	ec2.UseTestMetadata(false)
	ec2.UseTestImageData(false)
	t.LoggingSuite.TearDownSuite(c)
}

func (t *localServerSuite) SetUpTest(c *C) {
	t.LoggingSuite.SetUpTest(c)
	t.srv.startServer(c)
	t.Tests.SetUpTest(c)
	t.env = t.Tests.Env
}

func (t *localServerSuite) TearDownTest(c *C) {
	t.Tests.TearDownTest(c)
	t.srv.stopServer(c)
	t.LoggingSuite.TearDownTest(c)
}

func panicWrite(name string, cert, key []byte) error {
	panic("writeCertAndKey called unexpectedly")
}

func (t *localServerSuite) TestBootstrapInstanceUserDataAndState(c *C) {
	policy := t.env.AssignmentPolicy()
	c.Assert(policy, Equals, state.AssignUnused)

	err := environs.Bootstrap(t.env, true, panicWrite)
	c.Assert(err, IsNil)

	// check that the state holds the id of the bootstrap machine.
	state, err := ec2.LoadState(t.env)
	c.Assert(err, IsNil)
	c.Assert(state.StateInstances, HasLen, 1)

	insts, err := t.env.Instances(state.StateInstances)
	c.Assert(err, IsNil)
	c.Assert(insts, HasLen, 1)
	c.Check(insts[0].Id(), Equals, state.StateInstances[0])

	info, err := t.env.StateInfo()
	c.Assert(err, IsNil)
	c.Assert(info, NotNil)

	// check that the user data is configured to start zookeeper
	// and the machine and provisioning agents.
	inst := t.srv.ec2srv.Instance(string(insts[0].Id()))
	c.Assert(inst, NotNil)
	bootstrapDNS, err := insts[0].DNSName()
	c.Assert(err, IsNil)
	c.Assert(bootstrapDNS, Not(Equals), "")

	c.Logf("first instance: UserData: %q", inst.UserData)
	var x map[interface{}]interface{}
	err = goyaml.Unmarshal(inst.UserData, &x)
	c.Assert(err, IsNil)
	CheckPackage(c, x, "git", true)
	CheckScripts(c, x, "jujud bootstrap-state", true)
	// TODO check for provisioning agent
	// TODO check for machine agent

	// check that a new instance will be started without
	// zookeeper, with a machine agent, and without a
	// provisioning agent.
	info.EntityName = "machine-1"
	inst1, err := t.env.StartInstance("1", info, nil)
	c.Assert(err, IsNil)
	inst = t.srv.ec2srv.Instance(string(inst1.Id()))
	c.Assert(inst, NotNil)
	c.Logf("second instance: UserData: %q", inst.UserData)
	x = nil
	err = goyaml.Unmarshal(inst.UserData, &x)
	c.Assert(err, IsNil)
	CheckPackage(c, x, "zookeeperd", false)
	// TODO check for provisioning agent
	// TODO check for machine agent

	p := t.env.Provider()
	addr, err := p.PublicAddress()
	c.Assert(err, IsNil)
	c.Assert(addr, Equals, "public.dummy.address.example.com")
	addr, err = p.PrivateAddress()
	c.Assert(err, IsNil)
	c.Assert(addr, Equals, "private.dummy.address.example.com")

	err = t.env.Destroy(append(insts, inst1))
	c.Assert(err, IsNil)

	_, err = ec2.LoadState(t.env)
	c.Assert(err, NotNil)
}

// If match is true, CheckScripts checks that at least one script started
// by the cloudinit data matches the given regexp pattern, otherwise it
// checks that no script matches.  It's exported so it can be used by tests
// defined in ec2_test.
func CheckScripts(c *C, x map[interface{}]interface{}, pattern string, match bool) {
	scripts0 := x["runcmd"]
	if scripts0 == nil {
		c.Errorf("cloudinit has no entry for runcmd")
		return
	}
	scripts := scripts0.([]interface{})
	re := regexp.MustCompile(pattern)
	found := false
	for _, s0 := range scripts {
		s := s0.(string)
		if re.MatchString(s) {
			found = true
		}
	}
	switch {
	case match && !found:
		c.Errorf("script %q not found in %q", pattern, scripts)
	case !match && found:
		c.Errorf("script %q found but not expected in %q", pattern, scripts)
	}
}

// CheckPackage checks that the cloudinit will or won't install the given
// package, depending on the value of match.  It's exported so it can be
// used by tests defined outside the ec2 package.
func CheckPackage(c *C, x map[interface{}]interface{}, pkg string, match bool) {
	pkgs0 := x["packages"]
	if pkgs0 == nil {
		if match {
			c.Errorf("cloudinit has no entry for packages")
		}
		return
	}

	pkgs := pkgs0.([]interface{})

	found := false
	for _, p0 := range pkgs {
		p := p0.(string)
		if p == pkg {
			found = true
		}
	}
	switch {
	case match && !found:
		c.Errorf("package %q not found in %v", pkg, pkgs)
	case !match && found:
		c.Errorf("%q found but not expected in %v", pkg, pkgs)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path"
	"regexp"
	"strings"
	"template"
)

const defaultAddr = ":31798" // default webserver address

var h1TitlePattern = regexp.MustCompile(`<h1>(.+)</h1>`)

var (
	httpAddr            = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')")
	root                = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl")
	gitwebScript        = flag.String("gitwebscript", "/usr/lib/cgi-bin/gitweb.cgi", "Path to gitweb.cgi, or blank to disable.")
	gitwebFiles         = flag.String("gitwebfiles", "/usr/share/gitweb", "Path to gitweb's static files.")
	logDir              = flag.String("logdir", "-", "Directory to write log files to (one per hour), or empty to not log.")
	logStdout           = flag.Bool("logstdout", true, "Write to stdout?")
	pageHtml, errorHtml *template.Template
)

var fmap = template.FormatterMap{
	"":         textFmt,
	"html":     htmlFmt,
	"html-esc": htmlEscFmt,
}

// Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, false, x[0])
}

// Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, true, x[0])
}

// Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
	var buf bytes.Buffer
	writeAny(&buf, false, x[0])
	template.HTMLEscape(w, buf.Bytes())
}

// Write anything to w; optionally html-escaped.
func writeAny(w io.Writer, html bool, x interface{}) {
	switch v := x.(type) {
	case []byte:
		writeText(w, v, html)
	case string:
		writeText(w, []byte(v), html)
	default:
		if html {
			var buf bytes.Buffer
			fmt.Fprint(&buf, x)
			writeText(w, buf.Bytes(), true)
		} else {
			fmt.Fprint(w, x)
		}
	}
}

// Write text to w; optionally html-escaped.
func writeText(w io.Writer, text []byte, html bool) {
	if html {
		template.HTMLEscape(w, text)
		return
	}
	w.Write(text)
}

func applyTemplate(t *template.Template, name string, data interface{}) []byte {
	var buf bytes.Buffer
	if err := t.Execute(data, &buf); err != nil {
		log.Printf("%s.Execute: %s", name, err)
	}
	return buf.Bytes()
}

func servePage(w http.ResponseWriter, title, subtitle string, content []byte) {
	d := struct {
		Title    string
		Subtitle string
		Content  []byte
	}{
		title,
		subtitle,
		content,
	}

	if err := pageHtml.Execute(&d, w); err != nil {
		log.Printf("godocHTML.Execute: %s", err)
	}
}

func readTemplate(name string) *template.Template {
	fileName := path.Join(*root, "tmpl", name)
	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("ReadFile %s: %v", fileName, err)
	}
	t, err := template.Parse(string(data), fmap)
	if err != nil {
		log.Fatalf("%s: %v", fileName, err)
	}
	return t
}

func readTemplates() {
	pageHtml = readTemplate("page.html")
	errorHtml = readTemplate("error.html")
}

func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path!
	w.WriteHeader(http.StatusNotFound)
	servePage(w, "File "+relpath, "", contents)
}

func mainHandler(rw http.ResponseWriter, req *http.Request) {
	relPath := req.URL.Path[1:] // serveFile URL paths start with '/'
	if strings.Contains(relPath, "..") {
		return
	}

	if strings.HasPrefix(relPath, "gw/") {
		path := relPath[3:]
		http.Redirect(rw, req, "/code/?p=camlistore.git;f=" + path + ";hb=master", http.StatusFound)
		return
	}

	absPath := path.Join(*root, "content", relPath)
	fi, err := os.Lstat(absPath)
	if err != nil {
		log.Print(err)
		serveError(rw, req, relPath, err)
		return
	}
	if fi.IsDirectory() {
		relPath += "/index.html"
		absPath = path.Join(*root, "content", relPath)
		fi, err = os.Lstat(absPath)
		if err != nil {
			log.Print(err)
			serveError(rw, req, relPath, err)
			return
		}
	}

	switch {
	case fi.IsRegular():
		serveFile(rw, req, relPath, absPath)
	}
}

func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) {
	data, err := ioutil.ReadFile(absPath)
	if err != nil {
		serveError(rw, req, absPath, err)
		return
	}

	title := ""
	if m := h1TitlePattern.FindSubmatch(data); len(m) > 1 {
		title = string(m[1])
	}

	servePage(rw, title, "", data)
}

type gitwebHandler struct {
	Cgi    http.Handler
	Static http.Handler
}

func (h *gitwebHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if r.URL.RawPath == "/code/" ||
		strings.HasPrefix(r.URL.RawPath, "/code/?") {
		h.Cgi.ServeHTTP(rw, r)
	} else {
		h.Static.ServeHTTP(rw, r)
	}
}

type noWwwHandler struct {
	Handler http.Handler
}

func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	host := strings.ToLower(r.Host)
	if host == "www.camlistore.org" {
		http.Redirect(rw, r, "http://camlistore.org" + r.URL.RawPath, http.StatusFound)
		return
	}
	h.Handler.ServeHTTP(rw, r)
}

func main() {
	flag.Parse()
	readTemplates()

	if *root == "" {
		var err os.Error
		*root, err = os.Getwd()
		if err != nil {
			log.Fatalf("Failed to getwd: %v", err)
		}
	}

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/robots.txt", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/static/", http.FileServer(path.Join(*root, "static"), "/static/"))

	testCgi := &CgiHandler{ExecutablePath: path.Join(*root, "test.cgi"),
		Root: "/test.cgi",
	}
	mux.Handle("/test.cgi", testCgi)
	mux.Handle("/test.cgi/foo", testCgi)

	mux.Handle("/code", http.RedirectHandler("/code/", http.StatusFound))
	if *gitwebScript != "" {
		env := os.Environ()
		env = append(env, "GITWEB_CONFIG="+path.Join(*root, "gitweb-camli.conf"))
		env = append(env, "CAMWEB_ROOT="+path.Join(*root))
		mux.Handle("/code/", &gitwebHandler{
			Cgi: &CgiHandler{
				ExecutablePath: *gitwebScript,
				Root:           "/code/",
				Environ:        env,
			},
			Static: http.FileServer(*gitwebFiles, "/code/"),
		})
	}
	mux.HandleFunc("/", mainHandler)

	var handler http.Handler = &noWwwHandler{Handler: mux}
	if *logDir != "" || *logStdout {
		handler = NewLoggingHandler(handler, *logDir, *logStdout)
	}

	var listener net.Listener
	var err os.Error
	listener, err = net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("Listen error on %s: %v", *httpAddr, err)
	}

	listener = &withTimeoutListener{listener, connTimeoutNanos}
	defer listener.Close()

	if err = http.Serve(listener, handler); err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}

const connTimeoutNanos = 15e9

type withTimeoutListener struct {
	l            net.Listener
	timeoutNanos int64
}

func (wtl *withTimeoutListener) Accept() (c net.Conn, err os.Error) {
	c, err = wtl.l.Accept()
	if err == nil {
		c.SetTimeout(wtl.timeoutNanos)
	}
	return
}

func (wtl *withTimeoutListener) Close() os.Error {
	return wtl.l.Close()
}

func (wtl *withTimeoutListener) Addr() net.Addr {
	return wtl.l.Addr()
}
<commit_msg>camweb: rewrite broken URLs. source still a mystery.<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path"
	"regexp"
	"strings"
	"template"
)

const defaultAddr = ":31798" // default webserver address

var h1TitlePattern = regexp.MustCompile(`<h1>(.+)</h1>`)

var (
	httpAddr            = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')")
	root                = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl")
	gitwebScript        = flag.String("gitwebscript", "/usr/lib/cgi-bin/gitweb.cgi", "Path to gitweb.cgi, or blank to disable.")
	gitwebFiles         = flag.String("gitwebfiles", "/usr/share/gitweb", "Path to gitweb's static files.")
	logDir              = flag.String("logdir", "-", "Directory to write log files to (one per hour), or empty to not log.")
	logStdout           = flag.Bool("logstdout", true, "Write to stdout?")
	pageHtml, errorHtml *template.Template
)

var fmap = template.FormatterMap{
	"":         textFmt,
	"html":     htmlFmt,
	"html-esc": htmlEscFmt,
}

// Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, false, x[0])
}

// Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, true, x[0])
}

// Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
	var buf bytes.Buffer
	writeAny(&buf, false, x[0])
	template.HTMLEscape(w, buf.Bytes())
}

// Write anything to w; optionally html-escaped.
func writeAny(w io.Writer, html bool, x interface{}) {
	switch v := x.(type) {
	case []byte:
		writeText(w, v, html)
	case string:
		writeText(w, []byte(v), html)
	default:
		if html {
			var buf bytes.Buffer
			fmt.Fprint(&buf, x)
			writeText(w, buf.Bytes(), true)
		} else {
			fmt.Fprint(w, x)
		}
	}
}

// Write text to w; optionally html-escaped.
func writeText(w io.Writer, text []byte, html bool) {
	if html {
		template.HTMLEscape(w, text)
		return
	}
	w.Write(text)
}

func applyTemplate(t *template.Template, name string, data interface{}) []byte {
	var buf bytes.Buffer
	if err := t.Execute(data, &buf); err != nil {
		log.Printf("%s.Execute: %s", name, err)
	}
	return buf.Bytes()
}

func servePage(w http.ResponseWriter, title, subtitle string, content []byte) {
	d := struct {
		Title    string
		Subtitle string
		Content  []byte
	}{
		title,
		subtitle,
		content,
	}

	if err := pageHtml.Execute(&d, w); err != nil {
		log.Printf("godocHTML.Execute: %s", err)
	}
}

func readTemplate(name string) *template.Template {
	fileName := path.Join(*root, "tmpl", name)
	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("ReadFile %s: %v", fileName, err)
	}
	t, err := template.Parse(string(data), fmap)
	if err != nil {
		log.Fatalf("%s: %v", fileName, err)
	}
	return t
}

func readTemplates() {
	pageHtml = readTemplate("page.html")
	errorHtml = readTemplate("error.html")
}

func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path!
	w.WriteHeader(http.StatusNotFound)
	servePage(w, "File "+relpath, "", contents)
}

func mainHandler(rw http.ResponseWriter, req *http.Request) {
	relPath := req.URL.Path[1:] // serveFile URL paths start with '/'
	if strings.Contains(relPath, "..") {
		return
	}

	if strings.HasPrefix(relPath, "gw/") {
		path := relPath[3:]
		http.Redirect(rw, req, "/code/?p=camlistore.git;f=" + path + ";hb=master", http.StatusFound)
		return
	}

	absPath := path.Join(*root, "content", relPath)
	fi, err := os.Lstat(absPath)
	if err != nil {
		log.Print(err)
		serveError(rw, req, relPath, err)
		return
	}
	if fi.IsDirectory() {
		relPath += "/index.html"
		absPath = path.Join(*root, "content", relPath)
		fi, err = os.Lstat(absPath)
		if err != nil {
			log.Print(err)
			serveError(rw, req, relPath, err)
			return
		}
	}

	switch {
	case fi.IsRegular():
		serveFile(rw, req, relPath, absPath)
	}
}

func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) {
	data, err := ioutil.ReadFile(absPath)
	if err != nil {
		serveError(rw, req, absPath, err)
		return
	}

	title := ""
	if m := h1TitlePattern.FindSubmatch(data); len(m) > 1 {
		title = string(m[1])
	}

	servePage(rw, title, "", data)
}

type gitwebHandler struct {
	Cgi    http.Handler
	Static http.Handler
}

func (h *gitwebHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if r.URL.RawPath == "/code/" ||
		strings.HasPrefix(r.URL.RawPath, "/code/?") {
		h.Cgi.ServeHTTP(rw, r)
	} else {
		h.Static.ServeHTTP(rw, r)
	}
}

type noWwwHandler struct {
	Handler http.Handler
}

func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	host := strings.ToLower(r.Host)
	if host == "www.camlistore.org" {
		http.Redirect(rw, r, "http://camlistore.org" + r.URL.RawPath, http.StatusFound)
		return
	}
	h.Handler.ServeHTTP(rw, r)
}

func main() {
	flag.Parse()
	readTemplates()

	if *root == "" {
		var err os.Error
		*root, err = os.Getwd()
		if err != nil {
			log.Fatalf("Failed to getwd: %v", err)
		}
	}

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/robots.txt", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/static/", http.FileServer(path.Join(*root, "static"), "/static/"))

	testCgi := &CgiHandler{ExecutablePath: path.Join(*root, "test.cgi"),
		Root: "/test.cgi",
	}
	mux.Handle("/test.cgi", testCgi)
	mux.Handle("/test.cgi/foo", testCgi)

	mux.Handle("/code", http.RedirectHandler("/code/", http.StatusFound))
	if *gitwebScript != "" {
		env := os.Environ()
		env = append(env, "GITWEB_CONFIG="+path.Join(*root, "gitweb-camli.conf"))
		env = append(env, "CAMWEB_ROOT="+path.Join(*root))
		mux.Handle("/code/", &fixUpGitwebUrls{&gitwebHandler{
			Cgi: &CgiHandler{
				ExecutablePath: *gitwebScript,
				Root:           "/code/",
				Environ:        env,
			},
			Static: http.FileServer(*gitwebFiles, "/code/"),
		}})
	}
	mux.HandleFunc("/", mainHandler)

	var handler http.Handler = &noWwwHandler{Handler: mux}
	if *logDir != "" || *logStdout {
		handler = NewLoggingHandler(handler, *logDir, *logStdout)
	}

	var listener net.Listener
	var err os.Error
	listener, err = net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("Listen error on %s: %v", *httpAddr, err)
	}

	listener = &withTimeoutListener{listener, connTimeoutNanos}
	defer listener.Close()

	if err = http.Serve(listener, handler); err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}

const connTimeoutNanos = 15e9

type withTimeoutListener struct {
	l            net.Listener
	timeoutNanos int64
}

func (wtl *withTimeoutListener) Accept() (c net.Conn, err os.Error) {
	c, err = wtl.l.Accept()
	if err == nil {
		c.SetTimeout(wtl.timeoutNanos)
	}
	return
}

func (wtl *withTimeoutListener) Close() os.Error {
	return wtl.l.Close()
}

func (wtl *withTimeoutListener) Addr() net.Addr {
	return wtl.l.Addr()
}

type fixUpGitwebUrls struct {
	handler http.Handler
}

// Not sure what's making these broken URLs like:
//
//   http://localhost:8080/code/?p=camlistore.git%3Bf=doc/json-signing/json-signing.txt%3Bhb=master
//
// ... but something is.  Maybe Buzz?  For now just re-write them
// . Doesn't seem to be a bug in the CGI implementation, though, which
// is what I'd originally suspected.
func (fu *fixUpGitwebUrls) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	oldUrl := req.RawURL
	newUrl := strings.Replace(oldUrl, "%3B", ";", -1)
	if newUrl == oldUrl {
		fu.handler.ServeHTTP(rw, req)
		return
	}
	http.Redirect(rw, req, newUrl, http.StatusFound)
}
<|endoftext|>


<commit_before>//go:generate bitfanDoc
// The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.
// For example, syslog events usually have timestamps like this:
// `"Apr 17 09:32:01"`
// You would use the date format MMM dd HH:mm:ss to parse this.
// The date filter is especially important for sorting events and for backfilling old data. If you don’t get the date correct in your event, then searching for them later will likely sort out of order.
// In the absence of this filter, logstash will choose a timestamp based on the first time it sees the event (at input time), if the timestamp is not already set in the event. For example, with file input, the timestamp is set to the time of each read.
package date

import (
	"strconv"
	"time"

	"github.com/vjeantet/bitfan/processors"
	"github.com/vjeantet/jodaTime"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

// Parses dates from fields to use as the BitFan timestamp for an event
type processor struct {
	processors.Base

	matchFieldName string
	matchPatterns  []string
	opt            *options
}

type options struct {
	// If this filter is successful, add any arbitrary fields to this event.
	AddField map[string]interface{} `mapstructure:"add_field"`

	// The date formats allowed are anything allowed by Joda time format.
	// You can see the docs for this format http://www.joda.org/joda-time/key_format.html
	// An array with field name first, and format patterns following, [ field, formats... ]
	Match []string `mapstructure:"match"`

	// If this filter is successful, add arbitrary tags to the event. Tags can be dynamic
	// and include parts of the event using the %{field} syntax.
	AddTag []string `mapstructure:"add_tag"`

	// If this filter is successful, remove arbitrary fields from this event.
	RemoveField []string `mapstructure:"remove_field"`

	RemoveTag []string `mapstructure:"remove_tag"`

	// Append values to the tags field when there has been no successful match
	// Default value is ["_dateparsefailure"]
	TagOnFailure []string `mapstructure:"tag_on_failure"`

	// Store the matching timestamp into the given target field. If not provided,
	// default to updating the @timestamp field of the event
	Target string `mapstructure:"target"`

	// Specify a time zone canonical ID to be used for date parsing.
	// The valid IDs are listed on IANA Time Zone database, such as "America/New_York".
	// This is useful in case the time zone cannot be extracted from the value,
	// and is not the platform default. If this is not specified the platform default
	//  will be used. Canonical ID is good as it takes care of daylight saving time
	// for you For example, America/Los_Angeles or Europe/Paris are valid IDs.
	// This field can be dynamic and include parts of the event using the %{field} syntax
	Timezone string `mapstructure:"timezone"`
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	p.opt.Target = "@timestamp"
	p.opt.TagOnFailure = []string{"_dateparsefailure"}

	if err := p.ConfigureAndValidate(ctx, conf, p.opt); err != nil {
		return err
	}

	p.matchFieldName = p.opt.Match[0]
	p.matchPatterns = p.opt.Match[1:]

	return nil
}

func (p *processor) Receive(e processors.IPacket) error {
	dated := false
	var value string
	var err error
	value, err = e.Fields().ValueForPathString(p.matchFieldName)
	if err == nil {
		for _, layout := range p.matchPatterns {
			var t time.Time

			if p.opt.Timezone != "" {
				t, err = jodaTime.ParseInLocation(layout, value, p.opt.Timezone)
				if err != nil {
					return err
				}
			} else {
				if layout == "UNIX" {
					var i int64
					i, err = strconv.ParseInt(value, 10, 64)
					if err == nil {
						t = time.Unix(i, 0)
					}
				} else {
					t, err = jodaTime.Parse(layout, value)
				}

			}

			if err != nil {
				continue
			}

			dated = true
			e.Fields().SetValueForPath(t, p.opt.Target)
			processors.ProcessCommonFields2(e.Fields(),
				p.opt.AddField,
				p.opt.AddTag,
				p.opt.RemoveField,
				p.opt.RemoveTag,
			)
			break
		}
	}

	if dated == false {
		processors.AddTags(p.opt.TagOnFailure, e.Fields())
	}

	p.Send(e, 0)
	return nil
}
<commit_msg>processor date filter : fix error handling while Parsing with given timezone<commit_after>//go:generate bitfanDoc
// The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.
// For example, syslog events usually have timestamps like this:
// `"Apr 17 09:32:01"`
// You would use the date format MMM dd HH:mm:ss to parse this.
// The date filter is especially important for sorting events and for backfilling old data. If you don’t get the date correct in your event, then searching for them later will likely sort out of order.
// In the absence of this filter, logstash will choose a timestamp based on the first time it sees the event (at input time), if the timestamp is not already set in the event. For example, with file input, the timestamp is set to the time of each read.
package date

import (
	"strconv"
	"time"

	"github.com/vjeantet/bitfan/processors"
	"github.com/vjeantet/jodaTime"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

// Parses dates from fields to use as the BitFan timestamp for an event
type processor struct {
	processors.Base

	matchFieldName string
	matchPatterns  []string
	opt            *options
}

type options struct {
	// If this filter is successful, add any arbitrary fields to this event.
	AddField map[string]interface{} `mapstructure:"add_field"`

	// The date formats allowed are anything allowed by Joda time format.
	// You can see the docs for this format http://www.joda.org/joda-time/key_format.html
	// An array with field name first, and format patterns following, [ field, formats... ]
	Match []string `mapstructure:"match"`

	// If this filter is successful, add arbitrary tags to the event. Tags can be dynamic
	// and include parts of the event using the %{field} syntax.
	AddTag []string `mapstructure:"add_tag"`

	// If this filter is successful, remove arbitrary fields from this event.
	RemoveField []string `mapstructure:"remove_field"`

	RemoveTag []string `mapstructure:"remove_tag"`

	// Append values to the tags field when there has been no successful match
	// Default value is ["_dateparsefailure"]
	TagOnFailure []string `mapstructure:"tag_on_failure"`

	// Store the matching timestamp into the given target field. If not provided,
	// default to updating the @timestamp field of the event
	Target string `mapstructure:"target"`

	// Specify a time zone canonical ID to be used for date parsing.
	// The valid IDs are listed on IANA Time Zone database, such as "America/New_York".
	// This is useful in case the time zone cannot be extracted from the value,
	// and is not the platform default. If this is not specified the platform default
	//  will be used. Canonical ID is good as it takes care of daylight saving time
	// for you For example, America/Los_Angeles or Europe/Paris are valid IDs.
	// This field can be dynamic and include parts of the event using the %{field} syntax
	Timezone string `mapstructure:"timezone"`
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	p.opt.Target = "@timestamp"
	p.opt.TagOnFailure = []string{"_dateparsefailure"}

	if err := p.ConfigureAndValidate(ctx, conf, p.opt); err != nil {
		return err
	}

	p.matchFieldName = p.opt.Match[0]
	p.matchPatterns = p.opt.Match[1:]

	return nil
}

func (p *processor) Receive(e processors.IPacket) error {
	dated := false
	var value string
	var err error
	value, err = e.Fields().ValueForPathString(p.matchFieldName)
	if err == nil {
		for _, layout := range p.matchPatterns {
			var t time.Time

			if p.opt.Timezone != "" {
				t, err = jodaTime.ParseInLocation(layout, value, p.opt.Timezone)
			} else {
				if layout == "UNIX" {
					var i int64
					i, err = strconv.ParseInt(value, 10, 64)
					if err == nil {
						t = time.Unix(i, 0)
					}
				} else {
					t, err = jodaTime.Parse(layout, value)
				}

			}

			if err != nil {
				continue
			}

			dated = true
			e.Fields().SetValueForPath(t, p.opt.Target)
			processors.ProcessCommonFields2(e.Fields(),
				p.opt.AddField,
				p.opt.AddTag,
				p.opt.RemoveField,
				p.opt.RemoveTag,
			)
			break
		}
	}

	if dated == false {
		processors.AddTags(p.opt.TagOnFailure, e.Fields())
	}

	p.Send(e, 0)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/big"
	"io/ioutil"
	"log"
	"strings"
)

func main() {
	sum := big.NewInt(0)
	file, err := ioutil.ReadFile("names.txt")
	if err != nil {
		log.Fatal(err)
	}
	names := strings.Split(string(file[1:len(file)-1]),"\",\"")
	//fmt.Println(names, p)
	for index, name := range names {
		a := big.NewInt(0)
		for _, char := range name {
			b := big.NewInt( int64(char - '@'))
			a.Add(a, b)
		}
		fmt.Println(index)
		a.Mul(a, big.NewInt(int64(index+1)))
		fmt.Println(a)
		sum.Add(sum, a)
		fmt.Println(sum)
		if index == 3 {
			break
		}
	}
	fmt.Println(sum)
}
<commit_msg>projecteuler 22 correct solution<commit_after>package main

import (
	"fmt"
	"math/big"
	"io/ioutil"
	"log"
	"strings"
	"sort"
)

func main() {
	sum := big.NewInt(0)
	file, err := ioutil.ReadFile("names.txt")
	if err != nil {
		log.Fatal(err)
	}
	names := strings.Split(string(file[1:len(file)-1]),"\",\"")
	//fmt.Println(names, sum)
	sort.Strings(names)
	for index, name := range names {
		a := big.NewInt(0)
		for _, char := range name {
			b := big.NewInt( int64(char - '@'))
			//fmt.Println(b)
			a.Add(a, b)
		}
		//fmt.Println(index+1)
		//fmt.Println(a)
		a.Mul(a, big.NewInt(int64(index+1)))
		//fmt.Println(a)
		sum.Add(sum, a)
		//fmt.Println(sum)
	}
	fmt.Println(sum)
}
<|endoftext|>


<commit_before>package drive

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/erik/gruppo/converters"
	"github.com/erik/gruppo/model"
	"github.com/erik/gruppo/store"

	"github.com/google/uuid"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/drive/v3"
)

const (
	MimeTypeDocx        = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
	MimeTypeDriveFolder = "application/vnd.google-apps.folder"
	MimeTypeDriveDoc    = "application/vnd.google-apps.document"
)

type Configuration struct {
	ClientId             string
	ClientSecret         string
	RedirectURI          string
	WatchNotificationURI string
}

type GoogleDriveProvider struct {
	oauth  *oauth2.Config
	config *Configuration
}

type Client struct {
	service *drive.Service
	config  *Configuration
}

type folder struct {
	path string
	id   string
}

type File struct {
	Id     string
	Name   string
	Path   string
	Author string
}

const (
	fileChangeCreated = "created"
	fileChangeUpdated = "updated"
	fileChangeDeleted = "deleted"
)

type FileChange struct {
	File File
	Site model.Site
	Kind string // one of "created", "deleted", "updated"
}

// Either `File` or an error. It's like a union type, but fucking stupid.
type FileResult interface{}

func NewGoogleDriveProvider(conf Configuration) GoogleDriveProvider {
	oauth := &oauth2.Config{
		ClientID:     conf.ClientId,
		ClientSecret: conf.ClientSecret,
		RedirectURL:  conf.RedirectURI,

		Scopes:   []string{drive.DriveReadonlyScope},
		Endpoint: google.Endpoint,
	}

	return GoogleDriveProvider{oauth, &conf}
}

func (p GoogleDriveProvider) ClientForToken(tok *oauth2.Token) (*Client, error) {
	client := p.oauth.Client(context.TODO(), tok)

	svc, err := drive.New(client)
	if err != nil {
		return nil, err
	}

	return &Client{svc, p.config}, nil
}

func (c Client) Start(forceSync bool, site *model.Site, store *store.RedisStore, conf Configuration) {
	go c.changeWatcherRefresher(site.Drive.FolderId, site, store)
	go c.changeHandler(site, store)

	if forceSync {
		log.Printf("Starting file sync for site %s\n", site.HostPathPrefix())

		if err := c.ForceSync(*site, store); err != nil {
			log.Fatal(err)
		}
	}
}

func (c Client) ChangeHookRoute(site *model.Site) string {
	key := site.WebhookKey()

	// TODO: This is idiotic.
	url, err := url.Parse(c.config.WatchNotificationURI + key)
	if err != nil {
		panic(err)
	}

	return url.Path
}

func EnqueueFileChange(site *model.Site, fc FileChange, db *store.RedisStore) error {
	k := store.KeyForSite(*site, "changes")
	return db.AddSetJSON(k, fc)
}

func DequeueFileChanges(site *model.Site, db *store.RedisStore) ([]FileChange, error) {
	k := store.KeyForSite(*site, "changes")
	items, err := db.PopSetMembers(k)

	if err != nil {
		return nil, err
	}

	changes := make([]FileChange, len(items))
	for i, item := range items {
		dec := json.NewDecoder(strings.NewReader(item))
		if err := dec.Decode(&changes[i]); err != nil {
			return nil, err
		}
	}

	return changes, nil
}

func (c Client) changeHandler(site *model.Site, db *store.RedisStore) {
	fileChanges := []FileChange{}
	for {
		if len(fileChanges) == 0 {
			ch, err := DequeueFileChanges(site, db)
			if err != nil {
				log.Printf("Dequeue changes failed: %+v\n", err)
				return
			}

			fileChanges = append(fileChanges, ch...)
		}

		for _, ch := range fileChanges {
			log.Printf("TODO: handle file change %+v\n", ch)

			time.Sleep(30 * time.Second)
		}
	}
}

func (c Client) HandleChangeHook(site *model.Site, req *http.Request, db *store.RedisStore) error {
	headers := req.Header

	resource := headers["X-Goog-Resource-Id"]
	if len(resource) < 1 {
		log.Printf("Hook missing resource id")
		return nil
	}

	state := headers["X-Goog-Resource-State"]
	if len(state) < 1 {
		log.Printf("Hook missing resource state")
		return nil
	}

	change := FileChange{
		Site: *site,
		File: File{Id: resource[0]},
	}

	switch state[0] {
	case "sync":
		log.Printf("Hook registered successfully for %s\n", site.HostPathPrefix())
		return nil

	case "add":
		log.Printf("Adding resource: %+v\n", resource)
		change.Kind = fileChangeCreated

	case "update", "change":
		log.Printf("Updating resource: %+v\n", resource)
		change.Kind = fileChangeUpdated

	case "remove", "trash":
		log.Printf("Removing resource: %+v\n", resource)
		change.Kind = fileChangeDeleted

	default:
		log.Printf("Unknown value for resource state: %+v\n", state)
		return nil
	}

	return EnqueueFileChange(site, change, db)
}

func (c Client) changeWatcherRefresher(fileId string, site *model.Site, store *store.RedisStore) {
	key := site.WebhookKey()

	_, err := c.CreateChangeWatcher(fileId, key)
	if err != nil {
		log.Printf("ERROR: Failed to register file watcher: %+v\n", err)
	}

	// Watchers expire very frequently
	time.Sleep(55 * time.Minute)

}

func (c Client) CreateChangeWatcher(fileId string, key string) (string, error) {
	id := uuid.New().String()

	channel := &drive.Channel{
		Address: c.config.WatchNotificationURI + key,
		Id:      id,
		Type:    "web_hook",
	}

	_, err := c.service.Files.
		Watch(fileId, channel).
		Do()

	if err != nil {
		log.Printf("Failed to register file watcher: %+v\n", err)
		return "", err
	}

	return id, nil
}

func (c Client) ExportFile(file File, site model.Site, dir string) (*model.Post, error) {
	docx, err := c.ExportAsDocx(file)
	if err != nil {
		return nil, err
	}

	path := filepath.Join(dir, file.Id)
	if err := os.Mkdir(path, os.ModePerm); err != nil {
		return nil, err
	}

	inputFileName := filepath.Join(path, "input.docx")
	inputFile, err := os.Create(inputFileName)
	if err != nil {
		return nil, err
	}

	w := bufio.NewWriter(inputFile)
	if _, err := io.Copy(w, docx); err != nil {
		return nil, err
	}
	if err := w.Flush(); err != nil {
		return nil, err
	}

	md, err := converters.ConvertDocx(inputFileName, path)
	if err != nil {
		return nil, err
	}

	post := converters.ExtractPost(md)
	post.Author = file.Author
	post.Slug = post.GenerateSlug(file.Path)

	if err := converters.HandlePostMedia(&site, &post); err != nil {
		return nil, err
	}

	return &post, nil
}

// ForceSync ...
func (c Client) ForceSync(site model.Site, db *store.RedisStore) error {
	dir, err := ioutil.TempDir("/tmp", "exported-media-")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(dir)

	// Start with a clean slate
	if err := db.ClearSiteData(site); err != nil {
		return err
	}

	// Generated List of posts
	overview := []model.PostOverview{}

	for res := range c.List(site.Drive.FolderId) {
		file, ok := res.(File)
		if !ok {
			return res.(error)
		}

		post, err := c.ExportFile(file, site, dir)
		if err != nil {
			log.Printf("Failed to export file from drive: %+v\n", err)
			return err
		}

		go c.changeWatcherRefresher(file.Id, &site, db)

		overview = append(overview, post.Overview())

		log.Printf("[INFO] extracted post: %+v", post.Slug)

		if err := db.AddPost(site, *post); err != nil {
			return err
		}
	}

	if err := db.SetPostOverviews(site, overview); err != nil {
		return err
	}

	return nil
}

func (c Client) ExportAsDocx(file File) (io.Reader, error) {
	log.Printf("[INFO] exporting %s as .docx", file.Name)

	// TODO: Support large file download via chunked transfer.
	res, err := c.service.Files.
		Export(file.Id, MimeTypeDocx).
		Download()

	if err != nil {
		return nil, err
	}

	return res.Body, nil
}

func (c Client) listFolder(rootId string, files chan FileResult) {
	defer close(files)

	var current folder

	// Ids of folders that we haven't explored yet
	folders := []folder{
		folder{id: rootId},
	}

	for len(folders) > 0 {
		current, folders = folders[0], folders[1:]

		log.Printf("[INFO] exploring folder: %v\n", current)

		query := c.service.Files.
			List().
			PageSize(1000).
			Fields("nextPageToken, files(id, name, mimeType, lastModifyingUser(displayName))").
			Q(fmt.Sprintf("parents in \"%s\"", current.id))

		// Page through results
		for tok := "."; tok != ""; {
			result, err := query.Do()
			if err != nil {
				files <- err
				return
			}

			for _, f := range result.Files {
				switch f.MimeType {
				case MimeTypeDriveFolder:
					log.Printf("[INFO] queuing directory: %s\n", f.Name)
					folders = append(folders, folder{
						id:   f.Id,
						path: filepath.Join(current.path, f.Name),
					})

				case MimeTypeDriveDoc:
					files <- File{
						Id:     f.Id,
						Name:   f.Name,
						Author: f.LastModifyingUser.DisplayName,
						Path:   current.path,
						// TODO: Publish time (maybe from content?)
					}

				default:
					log.Printf("[INFO] skipping object of unknown type (%s): %v\n", f.MimeType, f)
				}
			}

			tok = result.NextPageToken
			query = query.PageToken(tok)
		}
	}
}

// Recursively list all Google Doc files nested under `folderId`.
// Implemented as breadth first search.
func (c Client) List(folderId string) <-chan FileResult {
	files := make(chan FileResult, 1)

	go c.listFolder(folderId, files)

	return files
}

// Request a token from the web, then returns the retrieved token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go to the following link in your browser then type the "+
		"authorization code: \n%v\n", authURL)

	var authCode string
	if _, err := fmt.Scan(&authCode); err != nil {
		log.Fatalf("Unable to read authorization code %v", err)
	}

	tok, err := config.Exchange(context.Background(), authCode)
	if err != nil {
		log.Fatalf("Unable to retrieve token from web %v", err)
	}

	return tok
}

// Retrieves a token from a local file.
// TODO: Remove this, store token somewhere sensible
func TokenFromFile(file string) (*oauth2.Token, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	tok := &oauth2.Token{}
	err = json.NewDecoder(f).Decode(tok)

	return tok, err
}

// Saves a token to a file path.
func saveToken(path string, token *oauth2.Token) {
	fmt.Printf("Saving credential file to: %s\n", path)
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)

	if err != nil {
		log.Fatalf("Unable to cache oauth token: %v", err)
	}

	defer f.Close()

	json.NewEncoder(f).Encode(token)
}
<commit_msg>Trigger force sync on change<commit_after>package drive

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/erik/gruppo/converters"
	"github.com/erik/gruppo/model"
	"github.com/erik/gruppo/store"

	"github.com/google/uuid"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/drive/v3"
)

const (
	MimeTypeDocx        = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
	MimeTypeDriveFolder = "application/vnd.google-apps.folder"
	MimeTypeDriveDoc    = "application/vnd.google-apps.document"
)

type Configuration struct {
	ClientId             string
	ClientSecret         string
	RedirectURI          string
	WatchNotificationURI string
}

type GoogleDriveProvider struct {
	oauth  *oauth2.Config
	config *Configuration
}

type Client struct {
	service *drive.Service
	config  *Configuration
}

type folder struct {
	path string
	id   string
}

type File struct {
	Id     string
	Name   string
	Path   string
	Author string
}

const (
	fileChangeCreated = "created"
	fileChangeUpdated = "updated"
	fileChangeDeleted = "deleted"
)

type FileChange struct {
	File File
	Site model.Site
	Kind string // one of "created", "deleted", "updated"
}

// Either `File` or an error. It's like a union type, but fucking stupid.
type FileResult interface{}

func NewGoogleDriveProvider(conf Configuration) GoogleDriveProvider {
	oauth := &oauth2.Config{
		ClientID:     conf.ClientId,
		ClientSecret: conf.ClientSecret,
		RedirectURL:  conf.RedirectURI,

		Scopes:   []string{drive.DriveReadonlyScope},
		Endpoint: google.Endpoint,
	}

	return GoogleDriveProvider{oauth, &conf}
}

func (p GoogleDriveProvider) ClientForToken(tok *oauth2.Token) (*Client, error) {
	client := p.oauth.Client(context.TODO(), tok)

	svc, err := drive.New(client)
	if err != nil {
		return nil, err
	}

	return &Client{svc, p.config}, nil
}

func (c Client) Start(forceSync bool, site *model.Site, store *store.RedisStore, conf Configuration) {
	go c.changeWatcherRefresher(site.Drive.FolderId, site, store)
	go c.changeHandler(site, store)

	if forceSync {
		log.Printf("Starting file sync for site %s\n", site.HostPathPrefix())

		if err := c.ForceSync(*site, store); err != nil {
			log.Fatal(err)
		}
	}
}

func (c Client) ChangeHookRoute(site *model.Site) string {
	key := site.WebhookKey()

	// TODO: This is idiotic.
	url, err := url.Parse(c.config.WatchNotificationURI + key)
	if err != nil {
		panic(err)
	}

	return url.Path
}

func EnqueueFileChange(site *model.Site, fc FileChange, db *store.RedisStore) error {
	k := store.KeyForSite(*site, "changes")
	return db.AddSetJSON(k, fc)
}

func DequeueFileChanges(site *model.Site, db *store.RedisStore) ([]FileChange, error) {
	k := store.KeyForSite(*site, "changes")
	items, err := db.PopSetMembers(k)

	if err != nil {
		return nil, err
	}

	changes := make([]FileChange, len(items))
	for i, item := range items {
		dec := json.NewDecoder(strings.NewReader(item))
		if err := dec.Decode(&changes[i]); err != nil {
			return nil, err
		}
	}

	return changes, nil
}

func (c Client) changeHandler(site *model.Site, db *store.RedisStore) {
	fileChanges := []FileChange{}
	for {
		if len(fileChanges) == 0 {
			ch, err := DequeueFileChanges(site, db)
			if err != nil {
				log.Printf("Dequeue changes failed: %+v\n", err)
				return
			}

			fileChanges = append(fileChanges, ch...)
		}

		for _, ch := range fileChanges {
			log.Printf("TODO: handle file change %+v\n", ch)

			// TODO: Haven't written a file specific version yet
			c.ForceSync(*site, db)

			time.Sleep(30 * time.Second)
		}
	}
}

func (c Client) HandleChangeHook(site *model.Site, req *http.Request, db *store.RedisStore) error {
	headers := req.Header

	resource := headers["X-Goog-Resource-Id"]
	if len(resource) < 1 {
		log.Printf("Hook missing resource id")
		return nil
	}

	state := headers["X-Goog-Resource-State"]
	if len(state) < 1 {
		log.Printf("Hook missing resource state")
		return nil
	}

	change := FileChange{
		Site: *site,
		File: File{Id: resource[0]},
	}

	switch state[0] {
	case "sync":
		log.Printf("Hook registered successfully for %s\n", site.HostPathPrefix())
		return nil

	case "add":
		log.Printf("Adding resource: %+v\n", resource)
		change.Kind = fileChangeCreated

	case "update", "change":
		log.Printf("Updating resource: %+v\n", resource)
		change.Kind = fileChangeUpdated

	case "remove", "trash":
		log.Printf("Removing resource: %+v\n", resource)
		change.Kind = fileChangeDeleted

	default:
		log.Printf("Unknown value for resource state: %+v\n", state)
		return nil
	}

	return EnqueueFileChange(site, change, db)
}

func (c Client) changeWatcherRefresher(fileId string, site *model.Site, store *store.RedisStore) {
	key := site.WebhookKey()

	_, err := c.CreateChangeWatcher(fileId, key)
	if err != nil {
		log.Printf("ERROR: Failed to register file watcher: %+v\n", err)
	}

	// Watchers expire very frequently
	time.Sleep(55 * time.Minute)

}

func (c Client) CreateChangeWatcher(fileId string, key string) (string, error) {
	id := uuid.New().String()

	channel := &drive.Channel{
		Address: c.config.WatchNotificationURI + key,
		Id:      id,
		Type:    "web_hook",
	}

	_, err := c.service.Files.
		Watch(fileId, channel).
		Do()

	if err != nil {
		log.Printf("Failed to register file watcher: %+v\n", err)
		return "", err
	}

	return id, nil
}

func (c Client) ExportFile(file File, site model.Site, dir string) (*model.Post, error) {
	docx, err := c.ExportAsDocx(file)
	if err != nil {
		return nil, err
	}

	path := filepath.Join(dir, file.Id)
	if err := os.Mkdir(path, os.ModePerm); err != nil {
		return nil, err
	}

	inputFileName := filepath.Join(path, "input.docx")
	inputFile, err := os.Create(inputFileName)
	if err != nil {
		return nil, err
	}

	w := bufio.NewWriter(inputFile)
	if _, err := io.Copy(w, docx); err != nil {
		return nil, err
	}
	if err := w.Flush(); err != nil {
		return nil, err
	}

	md, err := converters.ConvertDocx(inputFileName, path)
	if err != nil {
		return nil, err
	}

	post := converters.ExtractPost(md)
	post.Author = file.Author
	post.Slug = post.GenerateSlug(file.Path)

	if err := converters.HandlePostMedia(&site, &post); err != nil {
		return nil, err
	}

	return &post, nil
}

// ForceSync ...
func (c Client) ForceSync(site model.Site, db *store.RedisStore) error {
	dir, err := ioutil.TempDir("/tmp", "exported-media-")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(dir)

	// Start with a clean slate
	if err := db.ClearSiteData(site); err != nil {
		return err
	}

	// Generated List of posts
	overview := []model.PostOverview{}

	for res := range c.List(site.Drive.FolderId) {
		file, ok := res.(File)
		if !ok {
			return res.(error)
		}

		post, err := c.ExportFile(file, site, dir)
		if err != nil {
			log.Printf("Failed to export file from drive: %+v\n", err)
			return err
		}

		go c.changeWatcherRefresher(file.Id, &site, db)

		overview = append(overview, post.Overview())

		log.Printf("[INFO] extracted post: %+v", post.Slug)

		if err := db.AddPost(site, *post); err != nil {
			return err
		}
	}

	if err := db.SetPostOverviews(site, overview); err != nil {
		return err
	}

	return nil
}

func (c Client) ExportAsDocx(file File) (io.Reader, error) {
	log.Printf("[INFO] exporting %s as .docx", file.Name)

	// TODO: Support large file download via chunked transfer.
	res, err := c.service.Files.
		Export(file.Id, MimeTypeDocx).
		Download()

	if err != nil {
		return nil, err
	}

	return res.Body, nil
}

func (c Client) listFolder(rootId string, files chan FileResult) {
	defer close(files)

	var current folder

	// Ids of folders that we haven't explored yet
	folders := []folder{
		folder{id: rootId},
	}

	for len(folders) > 0 {
		current, folders = folders[0], folders[1:]

		log.Printf("[INFO] exploring folder: %v\n", current)

		query := c.service.Files.
			List().
			PageSize(1000).
			Fields("nextPageToken, files(id, name, mimeType, lastModifyingUser(displayName))").
			Q(fmt.Sprintf("parents in \"%s\"", current.id))

		// Page through results
		for tok := "."; tok != ""; {
			result, err := query.Do()
			if err != nil {
				files <- err
				return
			}

			for _, f := range result.Files {
				switch f.MimeType {
				case MimeTypeDriveFolder:
					log.Printf("[INFO] queuing directory: %s\n", f.Name)
					folders = append(folders, folder{
						id:   f.Id,
						path: filepath.Join(current.path, f.Name),
					})

				case MimeTypeDriveDoc:
					files <- File{
						Id:     f.Id,
						Name:   f.Name,
						Author: f.LastModifyingUser.DisplayName,
						Path:   current.path,
						// TODO: Publish time (maybe from content?)
					}

				default:
					log.Printf("[INFO] skipping object of unknown type (%s): %v\n", f.MimeType, f)
				}
			}

			tok = result.NextPageToken
			query = query.PageToken(tok)
		}
	}
}

// Recursively list all Google Doc files nested under `folderId`.
// Implemented as breadth first search.
func (c Client) List(folderId string) <-chan FileResult {
	files := make(chan FileResult, 1)

	go c.listFolder(folderId, files)

	return files
}

// Request a token from the web, then returns the retrieved token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go to the following link in your browser then type the "+
		"authorization code: \n%v\n", authURL)

	var authCode string
	if _, err := fmt.Scan(&authCode); err != nil {
		log.Fatalf("Unable to read authorization code %v", err)
	}

	tok, err := config.Exchange(context.Background(), authCode)
	if err != nil {
		log.Fatalf("Unable to retrieve token from web %v", err)
	}

	return tok
}

// Retrieves a token from a local file.
// TODO: Remove this, store token somewhere sensible
func TokenFromFile(file string) (*oauth2.Token, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	tok := &oauth2.Token{}
	err = json.NewDecoder(f).Decode(tok)

	return tok, err
}

// Saves a token to a file path.
func saveToken(path string, token *oauth2.Token) {
	fmt.Printf("Saving credential file to: %s\n", path)
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)

	if err != nil {
		log.Fatalf("Unable to cache oauth token: %v", err)
	}

	defer f.Close()

	json.NewEncoder(f).Encode(token)
}
<|endoftext|>