<commit_before>package secret

import (
	"sort"
	"testing"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/swarm"
	"github.com/gotestyourself/gotestyourself/skip"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

func TestSecretInspect(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName := "test_secret"
	secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil)

	secret, _, err := client.SecretInspectWithRaw(context.Background(), secretID)
	require.NoError(t, err)
	assert.Equal(t, secret.Spec.Name, testName)

	secret, _, err = client.SecretInspectWithRaw(context.Background(), testName)
	require.NoError(t, err)
	assert.Equal(t, secret.ID, secretID)
}

func TestSecretList(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName0 := "test0"
	testName1 := "test1"
	testNames := []string{testName0, testName1}
	sort.Strings(testNames)

	// create secret test0
	createSecret(ctx, t, client, testName0, []byte("TESTINGDATA0"), map[string]string{"type": "test"})

	// create secret test1
	secret1ID := createSecret(ctx, t, client, testName1, []byte("TESTINGDATA1"), map[string]string{"type": "production"})

	names := func(entries []swarmtypes.Secret) []string {
		values := []string{}
		for _, entry := range entries {
			values = append(values, entry.Spec.Name)
		}
		sort.Strings(values)
		return values
	}

	// test by `secret ls`
	entries, err := client.SecretList(ctx, types.SecretListOptions{})
	require.NoError(t, err)
	assert.Equal(t, names(entries), testNames)

	testCases := []struct {
		filters  filters.Args
		expected []string
	}{
		// test filter by name `secret ls --filter name=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("name", testName0)),
			expected: []string{testName0},
		},
		// test filter by id `secret ls --filter id=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("id", secret1ID)),
			expected: []string{testName1},
		},
		// test filter by label `secret ls --filter label=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("label", "type")),
			expected: testNames,
		},
		{
			filters:  filters.NewArgs(filters.Arg("label", "type=test")),
			expected: []string{testName0},
		},
		{
			filters:  filters.NewArgs(filters.Arg("label", "type=production")),
			expected: []string{testName1},
		},
	}
	for _, tc := range testCases {
		entries, err = client.SecretList(ctx, types.SecretListOptions{
			Filters: tc.filters,
		})
		require.NoError(t, err)
		assert.Equal(t, names(entries), tc.expected)

	}
}

func createSecret(ctx context.Context, t *testing.T, client client.APIClient, name string, data []byte, labels map[string]string) string {
	secret, err := client.SecretCreate(ctx, swarmtypes.SecretSpec{
		Annotations: swarmtypes.Annotations{
			Name:   name,
			Labels: labels,
		},
		Data: data,
	})
	require.NoError(t, err)
	assert.NotEqual(t, secret.ID, "")
	return secret.ID
}
<commit_msg>Migrates several swarm secrets from integration-cli to api tests<commit_after>package secret

import (
	"sort"
	"testing"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	swarmtypes "github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration/internal/swarm"
	"github.com/docker/docker/internal/testutil"
	"github.com/gotestyourself/gotestyourself/skip"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

func TestSecretInspect(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName := "test_secret"
	secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil)

	secret, _, err := client.SecretInspectWithRaw(context.Background(), secretID)
	require.NoError(t, err)
	assert.Equal(t, secret.Spec.Name, testName)

	secret, _, err = client.SecretInspectWithRaw(context.Background(), testName)
	require.NoError(t, err)
	assert.Equal(t, secretID, secretID)
}

func TestSecretList(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName0 := "test0"
	testName1 := "test1"
	testNames := []string{testName0, testName1}
	sort.Strings(testNames)

	// create secret test0
	createSecret(ctx, t, client, testName0, []byte("TESTINGDATA0"), map[string]string{"type": "test"})

	// create secret test1
	secret1ID := createSecret(ctx, t, client, testName1, []byte("TESTINGDATA1"), map[string]string{"type": "production"})

	names := func(entries []swarmtypes.Secret) []string {
		values := []string{}
		for _, entry := range entries {
			values = append(values, entry.Spec.Name)
		}
		sort.Strings(values)
		return values
	}

	// test by `secret ls`
	entries, err := client.SecretList(ctx, types.SecretListOptions{})
	require.NoError(t, err)
	assert.Equal(t, names(entries), testNames)

	testCases := []struct {
		filters  filters.Args
		expected []string
	}{
		// test filter by name `secret ls --filter name=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("name", testName0)),
			expected: []string{testName0},
		},
		// test filter by id `secret ls --filter id=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("id", secret1ID)),
			expected: []string{testName1},
		},
		// test filter by label `secret ls --filter label=xxx`
		{
			filters:  filters.NewArgs(filters.Arg("label", "type")),
			expected: testNames,
		},
		{
			filters:  filters.NewArgs(filters.Arg("label", "type=test")),
			expected: []string{testName0},
		},
		{
			filters:  filters.NewArgs(filters.Arg("label", "type=production")),
			expected: []string{testName1},
		},
	}
	for _, tc := range testCases {
		entries, err = client.SecretList(ctx, types.SecretListOptions{
			Filters: tc.filters,
		})
		require.NoError(t, err)
		assert.Equal(t, names(entries), tc.expected)

	}
}

func createSecret(ctx context.Context, t *testing.T, client client.APIClient, name string, data []byte, labels map[string]string) string {
	secret, err := client.SecretCreate(ctx, swarmtypes.SecretSpec{
		Annotations: swarmtypes.Annotations{
			Name:   name,
			Labels: labels,
		},
		Data: data,
	})
	require.NoError(t, err)
	assert.NotEqual(t, secret.ID, "")
	return secret.ID
}

func TestSecretsCreate(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName := "test_secret"
	createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil)
	require.NoError(t, err)

	// create an already existin secret, daemon should return a status code of 409
	_, err = client.SecretCreate(ctx, swarmtypes.SecretSpec{
		Annotations: swarmtypes.Annotations{
			Name: testName,
		},
		Data: []byte("TESTINGDATA"),
	})
	testutil.ErrorContains(t, err, "already exists")
}

func TestSecretsDelete(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName := "test_secret"
	secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil)
	require.NoError(t, err)

	insp, _, err := client.SecretInspectWithRaw(ctx, secretID)
	require.NoError(t, err)
	assert.Equal(t, insp.ID, secretID)

	err = client.SecretRemove(ctx, secretID)
	require.NoError(t, err)

	_, _, err = client.SecretInspectWithRaw(ctx, secretID)
	testutil.ErrorContains(t, err, "No such secret")

	err = client.SecretRemove(ctx, "non-existin")
	testutil.ErrorContains(t, err, "No such secret: non-existin")
}

func TestSecretsUpdate(t *testing.T) {
	skip.If(t, testEnv.DaemonInfo.OSType != "linux")

	defer setupTest(t)()
	d := swarm.NewSwarm(t, testEnv)
	defer d.Stop(t)
	client, err := client.NewClientWithOpts(client.WithHost((d.Sock())))
	require.NoError(t, err)

	ctx := context.Background()

	testName := "test_secret"
	secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil)
	require.NoError(t, err)

	insp, _, err := client.SecretInspectWithRaw(ctx, secretID)
	require.NoError(t, err)
	assert.Equal(t, insp.ID, secretID)

	// test UpdateSecret with full ID
	insp.Spec.Labels = map[string]string{"test": "test1"}
	err = client.SecretUpdate(ctx, secretID, insp.Version, insp.Spec)
	require.NoError(t, err)

	insp, _, err = client.SecretInspectWithRaw(ctx, secretID)
	require.NoError(t, err)
	assert.Equal(t, insp.Spec.Labels["test"], "test1")

	// test UpdateSecret with full name
	insp.Spec.Labels = map[string]string{"test": "test2"}
	err = client.SecretUpdate(ctx, testName, insp.Version, insp.Spec)
	require.NoError(t, err)

	insp, _, err = client.SecretInspectWithRaw(ctx, secretID)
	require.NoError(t, err)
	assert.Equal(t, insp.Spec.Labels["test"], "test2")

	// test UpdateSecret with prefix ID
	insp.Spec.Labels = map[string]string{"test": "test3"}
	err = client.SecretUpdate(ctx, secretID[:1], insp.Version, insp.Spec)
	require.NoError(t, err)

	insp, _, err = client.SecretInspectWithRaw(ctx, secretID)
	require.NoError(t, err)
	assert.Equal(t, insp.Spec.Labels["test"], "test3")

	// test UpdateSecret in updating Data which is not supported in daemon
	// this test will produce an error in func UpdateSecret
	insp.Spec.Data = []byte("TESTINGDATA2")
	err = client.SecretUpdate(ctx, secretID, insp.Version, insp.Spec)
	testutil.ErrorContains(t, err, "only updates to Labels are allowed")
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exportimport

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/pkg/logging"
	"go.opencensus.io/trace"
)

const lockPrefix = "import-lock-"

func (s *Server) handleImport(ctx context.Context) http.HandlerFunc {
	logger := logging.FromContext(ctx).Named("exportimport.HandleImport")

	return func(w http.ResponseWriter, r *http.Request) {
		_, span := trace.StartSpan(r.Context(), "(*keyrotation.handler).ServeHTTP")
		defer span.End()

		ctx, cancelFn := context.WithDeadline(r.Context(), time.Now().Add(s.config.MaxRuntime))
		defer cancelFn()
		logger.Info("starting export importer")
		defer func() {
			w.WriteHeader(http.StatusOK)
			w.Write([]byte("ok"))
		}()

		configs, err := s.exportImportDB.ActiveConfigs(ctx)
		if err != nil {
			logger.Errorw("unable to read active configs", "error", err)
		}

		for _, config := range configs {
			// Check how we're doing on max runtime.
			if deadlinePassed(ctx) {
				logger.Warnf("deadline passed, still work to do")
				return
			}

			// Obtain a lock to work on this import config.
			unlock, err := s.db.Lock(ctx, fmt.Sprintf("%s%d", lockPrefix, config.ID), s.config.MaxRuntime)
			if err != nil {
				if errors.Is(err, database.ErrAlreadyLocked) {
					logger.Warnw("import already locked", "config", config)
				}
				logger.Errorw("error locking import config", "config", config, "error", err)
				continue
			}
			defer func() {
				if err := unlock(); err != nil {
					logger.Errorf("failed to unlock: %v", err)
				}
			}()

			// Get the list of files that needs to be processed.
			openFiles, err := s.exportImportDB.GetOpenImportFiles(ctx, s.config.ImportLockTime, config)
			if err != nil {
				logger.Errorw("unable to read open export files", "config", config, "error", err)
			}
			if len(openFiles) == 0 {
				logger.Infow("no work to do", "config", config)
				continue
			}

			// Read in public keys.
			keys, err := s.exportImportDB.AllowedKeys(ctx, config)
			if err != nil {
				logger.Errorw("unable to read public keys", "config", config, "error", err)
				continue
			}
			logger.Debugw("allowed public keys for file", "publicKeys", keys)

			for _, file := range openFiles {
				// Check how we're doing on max runtime.
				if deadlinePassed(ctx) {
					logger.Warnw("deadline passed, still work to do", "config", config)
					return
				}

				if err := s.exportImportDB.LeaseImportFile(ctx, s.config.ImportLockTime, file); err != nil {
					logger.Warnw("unexpected race condition, file already locked", "file", file, "error", err)
					continue
				}

				// import the file.
				result, err := s.ImportExportFile(ctx, &ImportRequest{
					config:       s.config,
					exportImport: config,
					keys:         keys,
					file:         file,
				})
				if err != nil {
					logger.Errorf("error processing import file", "error", err)
					continue
				}
				logger.Infow("completed file import", "inserted", result.insertedKeys, "revised", result.revisedKeys, "dropped", result.droppedKeys)

				if err := s.exportImportDB.CompleteImportFile(ctx, file); err != nil {
					logger.Errorf("failed to mark file completed", "file", file, "error", err)
				}
			}
		}

	}
}

func deadlinePassed(ctx context.Context) bool {
	deadline, ok := ctx.Deadline()
	if !ok {
		return false
	}
	if time.Now().After(deadline) {
		return true
	}
	return false
}
<commit_msg>Update importer.go (#1075)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exportimport

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/pkg/logging"
	"go.opencensus.io/trace"
)

const lockPrefix = "import-lock-"

func (s *Server) handleImport(ctx context.Context) http.HandlerFunc {
	logger := logging.FromContext(ctx).Named("exportimport.HandleImport")

	return func(w http.ResponseWriter, r *http.Request) {
		_, span := trace.StartSpan(r.Context(), "(*keyrotation.handler).ServeHTTP")
		defer span.End()

		ctx, cancelFn := context.WithDeadline(r.Context(), time.Now().Add(s.config.MaxRuntime))
		defer cancelFn()
		logger.Info("starting export importer")
		defer func() {
			w.WriteHeader(http.StatusOK)
			w.Write([]byte("ok"))
		}()

		configs, err := s.exportImportDB.ActiveConfigs(ctx)
		if err != nil {
			logger.Errorw("unable to read active configs", "error", err)
		}

		for _, config := range configs {
			// Check how we're doing on max runtime.
			if deadlinePassed(ctx) {
				logger.Warnf("deadline passed, still work to do")
				return
			}

			// Obtain a lock to work on this import config.
			unlock, err := s.db.Lock(ctx, fmt.Sprintf("%s%d", lockPrefix, config.ID), s.config.MaxRuntime)
			if err != nil {
				if errors.Is(err, database.ErrAlreadyLocked) {
					logger.Warnw("import already locked", "config", config)
				}
				logger.Errorw("error locking import config", "config", config, "error", err)
				continue
			}
			defer func() {
				if err := unlock(); err != nil {
					logger.Errorw("failed to unlock", "error", err)
				}
			}()

			// Get the list of files that needs to be processed.
			openFiles, err := s.exportImportDB.GetOpenImportFiles(ctx, s.config.ImportLockTime, config)
			if err != nil {
				logger.Errorw("unable to read open export files", "config", config, "error", err)
			}
			if len(openFiles) == 0 {
				logger.Infow("no work to do", "config", config)
				continue
			}

			// Read in public keys.
			keys, err := s.exportImportDB.AllowedKeys(ctx, config)
			if err != nil {
				logger.Errorw("unable to read public keys", "config", config, "error", err)
				continue
			}
			logger.Debugw("allowed public keys for file", "publicKeys", keys)

			for _, file := range openFiles {
				// Check how we're doing on max runtime.
				if deadlinePassed(ctx) {
					logger.Warnw("deadline passed, still work to do", "config", config)
					return
				}

				if err := s.exportImportDB.LeaseImportFile(ctx, s.config.ImportLockTime, file); err != nil {
					logger.Warnw("unexpected race condition, file already locked", "file", file, "error", err)
					continue
				}

				// import the file.
				result, err := s.ImportExportFile(ctx, &ImportRequest{
					config:       s.config,
					exportImport: config,
					keys:         keys,
					file:         file,
				})
				if err != nil {
					logger.Errorw("error processing import file", "error", err)
					continue
				}
				logger.Infow("completed file import", "inserted", result.insertedKeys, "revised", result.revisedKeys, "dropped", result.droppedKeys)

				if err := s.exportImportDB.CompleteImportFile(ctx, file); err != nil {
					logger.Errorw("failed to mark file completed", "file", file, "error", err)
				}
			}
		}

	}
}

func deadlinePassed(ctx context.Context) bool {
	deadline, ok := ctx.Deadline()
	if !ok {
		return false
	}
	if time.Now().After(deadline) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package terraform

import (
	"errors"
	"strings"
	"testing"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/terraform/config/configschema"
	"github.com/hashicorp/terraform/tfdiags"

	"github.com/hashicorp/terraform/configs"
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform/addrs"
)

func TestEvalValidateResource_managedResource(t *testing.T) {
	mp := testProvider("aws")
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		expected := "aws_instance"
		if rt != expected {
			t.Fatalf("expected: %s, got: %s", expected, rt)
		}
		expected = "bar"
		val, _ := c.Get("foo")
		if val != expected {
			t.Fatalf("expected: %s, got: %s", expected, val)
		}
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode: addrs.ManagedResourceMode,
		Type: "aws_instance",
		Name: "foo",
		Config: configs.SynthBody("", map[string]cty.Value{
			"foo": cty.StringVal("bar"),
		}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "aws_instance",
				Name: "foo",
			},
		},
		Provider: &p,
		Config:   rc,
	}

	_, err := node.Eval(&MockEvalContext{})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !mp.ValidateResourceCalled {
		t.Fatal("Expected ValidateResource to be called, but it was not!")
	}
}

func TestEvalValidateResource_dataSource(t *testing.T) {
	mp := testProvider("aws")
	mp.ValidateDataSourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		expected := "aws_ami"
		if rt != expected {
			t.Fatalf("expected: %s, got: %s", expected, rt)
		}
		expected = "bar"
		val, _ := c.Get("foo")
		if val != expected {
			t.Fatalf("expected: %s, got: %s", expected, val)
		}
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode: addrs.DataResourceMode,
		Type: "aws_ami",
		Name: "foo",
		Config: configs.SynthBody("", map[string]cty.Value{
			"foo": cty.StringVal("bar"),
		}),
	}

	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.DataResourceMode,
				Type: "aws_ami",
				Name: "foo",
			},
		},
		Provider: &p,
		Config:   rc,
	}

	_, err := node.Eval(&MockEvalContext{})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !mp.ValidateDataSourceCalled {
		t.Fatal("Expected ValidateDataSource to be called, but it was not!")
	}
}

func TestEvalValidateResource_validReturnsNilError(t *testing.T) {
	mp := testProvider("aws")
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "aws_instance",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "aws_instance",
				Name: "foo",
			},
		},
		Provider: &p,
		Config:   rc,
	}

	_, err := node.Eval(&MockEvalContext{})
	if err != nil {
		t.Fatalf("Expected nil error, got: %s", err)
	}
}

func TestEvalValidateResource_warningsAndErrorsPassedThrough(t *testing.T) {
	mp := testProvider("aws")
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		ws = append(ws, "warn")
		es = append(es, errors.New("err"))
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "aws_instance",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "aws_instance",
				Name: "foo",
			},
		},
		Provider: &p,
		Config:   rc,
	}

	_, err := node.Eval(&MockEvalContext{})
	if err == nil {
		t.Fatal("Expected an error, got none!")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	bySeverity := map[tfdiags.Severity]tfdiags.Diagnostics{}
	for _, diag := range diags {
		bySeverity[diag.Severity()] = append(bySeverity[diag.Severity()], diag)
	}
	if len(bySeverity[tfdiags.Warning]) != 1 || bySeverity[tfdiags.Warning][0].Description().Summary != "warn" {
		t.Fatalf("Expected 1 warning 'warn', got: %#v", bySeverity[tfdiags.Warning])
	}
	if len(bySeverity[tfdiags.Error]) != 1 || bySeverity[tfdiags.Error][0].Description().Summary != "err" {
		t.Fatalf("Expected 1 error 'err', got: %#v", bySeverity[tfdiags.Error])
	}
}

func TestEvalValidateResource_ignoreWarnings(t *testing.T) {
	mp := testProvider("aws")
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		ws = append(ws, "warn")
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "aws_instance",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "aws_instance",
				Name: "foo",
			},
		},
		Provider: &p,
		Config:   rc,

		IgnoreWarnings: true,
	}

	_, err := node.Eval(&MockEvalContext{})
	if err != nil {
		t.Fatalf("Expected no error, got: %s", err)
	}
}

func TestEvalValidateProvisioner_valid(t *testing.T) {
	mp := &MockResourceProvisioner{}
	var p ResourceProvisioner = mp
	ctx := &MockEvalContext{}

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type:   "ssh",
			Config: hcl.EmptyBody(),
		},
	}

	result, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("node.Eval failed: %s", err)
	}
	if result != nil {
		t.Errorf("node.Eval returned non-nil result")
	}

	if !mp.ValidateCalled {
		t.Fatalf("p.Validate not called")
	}
}

func TestEvalValidateProvisioner_warning(t *testing.T) {
	mp := &MockResourceProvisioner{}
	var p ResourceProvisioner = mp
	ctx := &MockEvalContext{}

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type:   "ssh",
			Config: hcl.EmptyBody(),
		},
	}

	mp.ValidateReturnWarns = []string{"foo is deprecated"}

	_, err := node.Eval(ctx)
	if err == nil {
		t.Fatalf("node.Eval succeeded; want error")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	if len(diags) != 1 {
		t.Fatalf("wrong number of diagsnostics in %#v; want one warning", diags)
	}

	if got, want := diags[0].Description().Summary, mp.ValidateReturnWarns[0]; got != want {
		t.Fatalf("wrong warning %q; want %q", got, want)
	}
}

func TestEvalValidateProvisioner_connectionInvalid(t *testing.T) {
	var p ResourceProvisioner = &MockResourceProvisioner{}
	ctx := &MockEvalContext{}

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type: "ssh",
			Config: configs.SynthBody("", map[string]cty.Value{
				"bananananananana": cty.StringVal("foo"),
				"bazaz":            cty.StringVal("bar"),
			}),
		},
	}

	_, err := node.Eval(ctx)
	if err == nil {
		t.Fatalf("node.Eval succeeded; want error")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	if len(diags) != 2 {
		t.Fatalf("wrong number of diagsnostics in %#v; want two errors", diags)
	}

	errStr := diags.Err().Error()
	if !(strings.Contains(errStr, "bananananananana") && strings.Contains(errStr, "bazaz")) {
		t.Fatalf("wrong errors %q; want something about each of our invalid connInfo keys", errStr)
	}
}
<commit_msg>core: Fix tests for types in eval_validate.go<commit_after>package terraform

import (
	"errors"
	"strings"
	"testing"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/config/configschema"
	"github.com/hashicorp/terraform/configs"
	"github.com/hashicorp/terraform/tfdiags"
)

func TestEvalValidateResource_managedResource(t *testing.T) {
	mp := simpleMockProvider()
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		expected := "test_object"
		if rt != expected {
			t.Fatalf("wrong resource type\ngot:  %#v\nwant: %#v", rt, expected)
		}
		expected = "bar"
		val, _ := c.Get("test_string")
		if val != expected {
			t.Fatalf("wrong value for test_string\ngot:  %#v\nwant: %#v", val, expected)
		}
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode: addrs.ManagedResourceMode,
		Type: "test_object",
		Name: "foo",
		Config: configs.SynthBody("", map[string]cty.Value{
			"test_string": cty.StringVal("bar"),
		}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "aws_instance",
				Name: "foo",
			},
		},
		Provider:       &p,
		Config:         rc,
		ProviderSchema: &mp.GetSchemaReturn,
	}

	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	_, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !mp.ValidateResourceCalled {
		t.Fatal("Expected ValidateResource to be called, but it was not!")
	}
}

func TestEvalValidateResource_dataSource(t *testing.T) {
	mp := simpleMockProvider()
	mp.ValidateDataSourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		expected := "test_object"
		if rt != expected {
			t.Fatalf("expected: %s, got: %s", expected, rt)
		}
		expected = "bar"
		val, _ := c.Get("test_string")
		if val != expected {
			t.Fatalf("expected: %s, got: %s", expected, val)
		}
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode: addrs.DataResourceMode,
		Type: "test_object",
		Name: "foo",
		Config: configs.SynthBody("", map[string]cty.Value{
			"test_string": cty.StringVal("bar"),
		}),
	}

	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.DataResourceMode,
				Type: "aws_ami",
				Name: "foo",
			},
		},
		Provider:       &p,
		Config:         rc,
		ProviderSchema: &mp.GetSchemaReturn,
	}

	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	_, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if !mp.ValidateDataSourceCalled {
		t.Fatal("Expected ValidateDataSource to be called, but it was not!")
	}
}

func TestEvalValidateResource_validReturnsNilError(t *testing.T) {
	mp := simpleMockProvider()
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "test_object",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "test_object",
				Name: "foo",
			},
		},
		Provider:       &p,
		Config:         rc,
		ProviderSchema: &mp.GetSchemaReturn,
	}

	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	_, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("Expected nil error, got: %s", err)
	}
}

func TestEvalValidateResource_warningsAndErrorsPassedThrough(t *testing.T) {
	mp := simpleMockProvider()
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		ws = append(ws, "warn")
		es = append(es, errors.New("err"))
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "test_object",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "test_object",
				Name: "foo",
			},
		},
		Provider:       &p,
		Config:         rc,
		ProviderSchema: &mp.GetSchemaReturn,
	}

	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	_, err := node.Eval(ctx)
	if err == nil {
		t.Fatal("unexpected success; want error")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	bySeverity := map[tfdiags.Severity]tfdiags.Diagnostics{}
	for _, diag := range diags {
		bySeverity[diag.Severity()] = append(bySeverity[diag.Severity()], diag)
	}
	if len(bySeverity[tfdiags.Warning]) != 1 || bySeverity[tfdiags.Warning][0].Description().Summary != "warn" {
		t.Errorf("Expected 1 warning 'warn', got: %s", bySeverity[tfdiags.Warning].ErrWithWarnings())
	}
	if len(bySeverity[tfdiags.Error]) != 1 || bySeverity[tfdiags.Error][0].Description().Summary != "err" {
		t.Errorf("Expected 1 error 'err', got: %s", bySeverity[tfdiags.Error].Err())
	}
}

func TestEvalValidateResource_ignoreWarnings(t *testing.T) {
	mp := simpleMockProvider()
	mp.ValidateResourceFn = func(rt string, c *ResourceConfig) (ws []string, es []error) {
		ws = append(ws, "warn")
		return
	}

	p := ResourceProvider(mp)
	rc := &configs.Resource{
		Mode:   addrs.ManagedResourceMode,
		Type:   "test_object",
		Name:   "foo",
		Config: configs.SynthBody("", map[string]cty.Value{}),
	}
	node := &EvalValidateResource{
		Addr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "test-object",
				Name: "foo",
			},
		},
		Provider:       &p,
		Config:         rc,
		ProviderSchema: &mp.GetSchemaReturn,

		IgnoreWarnings: true,
	}

	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	_, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("Expected no error, got: %s", err)
	}
}

func TestEvalValidateProvisioner_valid(t *testing.T) {
	mp := &MockResourceProvisioner{}
	var p ResourceProvisioner = mp
	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type:   "ssh",
			Config: hcl.EmptyBody(),
		},
	}

	result, err := node.Eval(ctx)
	if err != nil {
		t.Fatalf("node.Eval failed: %s", err)
	}
	if result != nil {
		t.Errorf("node.Eval returned non-nil result")
	}

	if !mp.ValidateCalled {
		t.Fatalf("p.Validate not called")
	}
}

func TestEvalValidateProvisioner_warning(t *testing.T) {
	mp := &MockResourceProvisioner{}
	var p ResourceProvisioner = mp
	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type:   "ssh",
			Config: hcl.EmptyBody(),
		},
	}

	mp.ValidateReturnWarns = []string{"foo is deprecated"}

	_, err := node.Eval(ctx)
	if err == nil {
		t.Fatalf("node.Eval succeeded; want error")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	if len(diags) != 1 {
		t.Fatalf("wrong number of diagsnostics in %#v; want one warning", diags)
	}

	if got, want := diags[0].Description().Summary, mp.ValidateReturnWarns[0]; got != want {
		t.Fatalf("wrong warning %q; want %q", got, want)
	}
}

func TestEvalValidateProvisioner_connectionInvalid(t *testing.T) {
	var p ResourceProvisioner = &MockResourceProvisioner{}
	ctx := &MockEvalContext{}
	ctx.installSimpleEval()

	schema := &configschema.Block{}

	node := &EvalValidateProvisioner{
		ResourceAddr: addrs.ResourceInstance{
			Resource: addrs.Resource{
				Mode: addrs.ManagedResourceMode,
				Type: "foo",
				Name: "bar",
			},
		},
		Provisioner: &p,
		Schema:      &schema,
		Config: &configs.Provisioner{
			Type:   "baz",
			Config: hcl.EmptyBody(),
		},
		ConnConfig: &configs.Connection{
			Type: "ssh",
			Config: configs.SynthBody("", map[string]cty.Value{
				"bananananananana": cty.StringVal("foo"),
				"bazaz":            cty.StringVal("bar"),
			}),
		},
	}

	_, err := node.Eval(ctx)
	if err == nil {
		t.Fatalf("node.Eval succeeded; want error")
	}

	var diags tfdiags.Diagnostics
	diags = diags.Append(err)
	if len(diags) != 2 {
		t.Fatalf("wrong number of diagnostics; want two errors\n\n%s", diags.Err())
	}

	errStr := diags.Err().Error()
	if !(strings.Contains(errStr, "bananananananana") && strings.Contains(errStr, "bazaz")) {
		t.Fatalf("wrong errors %q; want something about each of our invalid connInfo keys", errStr)
	}
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"log"
	"sync"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/dag"
)

// ContextGraphWalker is the GraphWalker implementation used with the
// Context struct to walk and evaluate the graph.
type ContextGraphWalker struct {
	NullGraphWalker

	// Configurable values
	Context   *Context
	Operation walkOperation

	// Outputs, do not set these. Do not read these while the graph
	// is being walked.
	ValidationWarnings []string
	ValidationErrors   []error

	errorLock           sync.Mutex
	once                sync.Once
	contexts            map[string]*BuiltinEvalContext
	contextLock         sync.Mutex
	interpolaterVars    map[string]map[string]string
	interpolaterVarLock sync.Mutex
	providerCache       map[string]ResourceProvider
	providerConfigCache map[string]*ResourceConfig
	providerLock        sync.Mutex
	provisionerCache    map[string]ResourceProvisioner
	provisionerLock     sync.Mutex
}

func (w *ContextGraphWalker) EnterPath(path []string) EvalContext {
	w.once.Do(w.init)

	w.contextLock.Lock()
	defer w.contextLock.Unlock()

	// If we already have a context for this path cached, use that
	key := PathCacheKey(path)
	if ctx, ok := w.contexts[key]; ok {
		return ctx
	}

	// Setup the variables for this interpolater
	variables := make(map[string]string)
	if len(path) <= 1 {
		for k, v := range w.Context.variables {
			variables[k] = v
		}
	}
	w.interpolaterVarLock.Lock()
	if m, ok := w.interpolaterVars[key]; ok {
		for k, v := range m {
			variables[k] = v
		}
	}
	w.interpolaterVars[key] = variables
	w.interpolaterVarLock.Unlock()

	ctx := &BuiltinEvalContext{
		PathValue:           path,
		Hooks:               w.Context.hooks,
		InputValue:          w.Context.uiInput,
		Providers:           w.Context.providers,
		ProviderCache:       w.providerCache,
		ProviderConfigCache: w.providerConfigCache,
		ProviderInputConfig: w.Context.providerInputConfig,
		ProviderLock:        &w.providerLock,
		Provisioners:        w.Context.provisioners,
		ProvisionerCache:    w.provisionerCache,
		ProvisionerLock:     &w.provisionerLock,
		DiffValue:           w.Context.diff,
		DiffLock:            &w.Context.diffLock,
		StateValue:          w.Context.state,
		StateLock:           &w.Context.stateLock,
		Interpolater: &Interpolater{
			Operation: w.Operation,
			Module:    w.Context.module,
			State:     w.Context.state,
			StateLock: &w.Context.stateLock,
			Variables: variables,
		},
		InterpolaterVars:    w.interpolaterVars,
		InterpolaterVarLock: &w.interpolaterVarLock,
	}

	w.contexts[key] = ctx
	return ctx
}

func (w *ContextGraphWalker) EnterEvalTree(v dag.Vertex, n EvalNode) EvalNode {
	log.Printf("[TRACE] Entering eval tree: %s", dag.VertexName(v))

	// Acquire a lock on the semaphore
	w.Context.parallelSem.Acquire()

	// We want to filter the evaluation tree to only include operations
	// that belong in this operation.
	return EvalFilter(n, EvalNodeFilterOp(w.Operation))
}

func (w *ContextGraphWalker) ExitEvalTree(
	v dag.Vertex, output interface{}, err error) error {
	log.Printf("[TRACE] Exiting eval tree: %s", dag.VertexName(v))

	// Release the semaphore
	w.Context.parallelSem.Release()

	if err == nil {
		return nil
	}

	// Acquire the lock because anything is going to require a lock.
	w.errorLock.Lock()
	defer w.errorLock.Unlock()

	// Try to get a validation error out of it. If its not a validation
	// error, then just record the normal error.
	verr, ok := err.(*EvalValidateError)
	if !ok {
		return err
	}

	for _, msg := range verr.Warnings {
		w.ValidationWarnings = append(
			w.ValidationWarnings,
			fmt.Sprintf("%s: %s", dag.VertexName(v), msg))
	}
	for _, e := range verr.Errors {
		w.ValidationErrors = append(
			w.ValidationErrors,
			errwrap.Wrapf(fmt.Sprintf("%s: {{err}}", dag.VertexName(v)), e))
	}

	return nil
}

func (w *ContextGraphWalker) init() {
	w.contexts = make(map[string]*BuiltinEvalContext, 5)
	w.providerCache = make(map[string]ResourceProvider, 5)
	w.providerConfigCache = make(map[string]*ResourceConfig, 5)
	w.provisionerCache = make(map[string]ResourceProvisioner, 5)
	w.interpolaterVars = make(map[string]map[string]string, 5)
}
<commit_msg>core: log eval tree operations<commit_after>package terraform

import (
	"fmt"
	"log"
	"sync"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/dag"
)

// ContextGraphWalker is the GraphWalker implementation used with the
// Context struct to walk and evaluate the graph.
type ContextGraphWalker struct {
	NullGraphWalker

	// Configurable values
	Context   *Context
	Operation walkOperation

	// Outputs, do not set these. Do not read these while the graph
	// is being walked.
	ValidationWarnings []string
	ValidationErrors   []error

	errorLock           sync.Mutex
	once                sync.Once
	contexts            map[string]*BuiltinEvalContext
	contextLock         sync.Mutex
	interpolaterVars    map[string]map[string]string
	interpolaterVarLock sync.Mutex
	providerCache       map[string]ResourceProvider
	providerConfigCache map[string]*ResourceConfig
	providerLock        sync.Mutex
	provisionerCache    map[string]ResourceProvisioner
	provisionerLock     sync.Mutex
}

func (w *ContextGraphWalker) EnterPath(path []string) EvalContext {
	w.once.Do(w.init)

	w.contextLock.Lock()
	defer w.contextLock.Unlock()

	// If we already have a context for this path cached, use that
	key := PathCacheKey(path)
	if ctx, ok := w.contexts[key]; ok {
		return ctx
	}

	// Setup the variables for this interpolater
	variables := make(map[string]string)
	if len(path) <= 1 {
		for k, v := range w.Context.variables {
			variables[k] = v
		}
	}
	w.interpolaterVarLock.Lock()
	if m, ok := w.interpolaterVars[key]; ok {
		for k, v := range m {
			variables[k] = v
		}
	}
	w.interpolaterVars[key] = variables
	w.interpolaterVarLock.Unlock()

	ctx := &BuiltinEvalContext{
		PathValue:           path,
		Hooks:               w.Context.hooks,
		InputValue:          w.Context.uiInput,
		Providers:           w.Context.providers,
		ProviderCache:       w.providerCache,
		ProviderConfigCache: w.providerConfigCache,
		ProviderInputConfig: w.Context.providerInputConfig,
		ProviderLock:        &w.providerLock,
		Provisioners:        w.Context.provisioners,
		ProvisionerCache:    w.provisionerCache,
		ProvisionerLock:     &w.provisionerLock,
		DiffValue:           w.Context.diff,
		DiffLock:            &w.Context.diffLock,
		StateValue:          w.Context.state,
		StateLock:           &w.Context.stateLock,
		Interpolater: &Interpolater{
			Operation: w.Operation,
			Module:    w.Context.module,
			State:     w.Context.state,
			StateLock: &w.Context.stateLock,
			Variables: variables,
		},
		InterpolaterVars:    w.interpolaterVars,
		InterpolaterVarLock: &w.interpolaterVarLock,
	}

	w.contexts[key] = ctx
	return ctx
}

func (w *ContextGraphWalker) EnterEvalTree(v dag.Vertex, n EvalNode) EvalNode {
	log.Printf("[TRACE] [%s] Entering eval tree: %s",
		w.Operation, dag.VertexName(v))

	// Acquire a lock on the semaphore
	w.Context.parallelSem.Acquire()

	// We want to filter the evaluation tree to only include operations
	// that belong in this operation.
	return EvalFilter(n, EvalNodeFilterOp(w.Operation))
}

func (w *ContextGraphWalker) ExitEvalTree(
	v dag.Vertex, output interface{}, err error) error {
	log.Printf("[TRACE] [%s] Exiting eval tree: %s",
		w.Operation, dag.VertexName(v))

	// Release the semaphore
	w.Context.parallelSem.Release()

	if err == nil {
		return nil
	}

	// Acquire the lock because anything is going to require a lock.
	w.errorLock.Lock()
	defer w.errorLock.Unlock()

	// Try to get a validation error out of it. If its not a validation
	// error, then just record the normal error.
	verr, ok := err.(*EvalValidateError)
	if !ok {
		return err
	}

	for _, msg := range verr.Warnings {
		w.ValidationWarnings = append(
			w.ValidationWarnings,
			fmt.Sprintf("%s: %s", dag.VertexName(v), msg))
	}
	for _, e := range verr.Errors {
		w.ValidationErrors = append(
			w.ValidationErrors,
			errwrap.Wrapf(fmt.Sprintf("%s: {{err}}", dag.VertexName(v)), e))
	}

	return nil
}

func (w *ContextGraphWalker) init() {
	w.contexts = make(map[string]*BuiltinEvalContext, 5)
	w.providerCache = make(map[string]ResourceProvider, 5)
	w.providerConfigCache = make(map[string]*ResourceConfig, 5)
	w.provisionerCache = make(map[string]ResourceProvisioner, 5)
	w.interpolaterVars = make(map[string]map[string]string, 5)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/satori/go.uuid"
)

type somaNodeRequest struct {
	action string
	Node   proto.Node
	reply  chan somaResult
}

type somaNodeResult struct {
	ResultError error
	Node        proto.Node
}

func (a *somaNodeResult) SomaAppendError(r *somaResult, err error) {
	if err != nil {
		r.Nodes = append(r.Nodes, somaNodeResult{ResultError: err})
	}
}

func (a *somaNodeResult) SomaAppendResult(r *somaResult) {
	r.Nodes = append(r.Nodes, *a)
}

/* Read Access
 *
 */
type somaNodeReadHandler struct {
	input     chan somaNodeRequest
	shutdown  chan bool
	conn      *sql.DB
	list_stmt *sql.Stmt
	show_stmt *sql.Stmt
	conf_stmt *sql.Stmt
}

func (r *somaNodeReadHandler) run() {
	var err error

	r.list_stmt, err = r.conn.Prepare(`
SELECT node_id,
       node_name
FROM   soma.nodes
WHERE  node_online;`)
	if err != nil {
		log.Fatal("node/list: ", err)
	}

	r.show_stmt, err = r.conn.Prepare(`
SELECT node_id,
       node_asset_id,
       node_name,
       organizational_team_id,
       server_id,
       object_state,
       node_online,
       node_deleted
FROM   soma.nodes
WHERE  node_id = $1;`)
	if err != nil {
		log.Fatal("node/show: ", err)
	}

	r.conf_stmt, err = r.conn.Prepare(`
SELECT nodes.node_id,
       nodes.node_name,
       buckets.bucket_id,
       buckets.repository_id
FROM   soma.nodes
JOIN   soma.node_bucket_assignment
  ON   nodes.node_id = node_bucket_assignment.node_id
JOIN   soma.buckets
  ON   node_bucket_assignment.bucket_id = buckets.bucket_id
WHERE  nodes.node_id = $1;`)
	if err != nil {
		log.Fatal("node/get-config: ", err)
	}

runloop:
	for {
		select {
		case <-r.shutdown:
			break runloop
		case req := <-r.input:
			go func() {
				r.process(&req)
			}()
		}
	}
}

func (r *somaNodeReadHandler) process(q *somaNodeRequest) {
	var nodeId, nodeName, nodeTeam, nodeServer, nodeState, bucketId, repositoryId string
	var nodeAsset int
	var nodeOnline, nodeDeleted bool
	var rows *sql.Rows
	var err error
	result := somaResult{}

	switch q.action {
	case "list":
		log.Printf("R: node/list")
		rows, err = r.list_stmt.Query()
		defer rows.Close()
		if result.SetRequestError(err) {
			q.reply <- result
			return
		}

		for rows.Next() {
			err := rows.Scan(&nodeId, &nodeName)
			result.Append(err, &somaNodeResult{
				Node: proto.Node{
					Id:   nodeId,
					Name: nodeName,
				},
			})
		}
	case "show":
		log.Printf("R: node/show")
		err = r.show_stmt.QueryRow(q.Node.Id).Scan(
			&nodeId,
			&nodeAsset,
			&nodeName,
			&nodeTeam,
			&nodeServer,
			&nodeState,
			&nodeOnline,
			&nodeDeleted,
		)
		if err != nil {
			if err == sql.ErrNoRows {
				result.SetNotFound()
			} else {
				_ = result.SetRequestError(err)
			}
			q.reply <- result
			return
		}

		result.Append(err, &somaNodeResult{
			Node: proto.Node{
				Id:        nodeId,
				AssetId:   uint64(nodeAsset),
				Name:      nodeName,
				TeamId:    nodeTeam,
				ServerId:  nodeServer,
				State:     nodeState,
				IsOnline:  nodeOnline,
				IsDeleted: nodeDeleted,
			},
		})
	case "get_config":
		log.Printf("R: node/get_config")
		err = r.conf_stmt.QueryRow(q.Node.Id).Scan(
			&nodeId,
			&nodeName,
			&bucketId,
			&repositoryId,
		)
		if err != nil {
			if err == sql.ErrNoRows {
				result.SetNotFound()
			} else {
				_ = result.SetRequestError(err)
			}
			q.reply <- result
			return
		}

		result.Append(err, &somaNodeResult{
			Node: proto.Node{
				Id:   nodeId,
				Name: nodeName,
				Config: &proto.NodeConfig{
					RepositoryId: repositoryId,
					BucketId:     bucketId,
				},
			},
		})
	default:
		result.SetNotImplemented()
	}
	q.reply <- result
}

/* Write Access
 *
 */
type somaNodeWriteHandler struct {
	input    chan somaNodeRequest
	shutdown chan bool
	conn     *sql.DB
	add_stmt *sql.Stmt
	del_stmt *sql.Stmt
	prg_stmt *sql.Stmt
}

func (w *somaNodeWriteHandler) run() {
	var err error

	w.add_stmt, err = w.conn.Prepare(`
INSERT INTO soma.nodes (
	node_id,
	node_asset_id,
	node_name,
	organizational_team_id,
	server_id,
	object_state,
	node_online,
    node_deleted)
SELECT $1::uuid, $2::numeric, $3::varchar, $4, $5, $6, $7, $8
WHERE NOT EXISTS (
	SELECT node_id
	FROM   soma.nodes
	WHERE  node_id = $1::uuid
	OR     node_asset_id = $2::numeric
	OR     (node_name = $3::varchar AND node_online));`)
	if err != nil {
		log.Fatal("node/add: ", err)
	}
	defer w.add_stmt.Close()

	w.del_stmt, err = w.conn.Prepare(`
UPDATE soma.nodes
SET    node_deleted = 'yes'
WHERE  node_id = $1
AND    node_deleted = 'no';`)
	if err != nil {
		log.Fatal("node/delete: ", err)
	}
	defer w.del_stmt.Close()

	w.prg_stmt, err = w.conn.Prepare(`
DELETE FROM soma.nodes
WHERE       node_id = $1
AND         node_deleted;`)
	if err != nil {
		log.Fatal("node/purge: ", err)
	}
	defer w.prg_stmt.Close()

runloop:
	for {
		select {
		case <-w.shutdown:
			break runloop
		case req := <-w.input:
			w.process(&req)
		}
	}
}

func (w *somaNodeWriteHandler) process(q *somaNodeRequest) {
	var res sql.Result
	var err error
	result := somaResult{}

	switch q.action {
	case "add":
		log.Printf("R: node/add for %s", q.Node.Name)
		id := uuid.NewV4()
		if q.Node.ServerId == "" {
			q.Node.ServerId = "00000000-0000-0000-0000-000000000000"
		}
		res, err = w.add_stmt.Exec(
			id.String(),
			q.Node.AssetId,
			q.Node.Name,
			q.Node.TeamId,
			q.Node.ServerId,
			q.Node.State,
			q.Node.IsOnline,
			false,
		)
		q.Node.Id = id.String()
	case "delete":
		log.Printf("R: node/delete for %s", q.Node.Id)
		res, err = w.del_stmt.Exec(
			q.Node.Id,
		)
		// TODO trigger undeployment
	case "purge":
		log.Printf("R: node/purge for %s", q.Node.Id)
		res, err = w.prg_stmt.Exec(
			q.Node.Id,
		)
	default:
		log.Printf("R: unimplemented node/%s", q.action)
		result.SetNotImplemented()
		q.reply <- result
		return
	}
	if result.SetRequestError(err) {
		q.reply <- result
		return
	}

	rowCnt, _ := res.RowsAffected()
	switch {
	case rowCnt == 0:
		result.Append(errors.New("No rows affected"), &somaNodeResult{})
	case rowCnt > 1:
		result.Append(fmt.Errorf("Too many rows affected: %d", rowCnt),
			&somaNodeResult{})
	default:
		result.Append(nil, &somaNodeResult{
			Node: q.Node,
		})
	}
	q.reply <- result
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Create/Delete nodes in parallel<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/satori/go.uuid"
)

type somaNodeRequest struct {
	action string
	Node   proto.Node
	reply  chan somaResult
}

type somaNodeResult struct {
	ResultError error
	Node        proto.Node
}

func (a *somaNodeResult) SomaAppendError(r *somaResult, err error) {
	if err != nil {
		r.Nodes = append(r.Nodes, somaNodeResult{ResultError: err})
	}
}

func (a *somaNodeResult) SomaAppendResult(r *somaResult) {
	r.Nodes = append(r.Nodes, *a)
}

/* Read Access
 *
 */
type somaNodeReadHandler struct {
	input     chan somaNodeRequest
	shutdown  chan bool
	conn      *sql.DB
	list_stmt *sql.Stmt
	show_stmt *sql.Stmt
	conf_stmt *sql.Stmt
}

func (r *somaNodeReadHandler) run() {
	var err error

	r.list_stmt, err = r.conn.Prepare(`
SELECT node_id,
       node_name
FROM   soma.nodes
WHERE  node_online;`)
	if err != nil {
		log.Fatal("node/list: ", err)
	}

	r.show_stmt, err = r.conn.Prepare(`
SELECT node_id,
       node_asset_id,
       node_name,
       organizational_team_id,
       server_id,
       object_state,
       node_online,
       node_deleted
FROM   soma.nodes
WHERE  node_id = $1;`)
	if err != nil {
		log.Fatal("node/show: ", err)
	}

	r.conf_stmt, err = r.conn.Prepare(`
SELECT nodes.node_id,
       nodes.node_name,
       buckets.bucket_id,
       buckets.repository_id
FROM   soma.nodes
JOIN   soma.node_bucket_assignment
  ON   nodes.node_id = node_bucket_assignment.node_id
JOIN   soma.buckets
  ON   node_bucket_assignment.bucket_id = buckets.bucket_id
WHERE  nodes.node_id = $1;`)
	if err != nil {
		log.Fatal("node/get-config: ", err)
	}

runloop:
	for {
		select {
		case <-r.shutdown:
			break runloop
		case req := <-r.input:
			go func() {
				r.process(&req)
			}()
		}
	}
}

func (r *somaNodeReadHandler) process(q *somaNodeRequest) {
	var nodeId, nodeName, nodeTeam, nodeServer, nodeState, bucketId, repositoryId string
	var nodeAsset int
	var nodeOnline, nodeDeleted bool
	var rows *sql.Rows
	var err error
	result := somaResult{}

	switch q.action {
	case "list":
		log.Printf("R: node/list")
		rows, err = r.list_stmt.Query()
		defer rows.Close()
		if result.SetRequestError(err) {
			q.reply <- result
			return
		}

		for rows.Next() {
			err := rows.Scan(&nodeId, &nodeName)
			result.Append(err, &somaNodeResult{
				Node: proto.Node{
					Id:   nodeId,
					Name: nodeName,
				},
			})
		}
	case "show":
		log.Printf("R: node/show")
		err = r.show_stmt.QueryRow(q.Node.Id).Scan(
			&nodeId,
			&nodeAsset,
			&nodeName,
			&nodeTeam,
			&nodeServer,
			&nodeState,
			&nodeOnline,
			&nodeDeleted,
		)
		if err != nil {
			if err == sql.ErrNoRows {
				result.SetNotFound()
			} else {
				_ = result.SetRequestError(err)
			}
			q.reply <- result
			return
		}

		result.Append(err, &somaNodeResult{
			Node: proto.Node{
				Id:        nodeId,
				AssetId:   uint64(nodeAsset),
				Name:      nodeName,
				TeamId:    nodeTeam,
				ServerId:  nodeServer,
				State:     nodeState,
				IsOnline:  nodeOnline,
				IsDeleted: nodeDeleted,
			},
		})
	case "get_config":
		log.Printf("R: node/get_config")
		err = r.conf_stmt.QueryRow(q.Node.Id).Scan(
			&nodeId,
			&nodeName,
			&bucketId,
			&repositoryId,
		)
		if err != nil {
			if err == sql.ErrNoRows {
				result.SetNotFound()
			} else {
				_ = result.SetRequestError(err)
			}
			q.reply <- result
			return
		}

		result.Append(err, &somaNodeResult{
			Node: proto.Node{
				Id:   nodeId,
				Name: nodeName,
				Config: &proto.NodeConfig{
					RepositoryId: repositoryId,
					BucketId:     bucketId,
				},
			},
		})
	default:
		result.SetNotImplemented()
	}
	q.reply <- result
}

/* Write Access
 *
 */
type somaNodeWriteHandler struct {
	input    chan somaNodeRequest
	shutdown chan bool
	conn     *sql.DB
	add_stmt *sql.Stmt
	del_stmt *sql.Stmt
	prg_stmt *sql.Stmt
}

func (w *somaNodeWriteHandler) run() {
	var err error

	w.add_stmt, err = w.conn.Prepare(`
INSERT INTO soma.nodes (
	node_id,
	node_asset_id,
	node_name,
	organizational_team_id,
	server_id,
	object_state,
	node_online,
    node_deleted)
SELECT $1::uuid, $2::numeric, $3::varchar, $4, $5, $6, $7, $8
WHERE NOT EXISTS (
	SELECT node_id
	FROM   soma.nodes
	WHERE  node_id = $1::uuid
	OR     node_asset_id = $2::numeric
	OR     (node_name = $3::varchar AND node_online));`)
	if err != nil {
		log.Fatal("node/add: ", err)
	}
	defer w.add_stmt.Close()

	w.del_stmt, err = w.conn.Prepare(`
UPDATE soma.nodes
SET    node_deleted = 'yes'
WHERE  node_id = $1
AND    node_deleted = 'no';`)
	if err != nil {
		log.Fatal("node/delete: ", err)
	}
	defer w.del_stmt.Close()

	w.prg_stmt, err = w.conn.Prepare(`
DELETE FROM soma.nodes
WHERE       node_id = $1
AND         node_deleted;`)
	if err != nil {
		log.Fatal("node/purge: ", err)
	}
	defer w.prg_stmt.Close()

runloop:
	for {
		select {
		case <-w.shutdown:
			break runloop
		case req := <-w.input:
			go func() {
				w.process(&req)
			}()
		}
	}
}

func (w *somaNodeWriteHandler) process(q *somaNodeRequest) {
	var res sql.Result
	var err error
	result := somaResult{}

	switch q.action {
	case "add":
		log.Printf("R: node/add for %s", q.Node.Name)
		id := uuid.NewV4()
		if q.Node.ServerId == "" {
			q.Node.ServerId = "00000000-0000-0000-0000-000000000000"
		}
		res, err = w.add_stmt.Exec(
			id.String(),
			q.Node.AssetId,
			q.Node.Name,
			q.Node.TeamId,
			q.Node.ServerId,
			q.Node.State,
			q.Node.IsOnline,
			false,
		)
		q.Node.Id = id.String()
	case "delete":
		log.Printf("R: node/delete for %s", q.Node.Id)
		res, err = w.del_stmt.Exec(
			q.Node.Id,
		)
		// TODO trigger undeployment
	case "purge":
		log.Printf("R: node/purge for %s", q.Node.Id)
		res, err = w.prg_stmt.Exec(
			q.Node.Id,
		)
	default:
		log.Printf("R: unimplemented node/%s", q.action)
		result.SetNotImplemented()
		q.reply <- result
		return
	}
	if result.SetRequestError(err) {
		q.reply <- result
		return
	}

	rowCnt, _ := res.RowsAffected()
	switch {
	case rowCnt == 0:
		result.Append(errors.New("No rows affected"), &somaNodeResult{})
	case rowCnt > 1:
		result.Append(fmt.Errorf("Too many rows affected: %d", rowCnt),
			&somaNodeResult{})
	default:
		result.Append(nil, &somaNodeResult{
			Node: q.Node,
		})
	}
	q.reply <- result
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP.
// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished.
// It does not currently pre-empt stale generations, but will block until they are complete.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	fsnotify "gopkg.in/fsnotify.v1"
)

var (
	port = flag.Int("port", 8000, "Port on which to serve HTTP")

	mu      sync.Mutex   // Prevent multiple updates in parallel.
	toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero.

	wgMu sync.Mutex     // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel.
	wg   sync.WaitGroup // Indicates how many updates are pending.
)

func main() {
	flag.Parse()

	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatalf("Error making watcher: %v", err)
	}

	dir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting wd: %v", err)
	}
	for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) {
		if dir == "/" {
			log.Fatalf("Could not find git root")
		}
	}

	walker := makeWalker(dir, w)
	paths := []string{"api", "changelogs", "event-schemas", "scripts",
		"specification", "templating"}

	for _, p := range paths {
		filepath.Walk(path.Join(dir, p), walker)
	}

	wg.Add(1)
	populateOnce(dir)

	ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes
	go doPopulate(ch, dir)

	go watchFS(ch, w)
	fmt.Printf("Listening on port %d\n", *port)
	http.HandleFunc("/", serve)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))

}

func watchFS(ch chan struct{}, w *fsnotify.Watcher) {
	for {
		select {
		case e := <-w.Events:
			if filter(e) {
				fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name)
				ch <- struct{}{}
			}
		}
	}
}

func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
	return func(path string, _ os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Error walking: %v", err)
		}

		rel, err := filepath.Rel(base, path)
		if err != nil {
			log.Fatalf("Failed to get relative path of %s: %v", path, err)
		}

		// skip a few things that we know don't form part of the spec
		if rel == "api/node_modules" ||
			rel == "scripts/gen" ||
			rel == "scripts/tmp" {
			return filepath.SkipDir
		}

		// log.Printf("Adding watch on %s", path)
		if err := w.Add(path); err != nil {
			log.Fatalf("Failed to add watch on %s: %v", path, err)
		}
		return nil
	}
}

// Return true if event should trigger re-population
func filter(e fsnotify.Event) bool {
	// vim is *really* noisy about how it writes files
	if e.Op != fsnotify.Write {
		return false
	}

	// Avoid some temp files that vim writes
	if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") {
		return false
	}

	return true
}

func serve(w http.ResponseWriter, req *http.Request) {
	wgMu.Lock()
	wg.Wait()
	wgMu.Unlock()

	file := req.URL.Path
	if file[0] == '/' {
		file = file[1:]
	}
	if file == "" {
		file = "index.html"
	}
	m := toServe.Load().(bytesOrErr)
	if m.err != nil {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte(m.err.Error()))
		return
	}
	b, ok := m.bytes[file]
	if ok {
		w.Header().Set("Content-Type", "text/html")
		w.Write([]byte(b))
		return
	}
	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(404)
	w.Write([]byte("Not found"))
}

func populateOnce(dir string) {
	defer wg.Done()
	mu.Lock()
	defer mu.Unlock()
	cmd := exec.Command("python", "gendoc.py")
	cmd.Dir = path.Join(dir, "scripts")
	var b bytes.Buffer
	cmd.Stderr = &b
	err := cmd.Run()
	if err != nil {
		toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())})
		return
	}

	files := make(map[string][]byte)
	base := path.Join(dir, "scripts", "gen")
	walker := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		rel, err := filepath.Rel(base, path)
		if err != nil {
			return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
		}

		bytes, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		files[rel] = bytes
		return nil
	}

	err = filepath.Walk(base, walker)
	if err != nil {
		toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)})
		return
	}
	toServe.Store(bytesOrErr{files, nil})
}

func doPopulate(ch chan struct{}, dir string) {
	var pending int
	for {
		select {
		case <-ch:
			if pending == 0 {
				wgMu.Lock()
				wg.Add(1)
				wgMu.Unlock()
			}
			pending++
		case <-time.After(10 * time.Millisecond):
			if pending > 0 {
				pending = 0
				populateOnce(dir)
			}
		}
	}
}

func exists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
}

type bytesOrErr struct {
	bytes map[string][]byte // filename -> contents
	err   error
}
<commit_msg>Serve the API docs with continuserv<commit_after>// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP.
// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished.
// It does not currently pre-empt stale generations, but will block until they are complete.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	fsnotify "gopkg.in/fsnotify.v1"
)

var (
	port = flag.Int("port", 8000, "Port on which to serve HTTP")

	mu      sync.Mutex   // Prevent multiple updates in parallel.
	toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero.

	wgMu sync.Mutex     // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel.
	wg   sync.WaitGroup // Indicates how many updates are pending.
)

func main() {
	flag.Parse()

	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatalf("Error making watcher: %v", err)
	}

	dir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting wd: %v", err)
	}
	for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) {
		if dir == "/" {
			log.Fatalf("Could not find git root")
		}
	}

	walker := makeWalker(dir, w)
	paths := []string{"api", "changelogs", "event-schemas", "scripts",
		"specification", "templating"}

	for _, p := range paths {
		filepath.Walk(path.Join(dir, p), walker)
	}

	wg.Add(1)
	populateOnce(dir)

	ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes
	go doPopulate(ch, dir)

	go watchFS(ch, w)
	fmt.Printf("Listening on port %d\n", *port)
	http.HandleFunc("/", serve)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))

}

func watchFS(ch chan struct{}, w *fsnotify.Watcher) {
	for {
		select {
		case e := <-w.Events:
			if filter(e) {
				fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name)
				ch <- struct{}{}
			}
		}
	}
}

func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
	return func(path string, i os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Error walking: %v", err)
		}
		if !i.IsDir() {
			// we set watches on directories, not files
			return nil
		}

		rel, err := filepath.Rel(base, path)
		if err != nil {
			log.Fatalf("Failed to get relative path of %s: %v", path, err)
		}

		// skip a few things that we know don't form part of the spec
		if rel == "api/node_modules" ||
			rel == "scripts/gen" ||
			rel == "scripts/tmp" {
			return filepath.SkipDir
		}

		// log.Printf("Adding watch on %s", path)
		if err := w.Add(path); err != nil {
			log.Fatalf("Failed to add watch on %s: %v", path, err)
		}
		return nil
	}
}

// Return true if event should trigger re-population
func filter(e fsnotify.Event) bool {
	// vim is *really* noisy about how it writes files
	if e.Op != fsnotify.Write {
		return false
	}

	// Avoid some temp files that vim writes
	if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") {
		return false
	}

	return true
}

func serve(w http.ResponseWriter, req *http.Request) {
	wgMu.Lock()
	wg.Wait()
	wgMu.Unlock()

	m := toServe.Load().(bytesOrErr)
	if m.err != nil {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte(m.err.Error()))
		return
	}

	ok := true
	var b []byte

	file := req.URL.Path
	if file[0] == '/' {
		file = file[1:]
	}
	b, ok = m.bytes[file]

	if ok && file == "api-docs.json" {
		w.Header().Set("Access-Control-Allow-Origin", "*")
	}

	if ok {
		w.Header().Set("Content-Type", "text/html")
		w.Write([]byte(b))
		return
	}
	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(404)
	w.Write([]byte("Not found"))
}

func generate(dir string) (map[string][]byte, error) {
	cmd := exec.Command("python", "gendoc.py")
	cmd.Dir = path.Join(dir, "scripts")
	var b bytes.Buffer
	cmd.Stderr = &b
	err := cmd.Run()
	if err != nil {
		return nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
	}

	// cheekily dump the swagger docs into the gen directory so that it is
	// easy to serve
	cmd = exec.Command("python", "dump-swagger.py", "gen/api-docs.json")
	cmd.Dir = path.Join(dir, "scripts")
	cmd.Stderr = &b
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("error generating api docs: %v\nOutput from dump-swagger:\n%v", err, b.String())
	}

	files := make(map[string][]byte)
	base := path.Join(dir, "scripts", "gen")
	walker := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		rel, err := filepath.Rel(base, path)
		if err != nil {
			return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
		}

		bytes, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		files[rel] = bytes
		return nil
	}

	if err := filepath.Walk(base, walker); err != nil {
		return nil, fmt.Errorf("error reading spec: %v", err)
	}

	// load the special index
	indexpath := path.Join(dir, "scripts", "continuserv", "index.html")
	bytes, err := ioutil.ReadFile(indexpath)
	if err != nil {
		return nil, fmt.Errorf("error reading index: %v", err)
	}
	files[""] = bytes

	return files, nil
}

func populateOnce(dir string) {
	defer wg.Done()
	mu.Lock()
	defer mu.Unlock()

	files, err := generate(dir)
	toServe.Store(bytesOrErr{files, err})
}

func doPopulate(ch chan struct{}, dir string) {
	var pending int
	for {
		select {
		case <-ch:
			if pending == 0 {
				wgMu.Lock()
				wg.Add(1)
				wgMu.Unlock()
			}
			pending++
		case <-time.After(10 * time.Millisecond):
			if pending > 0 {
				pending = 0
				populateOnce(dir)
			}
		}
	}
}

func exists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
}

type bytesOrErr struct {
	bytes map[string][]byte // filename -> contents
	err   error
}
<|endoftext|>


<commit_before>// Copyright (C) 2014-2015 Thomas Lokshall
// Use of this source code is governed by the MIT license.
// See LICENSE.md for details.

// Package json implements a way to easily read and write JSON configuration files.
//
// To read a configuration file, simply call the LoadFile method on a new Configuration object:
//		import cfg "gopkg.in/zhevron/gov1/json"
//		if err := cfg.LoadFile("/path/to/file.json"); err != nil {
//			// Failed to load the configuration file
//		}
//
// To save the current configuration to a file, call the SaveFile method:
//		if err := cfg.SaveFile("/path/to/file.json"); err != nil {
//			// Failed to save the configuration file
//		}
//
// You can easily access values with the Get method or any of the Get_Type_ methods:
//		host := cfg.GetString("host", "localhost")
//		port := cfg.GetInt("port", 10000)
//
// You can just as easily set a value using the Set method:
//		cfg.Set("host", host)
//		cfg.Set("port", port)
//
// Nested objects are accessed by adding a dot (.) between the value names.
// If you have a JSON file that looks like:
//		{
//			"some": {
//				"value": 1
//			}
//		}
// The value can be accessed by calling the Get method:
//		val := cfg.GetInt("some.value", 1)
package json

import (
	"encoding/json"
	"io/ioutil"
	"reflect"
	"strings"
)

// stringMap is a shorthand type for the map that holds the variables.
type stringMap map[string]interface{}

// vars holds the variables currently loaded into the package.
var vars stringMap

// Initialize the vars map on init.
func init() {
	vars = make(stringMap)
}

// Load parses the JSON in data to it's internal map.
func Load(data string) error {
	m := make(stringMap)
	if err := json.Unmarshal([]byte(data), &m); err != nil {
		return err
	}
	vars = loadMap(m)
	return nil
}

// loadMap parses a loaded map structure and adds it to the current configuration.
func loadMap(m map[string]interface{}) stringMap {
	res := make(stringMap)
	for k, v := range m {
		if reflect.TypeOf(v).Kind() == reflect.Map {
			res[k] = loadMap(v.(map[string]interface{}))
		} else {
			res[k] = v
		}
	}
	return res
}

// Save returns the current configuration in JSON format.
func Save() (string, error) {
	b, err := json.Marshal(&vars)
	return string(b), err
}

// LoadFile reads a specified file into memory and parses it using Load().
func LoadFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	jsonData := string(b[:])
	return Load(jsonData)
}

// SaveFile writes the output from Save() to the specified file.
func SaveFile(path string) error {
	data, err := Save()
	if err == nil {
		err = ioutil.WriteFile(path, []byte(data), 0644)
	}
	return err
}

// Set sets a given value in the configuration to an arbitrary type.
func Set(key string, val interface{}) {
	vars[key] = val
}

// Remove deletes the value with the given key from the configuration.
func Remove(key string) {
	delete(vars, key)
}

// Get returns the value of the given key from the current configuration or the value of def if not found.
// The value is returned as an interface{}.
func Get(key string, def interface{}) interface{} {
	return getFromMap(vars, key, def)
}

// getFromMap returns the value of the given key from the specified map or the value of def if not found.
// The value is returned as an interface{}.
// If the key contains a dot (.), the function will be called recursively.
func getFromMap(m stringMap, key string, def interface{}) interface{} {
	if n := strings.Index(key, "."); n != -1 {
		return getFromMap(m[key[:n]].(stringMap), key[n+1:], def)
	}
	if v, found := m[key]; found {
		return v
	}
	return def
}

// GetInt returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int.
func GetInt(key string, def int) int {
	return Get(key, def).(int)
}

// GetInt8 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int8.
func GetInt8(key string, def int8) int8 {
	return int8(Get(key, def).(int))
}

// GetInt16 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int16.
func GetInt16(key string, def int16) int16 {
	return int16(Get(key, def).(int))
}

// GetInt32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int32.
func GetInt32(key string, def int32) int32 {
	return int32(Get(key, def).(int))
}

// GetInt64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int64.
func GetInt64(key string, def int64) int64 {
	return int64(Get(key, def).(int))
}

// GetUint returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint.
func GetUint(key string, def uint) uint {
	return uint(Get(key, def).(int))
}

// GetUint8 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint8.
func GetUint8(key string, def uint8) uint8 {
	return uint8(Get(key, def).(int))
}

// GetUint16 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint16.
func GetUint16(key string, def uint16) uint16 {
	return uint16(Get(key, def).(int))
}

// GetUint32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint32.
func GetUint32(key string, def uint32) uint32 {
	return uint32(Get(key, def).(int))
}

// GetUint64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint64.
func GetUint64(key string, def uint64) uint64 {
	return uint64(Get(key, def).(int))
}

// GetFloat32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a float32.
func GetFloat32(key string, def float32) float32 {
	return float32(Get(key, def).(float64))
}

// GetFloat64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a float64.
func GetFloat64(key string, def float64) float64 {
	return Get(key, def).(float64)
}

// GetBool returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a bool.
func GetBool(key string, def bool) bool {
	return Get(key, def).(bool)
}

// GetString returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a string.
func GetString(key string, def string) string {
	return Get(key, def).(string)
}
<commit_msg>Added mutex to JSON package<commit_after>// Copyright (C) 2014-2015 Thomas Lokshall
// Use of this source code is governed by the MIT license.
// See LICENSE.md for details.

// Package json implements a way to easily read and write JSON configuration files.
//
// To read a configuration file, simply call the LoadFile method on a new Configuration object:
//		import cfg "gopkg.in/zhevron/gov1/json"
//		if err := cfg.LoadFile("/path/to/file.json"); err != nil {
//			// Failed to load the configuration file
//		}
//
// To save the current configuration to a file, call the SaveFile method:
//		if err := cfg.SaveFile("/path/to/file.json"); err != nil {
//			// Failed to save the configuration file
//		}
//
// You can easily access values with the Get method or any of the Get_Type_ methods:
//		host := cfg.GetString("host", "localhost")
//		port := cfg.GetInt("port", 10000)
//
// You can just as easily set a value using the Set method:
//		cfg.Set("host", host)
//		cfg.Set("port", port)
//
// Nested objects are accessed by adding a dot (.) between the value names.
// If you have a JSON file that looks like:
//		{
//			"some": {
//				"value": 1
//			}
//		}
// The value can be accessed by calling the Get method:
//		val := cfg.GetInt("some.value", 1)
package json

import (
	"encoding/json"
	"io/ioutil"
	"reflect"
	"strings"
	"sync"
)

// stringMap is a shorthand type for the map that holds the variables.
type stringMap map[string]interface{}

// vars holds the variables currently loaded into the package.
var vars stringMap

// mutex holds the sync.Mutex used to control concurrent reads/writes.
var mutex *sync.Mutex

// Initialize the vars map on init.
func init() {
	vars = make(stringMap)
	mutex = &sync.Mutex{}
}

// Load parses the JSON in data to it's internal map.
func Load(data string) error {
	m := make(stringMap)
	if err := json.Unmarshal([]byte(data), &m); err != nil {
		return err
	}
	mutex.Lock()
	vars = loadMap(m)
	mutex.Unlock()
	return nil
}

// loadMap parses a loaded map structure and adds it to the current configuration.
func loadMap(m map[string]interface{}) stringMap {
	res := make(stringMap)
	for k, v := range m {
		if reflect.TypeOf(v).Kind() == reflect.Map {
			res[k] = loadMap(v.(map[string]interface{}))
		} else {
			res[k] = v
		}
	}
	return res
}

// Save returns the current configuration in JSON format.
func Save() (string, error) {
	mutex.Lock()
	b, err := json.Marshal(&vars)
	mutex.Unlock()
	return string(b), err
}

// LoadFile reads a specified file into memory and parses it using Load().
func LoadFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	jsonData := string(b[:])
	return Load(jsonData)
}

// SaveFile writes the output from Save() to the specified file.
func SaveFile(path string) error {
	data, err := Save()
	if err == nil {
		err = ioutil.WriteFile(path, []byte(data), 0644)
	}
	return err
}

// Set sets a given value in the configuration to an arbitrary type.
func Set(key string, val interface{}) {
	mutex.Lock()
	vars[key] = val
	mutex.Unlock()
}

// Remove deletes the value with the given key from the configuration.
func Remove(key string) {
	mutex.Lock()
	delete(vars, key)
	mutex.Unlock()
}

// Get returns the value of the given key from the current configuration or the value of def if not found.
// The value is returned as an interface{}.
func Get(key string, def interface{}) interface{} {
	mutex.Lock()
	v := getFromMap(vars, key, def)
	mutex.Unlock()
	return v
}

// getFromMap returns the value of the given key from the specified map or the value of def if not found.
// The value is returned as an interface{}.
// If the key contains a dot (.), the function will be called recursively.
func getFromMap(m stringMap, key string, def interface{}) interface{} {
	if n := strings.Index(key, "."); n != -1 {
		return getFromMap(m[key[:n]].(stringMap), key[n+1:], def)
	}
	if v, found := m[key]; found {
		return v
	}
	return def
}

// GetInt returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int.
func GetInt(key string, def int) int {
	return Get(key, def).(int)
}

// GetInt8 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int8.
func GetInt8(key string, def int8) int8 {
	return int8(Get(key, def).(int))
}

// GetInt16 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int16.
func GetInt16(key string, def int16) int16 {
	return int16(Get(key, def).(int))
}

// GetInt32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int32.
func GetInt32(key string, def int32) int32 {
	return int32(Get(key, def).(int))
}

// GetInt64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an int64.
func GetInt64(key string, def int64) int64 {
	return int64(Get(key, def).(int))
}

// GetUint returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint.
func GetUint(key string, def uint) uint {
	return uint(Get(key, def).(int))
}

// GetUint8 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint8.
func GetUint8(key string, def uint8) uint8 {
	return uint8(Get(key, def).(int))
}

// GetUint16 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint16.
func GetUint16(key string, def uint16) uint16 {
	return uint16(Get(key, def).(int))
}

// GetUint32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint32.
func GetUint32(key string, def uint32) uint32 {
	return uint32(Get(key, def).(int))
}

// GetUint64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as an uint64.
func GetUint64(key string, def uint64) uint64 {
	return uint64(Get(key, def).(int))
}

// GetFloat32 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a float32.
func GetFloat32(key string, def float32) float32 {
	return float32(Get(key, def).(float64))
}

// GetFloat64 returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a float64.
func GetFloat64(key string, def float64) float64 {
	return Get(key, def).(float64)
}

// GetBool returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a bool.
func GetBool(key string, def bool) bool {
	return Get(key, def).(bool)
}

// GetString returns the value of the given key from the current configuration or the value of def if not found.
// The value is returns as a string.
func GetString(key string, def string) string {
	return Get(key, def).(string)
}
<|endoftext|>