<commit_before>package list_test

import (
	"fmt"
	"net"
	"os"
	"testing"
	"time"

	"github.com/MasterOfBinary/redistypes/internal"
	"github.com/MasterOfBinary/redistypes/internal/test"
	"github.com/MasterOfBinary/redistypes/list"
	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
)

var conn redis.Conn

const (
	forwardSlice = false
	reverseSlice = true
)

type scenarioStruct struct {
	name    string
	add     []interface{}
	wantErr bool
}

func ExampleNewRedisList() {
	netConn, _ := net.Dial("tcp", internal.GetHostAndPort())

	conn := redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	l := list.NewRedisList(conn, test.RandomKey())

	values, _ := l.Range(0, -1)
	fmt.Println("Count:", len(values))
	// Output: Count: 0
}

func ExampleRedisList_Range() {
	netConn, _ := net.Dial("tcp", internal.GetHostAndPort())

	conn := redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	l := list.NewRedisList(conn, test.RandomKey())

	_, _ = l.RightPush("hello", "world", "how", "are", "you", "today")

	// Get the entire range
	values, _ := redis.Strings(l.Range(0, -1))
	fmt.Println(values)

	// Starting from the middle
	values, _ = redis.Strings(l.Range(2, -1))
	fmt.Println(values)

	// From beginning to middle
	values, _ = redis.Strings(l.Range(0, 1))
	fmt.Println(values)

	// Output:
	// [hello world how are you today]
	// [how are you today]
	// [hello world]
}

func verifySlice(t *testing.T, l list.List, wantCount int, scenarios []scenarioStruct, reverse bool) {
	got, err := l.Range(0, -1)
	assert.Nil(t, err)
	assert.EqualValues(t, wantCount, len(got))
	assert.Len(t, got, wantCount)
	i := 0
	for _, scenario := range scenarios {
		if !scenario.wantErr {
			for _, item := range scenario.add {
				index := i
				if reverse == reverseSlice {
					index = len(got) - i - 1
				}
				test.AssertEqual(t, item, got[index])
				i++
			}
		}
	}
}

func TestRedisHyperLogLog_Name(t *testing.T) {
	name := test.RandomKey()
	l := list.NewRedisList(conn, name)
	assert.Equal(t, name, l.Name())
}

func TestRedisList_LeftPop(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		item, err := l.LeftPop()
		assert.Nil(t, err)
		assert.Nil(t, item)
	})

	t.Run("list with one item", func(t *testing.T) {
		_, _ = l.LeftPush("abc")
		item, err := redis.String(l.LeftPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "abc", item)
	})

	t.Run("list with multiple items", func(t *testing.T) {
		_, _ = l.LeftPush("def", "ghi")
		item, err := redis.String(l.LeftPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "ghi", item)
	})
}

func TestRedisList_LeftPush(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	scenarios := []scenarioStruct{
		{
			name: "non-existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integers",
			add:  test.IntsToInterfaceSlice(1, 2),
		},
		{
			name: "add a string",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name:    "no items",
			add:     nil,
			wantErr: true,
		},
	}

	wantCount := 0

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.LeftPush(scenario.add...)
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, reverseSlice)
	})
}

func TestRedisList_LeftPushX(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		count, err := l.LeftPushX("abc")
		assert.Nil(t, err)
		assert.EqualValues(t, 0, count)
	})

	_, _ = l.RightPush("abc")

	scenarios := []scenarioStruct{
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integer",
			add:  test.IntsToInterfaceSlice(1),
		},
	}

	wantCount := 1

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.LeftPushX(scenario.add[0])
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	// Pop the one added by RightPush
	_, _ = l.RightPop()
	wantCount--

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, reverseSlice)
	})
}

func TestRedisList_Range(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		items, err := l.Range(0, -1)
		assert.Nil(t, err)
		assert.Empty(t, items)
	})

	added := test.StringsToInterfaceSlice("abc", "def", "ghi")
	added = append(added, 5)
	added = append(added, "jkl")
	added = append(added, 10)

	_, _ = l.RightPush(added...)

	t.Run("full range", func(t *testing.T) {
		items, err := l.Range(0, -1)
		assert.Nil(t, err)
		assert.Len(t, items, len(added))
		for i, item := range items {
			test.AssertEqual(t, added[i], item)
		}
	})

	t.Run("from start to middle", func(t *testing.T) {
		items, err := l.Range(0, 2)
		assert.Nil(t, err)
		assert.Len(t, items, 3)
		for i, item := range items {
			test.AssertEqual(t, added[i], item)
		}
	})

	t.Run("from middle to end", func(t *testing.T) {
		items, err := l.Range(3, -1)
		assert.Nil(t, err)
		assert.Len(t, items, 3)
		for i, item := range items {
			test.AssertEqual(t, added[i+3], item)
		}
	})
}

func TestRedisList_RightPop(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		item, err := l.RightPop()
		assert.Nil(t, err)
		assert.Nil(t, item)
	})

	t.Run("list with one item", func(t *testing.T) {
		_, _ = l.RightPush("abc")
		item, err := redis.String(l.RightPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "abc", item)
	})

	t.Run("list with multiple items", func(t *testing.T) {
		_, _ = l.RightPush("def", "ghi")
		item, err := redis.String(l.RightPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "ghi", item)
	})
}

func TestRedisList_RightPush(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	scenarios := []scenarioStruct{
		{
			name: "non-existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integers",
			add:  test.IntsToInterfaceSlice(1, 2),
		},
		{
			name: "add a string",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name:    "no items",
			add:     nil,
			wantErr: true,
		},
	}

	wantCount := 0

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.RightPush(scenario.add...)
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, forwardSlice)
	})
}

func TestRedisList_RightPushX(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		count, err := l.RightPushX("abc")
		assert.Nil(t, err)
		assert.EqualValues(t, 0, count)
	})

	_, _ = l.RightPush("abc")

	scenarios := []scenarioStruct{
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integer",
			add:  test.IntsToInterfaceSlice(1),
		},
	}

	wantCount := 1

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.RightPushX(scenario.add[0])
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	// Pop the one added by RightPush
	_, _ = l.LeftPop()
	wantCount--

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, forwardSlice)
	})
}

func TestMain(m *testing.M) {
	netConn, err := net.Dial("tcp", internal.GetHostAndPort())
	if err != nil {
		fmt.Printf("Error opening net connection, err: %v", err)
		os.Exit(1)
	}

	conn = redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	os.Exit(m.Run())
}
<commit_msg>Change list example to refer to interface<commit_after>package list_test

import (
	"fmt"
	"net"
	"os"
	"testing"
	"time"

	"github.com/MasterOfBinary/redistypes/internal"
	"github.com/MasterOfBinary/redistypes/internal/test"
	"github.com/MasterOfBinary/redistypes/list"
	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
)

var conn redis.Conn

const (
	forwardSlice = false
	reverseSlice = true
)

type scenarioStruct struct {
	name    string
	add     []interface{}
	wantErr bool
}

func ExampleNewRedisList() {
	netConn, _ := net.Dial("tcp", internal.GetHostAndPort())

	conn := redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	l := list.NewRedisList(conn, test.RandomKey())

	values, _ := l.Range(0, -1)
	fmt.Println("Count:", len(values))
	// Output: Count: 0
}

func ExampleList_Range() {
	netConn, _ := net.Dial("tcp", internal.GetHostAndPort())

	conn := redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	l := list.NewRedisList(conn, test.RandomKey())

	_, _ = l.RightPush("hello", "world", "how", "are", "you", "today")

	// Get the entire range
	values, _ := redis.Strings(l.Range(0, -1))
	fmt.Println(values)

	// Starting from the middle
	values, _ = redis.Strings(l.Range(2, -1))
	fmt.Println(values)

	// From beginning to middle
	values, _ = redis.Strings(l.Range(0, 1))
	fmt.Println(values)

	// Output:
	// [hello world how are you today]
	// [how are you today]
	// [hello world]
}

func verifySlice(t *testing.T, l list.List, wantCount int, scenarios []scenarioStruct, reverse bool) {
	got, err := l.Range(0, -1)
	assert.Nil(t, err)
	assert.EqualValues(t, wantCount, len(got))
	assert.Len(t, got, wantCount)
	i := 0
	for _, scenario := range scenarios {
		if !scenario.wantErr {
			for _, item := range scenario.add {
				index := i
				if reverse == reverseSlice {
					index = len(got) - i - 1
				}
				test.AssertEqual(t, item, got[index])
				i++
			}
		}
	}
}

func TestRedisHyperLogLog_Name(t *testing.T) {
	name := test.RandomKey()
	l := list.NewRedisList(conn, name)
	assert.Equal(t, name, l.Name())
}

func TestRedisList_LeftPop(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		item, err := l.LeftPop()
		assert.Nil(t, err)
		assert.Nil(t, item)
	})

	t.Run("list with one item", func(t *testing.T) {
		_, _ = l.LeftPush("abc")
		item, err := redis.String(l.LeftPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "abc", item)
	})

	t.Run("list with multiple items", func(t *testing.T) {
		_, _ = l.LeftPush("def", "ghi")
		item, err := redis.String(l.LeftPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "ghi", item)
	})
}

func TestRedisList_LeftPush(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	scenarios := []scenarioStruct{
		{
			name: "non-existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integers",
			add:  test.IntsToInterfaceSlice(1, 2),
		},
		{
			name: "add a string",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name:    "no items",
			add:     nil,
			wantErr: true,
		},
	}

	wantCount := 0

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.LeftPush(scenario.add...)
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, reverseSlice)
	})
}

func TestRedisList_LeftPushX(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		count, err := l.LeftPushX("abc")
		assert.Nil(t, err)
		assert.EqualValues(t, 0, count)
	})

	_, _ = l.RightPush("abc")

	scenarios := []scenarioStruct{
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integer",
			add:  test.IntsToInterfaceSlice(1),
		},
	}

	wantCount := 1

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.LeftPushX(scenario.add[0])
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	// Pop the one added by RightPush
	_, _ = l.RightPop()
	wantCount--

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, reverseSlice)
	})
}

func TestRedisList_Range(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		items, err := l.Range(0, -1)
		assert.Nil(t, err)
		assert.Empty(t, items)
	})

	added := test.StringsToInterfaceSlice("abc", "def", "ghi")
	added = append(added, 5)
	added = append(added, "jkl")
	added = append(added, 10)

	_, _ = l.RightPush(added...)

	t.Run("full range", func(t *testing.T) {
		items, err := l.Range(0, -1)
		assert.Nil(t, err)
		assert.Len(t, items, len(added))
		for i, item := range items {
			test.AssertEqual(t, added[i], item)
		}
	})

	t.Run("from start to middle", func(t *testing.T) {
		items, err := l.Range(0, 2)
		assert.Nil(t, err)
		assert.Len(t, items, 3)
		for i, item := range items {
			test.AssertEqual(t, added[i], item)
		}
	})

	t.Run("from middle to end", func(t *testing.T) {
		items, err := l.Range(3, -1)
		assert.Nil(t, err)
		assert.Len(t, items, 3)
		for i, item := range items {
			test.AssertEqual(t, added[i+3], item)
		}
	})
}

func TestRedisList_RightPop(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		item, err := l.RightPop()
		assert.Nil(t, err)
		assert.Nil(t, item)
	})

	t.Run("list with one item", func(t *testing.T) {
		_, _ = l.RightPush("abc")
		item, err := redis.String(l.RightPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "abc", item)
	})

	t.Run("list with multiple items", func(t *testing.T) {
		_, _ = l.RightPush("def", "ghi")
		item, err := redis.String(l.RightPop())
		assert.Nil(t, err)
		assert.EqualValues(t, "ghi", item)
	})
}

func TestRedisList_RightPush(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	scenarios := []scenarioStruct{
		{
			name: "non-existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integers",
			add:  test.IntsToInterfaceSlice(1, 2),
		},
		{
			name: "add a string",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name:    "no items",
			add:     nil,
			wantErr: true,
		},
	}

	wantCount := 0

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.RightPush(scenario.add...)
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, forwardSlice)
	})
}

func TestRedisList_RightPushX(t *testing.T) {
	l := list.NewRedisList(conn, test.RandomKey())

	t.Run("non-existing key", func(t *testing.T) {
		count, err := l.RightPushX("abc")
		assert.Nil(t, err)
		assert.EqualValues(t, 0, count)
	})

	_, _ = l.RightPush("abc")

	scenarios := []scenarioStruct{
		{
			name: "existing key",
			add:  test.StringsToInterfaceSlice("abc"),
		},
		{
			name: "add integer",
			add:  test.IntsToInterfaceSlice(1),
		},
	}

	wantCount := 1

	for _, scenario := range scenarios {
		t.Run(scenario.name, func(t *testing.T) {
			count, err := l.RightPushX(scenario.add[0])
			if scenario.wantErr {
				assert.NotNil(t, err)
			} else {
				assert.Nil(t, err)
				wantCount += len(scenario.add)
				assert.EqualValues(t, wantCount, count)
			}
		})
	}

	// Pop the one added by RightPush
	_, _ = l.LeftPop()
	wantCount--

	t.Run("verify list data", func(t *testing.T) {
		verifySlice(t, l, wantCount, scenarios, forwardSlice)
	})
}

func TestMain(m *testing.M) {
	netConn, err := net.Dial("tcp", internal.GetHostAndPort())
	if err != nil {
		fmt.Printf("Error opening net connection, err: %v", err)
		os.Exit(1)
	}

	conn = redis.NewConn(netConn, time.Second, time.Second)
	defer conn.Close()

	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Pani Networks Inc
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package listener's nodes.go contains a subset of listener functionality
// where it connects to kubernetes using kubernetes client-go and watches
// node creation/deletion events and then adds/deletes the nodes
// to/from romana cluster appropriately.
package listener

import (
	"errors"
	"fmt"
	"net"
	"time"

	log "github.com/romana/rlog"

	romanaApi "github.com/romana/core/common/api"
	romanaErrors "github.com/romana/core/common/api/errors"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

func (l *KubeListener) kubeClientInit() error {
	var err error

	// Try generating config for kubernetes client-go from
	// in-cluster variables like KUBERNETES_SERVICE_HOST, etc
	// so that we can connect to kubernetes using them.
	kConfig, err := rest.InClusterConfig()
	if err != nil {
		return fmt.Errorf("Error: %s", err)
	}

	// Get a set of REST clients which connect to kubernetes services
	// from the config generated above.
	l.kubeClientSet, err = kubernetes.NewForConfig(kConfig)
	if err != nil {
		return fmt.Errorf("Error while connecting to kubernetes: %s", err)
	}

	return nil
}

// ProcessNodeEvents processes kubernetes node events, there by
// adding/deleting nodes to/from romana cluster automatically
// when they are added/removed to/from kubernetes cluster.
func (l *KubeListener) ProcessNodeEvents(done <-chan struct{}) {
	log.Debug("In ProcessNodeEvents()")

	// nodeWatcher is a new ListWatch object created from the specified
	// kubeClientSet which k8s.io/client-go exports for watching node events.
	nodeWatcher := cache.NewListWatchFromClient(
		l.kubeClientSet.CoreV1Client.RESTClient(),
		"nodes",
		api.NamespaceAll,
		fields.Everything())

	// Setup a notifications for specific events using NewInformer.
	_, nodeInformer := cache.NewInformer(
		nodeWatcher,
		&v1.Node{},
		time.Minute,
		cache.ResourceEventHandlerFuncs{
			AddFunc:    l.kubernetesAddNodeEventHandler,
			UpdateFunc: l.kubernetesUpdateNodeEventHandler,
			DeleteFunc: l.kubernetesDeleteNodeEventHandler,
		},
	)

	log.Infof("Starting receving node events.")
	go nodeInformer.Run(done)
}

// kubernetesAddNodeEventHandler is called when Kubernetes reports an
// add-node event It connects to the Romana REST API and adds the node
// to the Romana cluster.
func (l *KubeListener) kubernetesAddNodeEventHandler(n interface{}) {
	log.Debug("In kubernetesAddNodeEventHandler()")
	node, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Add Event received for node(%s) ", n)
		return
	}

	log.Debugf("Add Event received for node(%s, %s, %d) ",
		node.Name, node.Status.Addresses, len(node.Status.Addresses))

	if err := romanaHostAdd(l, node); err != nil {
		log.Errorf("Error processing Add Event received for node(%s): %s",
			node.Name, err)
		return
	}
}

// kubernetesUpdateNodeEventHandler currently doesn't do anything yet.
// TODO: If node shows up with new IP or romana CIDR,
//       then accommodate it if possible.
func (l *KubeListener) kubernetesUpdateNodeEventHandler(o, n interface{}) {
	log.Debug("In kubernetesUpdateNodeEventHandler()")

	// node, ok := n.(*v1.Node)
	_, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Update Event received for node(%s) ", n)
		return
	}

	// Disable this for now, update events are sent every
	// 10 seconds per node, thus this could fill up the log
	// easily in very small amount of time.
	// log.Info("Update Event received for node: ",node.Name)
}

// kubernetesDeleteNodeEventHandler is called when Kubernetes reports a
// delete-node event It connects to the Romana REST API and deletes the
// node from the Romana cluster.
func (l *KubeListener) kubernetesDeleteNodeEventHandler(n interface{}) {
	log.Debug("In kubernetesDeleteNodeEventHandler()")
	node, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Delete Event received for node(%s) ", n)
		return
	}

	log.Debugf("Delete Event received for node(%s, %s) ",
		node.Name, node.Status.Addresses)

	if err := romanaHostRemove(l, node.Name); err != nil {
		log.Errorf("Error processing Delete Event received for node(%s): %v",
			node.Name, err)
		return
	}

	log.Infof("Node (%s) successfully removed from romana cluster.", node.Name)
}

// romanaHostAdd connects to romana API and adds a node to
// the romana cluster.
func romanaHostAdd(l *KubeListener, node *v1.Node) error {
	if node.Name == "" || len(node.Status.Addresses) < 1 {
		log.Errorf("Error: received invalid host name or IP Address: (%s)", node)
		return errors.New("Error: received invalid host name or IP Address.")
	}
	hostname := node.Name
	hostIP := net.ParseIP(node.Status.Addresses[0].Address)
	log.Infof("KubeListener: Adding host IP: %s, Name: %s, Labels: %v\n", hostIP, hostname, node.GetLabels())
	host := romanaApi.Host{IP: hostIP,
		Name: hostname,
		Tags: node.GetLabels(),
	}
	l.clientMu.Lock()
	defer l.clientMu.Unlock()
	err := l.client.IPAM.AddHost(host)
	if err == nil {
		log.Infof("Node (%s) successfully added to romana cluster.", node.Name)
		return nil
	}
	if err, ok := err.(romanaErrors.RomanaExistsError); ok {
		log.Infof("Host %s already exists, ignoring.", host)
		return nil
	} else {
		return err
	}
}

// romanaHostRemove connects to romana API and removes a node from
// the romana cluster.
func romanaHostRemove(l *KubeListener, node string) error {
	log.Infof("KubeListener: Removing host %s\n", node)
	if node == "" {
		log.Errorf("Error: received invalid node name (%s)", node)
		return errors.New("Error: received invalid node name.")
	}
	return l.client.IPAM.RemoveHost(romanaApi.Host{Name: node})
}
<commit_msg>Changes for rebase<commit_after>// Copyright (c) 2016-2017 Pani Networks Inc
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package listener's nodes.go contains a subset of listener functionality
// where it connects to kubernetes using kubernetes client-go and watches
// node creation/deletion events and then adds/deletes the nodes
// to/from romana cluster appropriately.
package listener

import (
	"errors"
	"fmt"
	"net"
	"time"

	log "github.com/romana/rlog"

	romanaApi "github.com/romana/core/common/api"
	romanaErrors "github.com/romana/core/common/api/errors"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

func (l *KubeListener) kubeClientInit() error {
	var err error

	// Try generating config for kubernetes client-go from
	// in-cluster variables like KUBERNETES_SERVICE_HOST, etc
	// so that we can connect to kubernetes using them.
	kConfig, err := rest.InClusterConfig()
	if err != nil {
		return fmt.Errorf("Error: %s", err)
	}

	// Get a set of REST clients which connect to kubernetes services
	// from the config generated above.
	l.kubeClientSet, err = kubernetes.NewForConfig(kConfig)
	if err != nil {
		return fmt.Errorf("Error while connecting to kubernetes: %s", err)
	}

	return nil
}

// ProcessNodeEvents processes kubernetes node events, there by
// adding/deleting nodes to/from romana cluster automatically
// when they are added/removed to/from kubernetes cluster.
func (l *KubeListener) ProcessNodeEvents(done <-chan struct{}) {
	log.Debug("In ProcessNodeEvents()")

	// nodeWatcher is a new ListWatch object created from the specified
	// kubeClientSet which k8s.io/client-go exports for watching node events.
	nodeWatcher := cache.NewListWatchFromClient(
		l.kubeClientSet.CoreV1Client.RESTClient(),
		"nodes",
		api.NamespaceAll,
		fields.Everything())

	// Setup a notifications for specific events using NewInformer.
	_, nodeInformer := cache.NewInformer(
		nodeWatcher,
		&v1.Node{},
		time.Minute,
		cache.ResourceEventHandlerFuncs{
			AddFunc:    l.kubernetesAddNodeEventHandler,
			UpdateFunc: l.kubernetesUpdateNodeEventHandler,
			DeleteFunc: l.kubernetesDeleteNodeEventHandler,
		},
	)

	log.Infof("Starting receving node events.")
	go nodeInformer.Run(done)
}

// kubernetesAddNodeEventHandler is called when Kubernetes reports an
// add-node event It connects to the Romana REST API and adds the node
// to the Romana cluster.
func (l *KubeListener) kubernetesAddNodeEventHandler(n interface{}) {
	log.Debug("In kubernetesAddNodeEventHandler()")
	node, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Add Event received for node(%s) ", n)
		return
	}

	log.Debugf("Add Event received for node(%s, %s, %d) ",
		node.Name, node.Status.Addresses, len(node.Status.Addresses))

	if err := romanaHostAdd(l, node); err != nil {
		log.Errorf("Error processing Add Event received for node(%s): %s",
			node.Name, err)
		return
	}
}

// kubernetesUpdateNodeEventHandler currently doesn't do anything yet.
// TODO: If node shows up with new IP or romana CIDR,
//       then accommodate it if possible.
func (l *KubeListener) kubernetesUpdateNodeEventHandler(o, n interface{}) {
	log.Debug("In kubernetesUpdateNodeEventHandler()")

	// node, ok := n.(*v1.Node)
	_, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Update Event received for node(%s) ", n)
		return
	}

	// Disable this for now, update events are sent every
	// 10 seconds per node, thus this could fill up the log
	// easily in very small amount of time.
	// log.Info("Update Event received for node: ",node.Name)
}

// kubernetesDeleteNodeEventHandler is called when Kubernetes reports a
// delete-node event It connects to the Romana REST API and deletes the
// node from the Romana cluster.
func (l *KubeListener) kubernetesDeleteNodeEventHandler(n interface{}) {
	log.Debug("In kubernetesDeleteNodeEventHandler()")
	node, ok := n.(*v1.Node)
	if !ok {
		log.Errorf("Error processing Delete Event received for node(%s) ", n)
		return
	}

	log.Debugf("Delete Event received for node(%s, %s) ",
		node.Name, node.Status.Addresses)

	if err := romanaHostRemove(l, node.Name); err != nil {
		log.Errorf("Error processing Delete Event received for node(%s): %v",
			node.Name, err)
		return
	}

	log.Infof("Node (%s) successfully removed from romana cluster.", node.Name)
}

// romanaHostAdd connects to romana API and adds a node to
// the romana cluster.
func romanaHostAdd(l *KubeListener, node *v1.Node) error {
	if node.Name == "" || len(node.Status.Addresses) < 1 {
		log.Errorf("Error: received invalid host name or IP Address: (%s)", node)
		return errors.New("Error: received invalid host name or IP Address.")
	}
	hostname := node.Name
	hostIP := net.ParseIP(node.Status.Addresses[0].Address)
	log.Infof("KubeListener: Adding host IP: %s, Name: %s, Labels: %v\n", hostIP, hostname, node.GetLabels())
	host := romanaApi.Host{IP: hostIP,
		Name: hostname,
		Tags: node.GetLabels(),
	}
	err := l.client.IPAM.AddHost(host)
	if err == nil {
		log.Infof("Node (%s) successfully added to romana cluster.", node.Name)
		return nil
	}
	if err, ok := err.(romanaErrors.RomanaExistsError); ok {
		log.Infof("Host %s already exists, ignoring.", host)
		return nil
	} else {
		return err
	}
}

// romanaHostRemove connects to romana API and removes a node from
// the romana cluster.
func romanaHostRemove(l *KubeListener, node string) error {
	log.Infof("KubeListener: Removing host %s\n", node)
	if node == "" {
		log.Errorf("Error: received invalid node name (%s)", node)
		return errors.New("Error: received invalid node name.")
	}
	return l.client.IPAM.RemoveHost(romanaApi.Host{Name: node})
}
<|endoftext|>


<commit_before>package vfs

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/libcontainer/label"
)

func init() {
	graphdriver.Register("vfs", Init)
}

func Init(home string, options []string) (graphdriver.Driver, error) {
	d := &Driver{
		home: home,
	}
	return graphdriver.NaiveDiffDriver(d), nil
}

type Driver struct {
	home string
}

func (d *Driver) String() string {
	return "vfs"
}

func (d *Driver) Status() [][2]string {
	return nil
}

func (d *Driver) Cleanup() error {
	return nil
}

func isGNUcoreutils() bool {
	if stdout, err := exec.Command("cp", "--version").Output(); err == nil {
		return bytes.Contains(stdout, []byte("GNU coreutils"))
	}

	return false
}

func copyDir(src, dst string) error {
	argv := make([]string, 0, 4)

	if isGNUcoreutils() {
		argv = append(argv, "-aT", "--reflink=auto", src, dst)
	} else {
		argv = append(argv, "-a", src+"/.", dst+"/.")
	}

	if output, err := exec.Command("cp", argv...).CombinedOutput(); err != nil {
		return fmt.Errorf("Error VFS copying directory: %s (%s)", err, output)
	}
	return nil
}

func (d *Driver) Create(id, parent string) error {
	dir := d.dir(id)
	if err := os.MkdirAll(path.Dir(dir), 0700); err != nil {
		return err
	}
	if err := os.Mkdir(dir, 0755); err != nil {
		return err
	}
	opts := []string{"level:s0"}
	if _, mountLabel, err := label.InitLabels(opts); err == nil {
		label.Relabel(dir, mountLabel, "")
	}
	if parent == "" {
		return nil
	}
	parentDir, err := d.Get(parent, "")
	if err != nil {
		return fmt.Errorf("%s: %s", parent, err)
	}
	if err := copyDir(parentDir, dir); err != nil {
		return err
	}
	return nil
}

func (d *Driver) dir(id string) string {
	return path.Join(d.home, "dir", path.Base(id))
}

func (d *Driver) Remove(id string) error {
	if _, err := os.Stat(d.dir(id)); err != nil {
		return err
	}
	return os.RemoveAll(d.dir(id))
}

func (d *Driver) Get(id, mountLabel string) (string, error) {
	dir := d.dir(id)
	if st, err := os.Stat(dir); err != nil {
		return "", err
	} else if !st.IsDir() {
		return "", fmt.Errorf("%s: not a directory", dir)
	}
	return dir, nil
}

func (d *Driver) Put(id string) {
	// The vfs driver has no runtime resources (e.g. mounts)
	// to clean up, so we don't need anything here
}

func (d *Driver) Exists(id string) bool {
	_, err := os.Stat(d.dir(id))
	return err == nil
}
<commit_msg>Use archive.CopyWithTar in vfs.Create<commit_after>package vfs

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/libcontainer/label"
)

func init() {
	graphdriver.Register("vfs", Init)
}

func Init(home string, options []string) (graphdriver.Driver, error) {
	d := &Driver{
		home: home,
	}
	return graphdriver.NaiveDiffDriver(d), nil
}

type Driver struct {
	home string
}

func (d *Driver) String() string {
	return "vfs"
}

func (d *Driver) Status() [][2]string {
	return nil
}

func (d *Driver) Cleanup() error {
	return nil
}

func isGNUcoreutils() bool {
	if stdout, err := exec.Command("cp", "--version").Output(); err == nil {
		return bytes.Contains(stdout, []byte("GNU coreutils"))
	}

	return false
}

func (d *Driver) Create(id, parent string) error {
	dir := d.dir(id)
	if err := os.MkdirAll(path.Dir(dir), 0700); err != nil {
		return err
	}
	if err := os.Mkdir(dir, 0755); err != nil {
		return err
	}
	opts := []string{"level:s0"}
	if _, mountLabel, err := label.InitLabels(opts); err == nil {
		label.Relabel(dir, mountLabel, "")
	}
	if parent == "" {
		return nil
	}
	parentDir, err := d.Get(parent, "")
	if err != nil {
		return fmt.Errorf("%s: %s", parent, err)
	}
	if err := archive.CopyWithTar(parentDir, dir); err != nil {
		return err
	}
	return nil
}

func (d *Driver) dir(id string) string {
	return path.Join(d.home, "dir", path.Base(id))
}

func (d *Driver) Remove(id string) error {
	if _, err := os.Stat(d.dir(id)); err != nil {
		return err
	}
	return os.RemoveAll(d.dir(id))
}

func (d *Driver) Get(id, mountLabel string) (string, error) {
	dir := d.dir(id)
	if st, err := os.Stat(dir); err != nil {
		return "", err
	} else if !st.IsDir() {
		return "", fmt.Errorf("%s: not a directory", dir)
	}
	return dir, nil
}

func (d *Driver) Put(id string) {
	// The vfs driver has no runtime resources (e.g. mounts)
	// to clean up, so we don't need anything here
}

func (d *Driver) Exists(id string) bool {
	_, err := os.Stat(d.dir(id))
	return err == nil
}
<|endoftext|>


<commit_before>package netplugin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math"
	"net"
	"os/exec"
	"strings"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gardener"
	"code.cloudfoundry.org/guardian/kawasaki"
	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry/gunk/command_runner"
)

const NetworkPropertyPrefix = "network."
const NetOutKey = NetworkPropertyPrefix + "external-networker.net-out"

type externalBinaryNetworker struct {
	commandRunner    command_runner.CommandRunner
	configStore      kawasaki.ConfigStore
	portPool         kawasaki.PortPool
	externalIP       net.IP
	dnsServers       []net.IP
	resolvConfigurer kawasaki.DnsResolvConfigurer
	path             string
	extraArg         []string
}

func New(
	commandRunner command_runner.CommandRunner,
	configStore kawasaki.ConfigStore,
	portPool kawasaki.PortPool,
	externalIP net.IP,
	dnsServers []net.IP,
	resolvConfigurer kawasaki.DnsResolvConfigurer,
	path string,
	extraArg []string,
) ExternalNetworker {
	return &externalBinaryNetworker{
		commandRunner:    commandRunner,
		configStore:      configStore,
		portPool:         portPool,
		externalIP:       externalIP,
		dnsServers:       dnsServers,
		resolvConfigurer: resolvConfigurer,
		path:             path,
		extraArg:         extraArg,
	}
}

type ExternalNetworker interface {
	gardener.Networker
	gardener.Starter
}

func (p *externalBinaryNetworker) Start() error { return nil }

func networkProperties(containerProperties garden.Properties) garden.Properties {
	properties := garden.Properties{}

	for k, value := range containerProperties {
		if strings.HasPrefix(k, NetworkPropertyPrefix) {
			key := strings.TrimPrefix(k, NetworkPropertyPrefix)
			properties[key] = value
		}
	}

	return properties
}

type UpInputs struct {
	Pid        int
	Properties map[string]string
}
type UpOutputs struct {
	Properties map[string]string
}

func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden.ContainerSpec, pid int) error {
	p.configStore.Set(containerSpec.Handle, gardener.ExternalIPKey, p.externalIP.String())

	inputs := UpInputs{
		Pid:        pid,
		Properties: networkProperties(containerSpec.Properties),
	}
	outputs := UpOutputs{}
	err := p.exec2(log, "up", containerSpec.Handle, inputs, &outputs)
	if err != nil {
		return err
	}

	for k, v := range outputs.Properties {
		p.configStore.Set(containerSpec.Handle, k, v)
	}

	containerIP, ok := p.configStore.Get(containerSpec.Handle, gardener.ContainerIPKey)
	if !ok {
		return fmt.Errorf("plugin failed to set a container ip")
	}

	log.Info("external-binary-write-dns-to-config", lager.Data{
		"dnsServers": p.dnsServers,
	})
	cfg := kawasaki.NetworkConfig{
		ContainerIP:     net.ParseIP(containerIP),
		BridgeIP:        net.ParseIP(containerIP),
		ContainerHandle: containerSpec.Handle,
		DNSServers:      p.dnsServers,
	}

	err = p.resolvConfigurer.Configure(log, cfg, pid)
	if err != nil {
		return err
	}

	return nil
}

func (p *externalBinaryNetworker) Destroy(log lager.Logger, handle string) error {
	pathAndExtraArgs := append([]string{p.path}, p.extraArg...)

	networkPluginFlags := []string{
		"--handle", handle,
	}

	downArgs := append(pathAndExtraArgs, "--action", "down")
	downArgs = append(downArgs, networkPluginFlags...)

	cmd := exec.Command(p.path)
	cmd.Args = downArgs
	return p.commandRunner.Run(cmd)
}

func (p *externalBinaryNetworker) Restore(log lager.Logger, handle string) error {
	return nil
}

func (p *externalBinaryNetworker) Capacity() (m uint64) {
	return math.MaxUint64
}

type NetInInputs struct {
	HostIP        string
	HostPort      uint32
	ContainerIP   string
	ContainerPort uint32
}

type NetInOutputs struct {
	HostPort      uint32 `json:"host_port"`
	ContainerPort uint32 `json:"container_port"`
}

func (p *externalBinaryNetworker) NetIn(log lager.Logger, handle string, hostPort, containerPort uint32) (uint32, uint32, error) {
	containerIP, ok := p.configStore.Get(handle, gardener.ContainerIPKey)
	if !ok {
		return 0, 0, fmt.Errorf("cannot find container [%s]\n", handle)
	}

	inputs := NetInInputs{
		HostIP:        p.externalIP.String(),
		ContainerIP:   containerIP,
		HostPort:      hostPort,
		ContainerPort: containerPort,
	}

	var outputs NetInOutputs

	err := p.exec2(log, "net-in", handle, inputs, &outputs)
	if err != nil {
		return 0, 0, err
	}

	err = kawasaki.AddPortMapping(log, p.configStore, handle, garden.PortMapping{
		HostPort:      outputs.HostPort,
		ContainerPort: outputs.ContainerPort,
	})
	if err != nil {
		return 0, 0, err
	}

	return outputs.HostPort, outputs.ContainerPort, err
}

type NetOutInputs struct {
	ContainerIP string            `json:"container_ip"`
	NetOutRule  garden.NetOutRule `json:"netout_rule"`
}

func (p *externalBinaryNetworker) NetOut(log lager.Logger, handle string, rule garden.NetOutRule) error {
	containerIP, ok := p.configStore.Get(handle, gardener.ContainerIPKey)
	if !ok {
		return fmt.Errorf("cannot find container [%s]\n", handle)
	}

	inputs := NetOutInputs{
		ContainerIP: containerIP,
		NetOutRule:  rule,
	}

	err := p.exec2(log, "net-out", handle, inputs, nil)
	if err != nil {
		return err
	}

	return nil
}

func (p *externalBinaryNetworker) exec2(log lager.Logger, action, handle string,
	inputData interface{}, outputData interface{}) error {

	stdinBytes, err := json.Marshal(inputData)
	if err != nil {
		return err
	}

	args := append(p.extraArg, "--action", action, "--handle", handle)
	cmd := exec.Command(p.path, args...)
	stdout := &bytes.Buffer{}
	cmd.Stdout = stdout
	stderr := &bytes.Buffer{}
	cmd.Stderr = stderr
	cmd.Stdin = bytes.NewReader(stdinBytes)

	err = p.commandRunner.Run(cmd)

	logData := lager.Data{"stderr": stderr.String(), "stdout": stdout.String()}
	if err != nil {
		log.Error("external-networker-result", err, logData)
		return fmt.Errorf("external networker: %s", err)
	}

	if outputData != nil {
		err = json.Unmarshal(stdout.Bytes(), outputData)
		if err != nil {
			log.Error("external-networker-result", err, logData)
			return fmt.Errorf("unmarshaling result from external networker: %s", err)
		}
	}

	log.Debug("external-networker-result", logData)
	return nil
}
<commit_msg>Remove old unused NetOutKey constant<commit_after>package netplugin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math"
	"net"
	"os/exec"
	"strings"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gardener"
	"code.cloudfoundry.org/guardian/kawasaki"
	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry/gunk/command_runner"
)

const NetworkPropertyPrefix = "network."

type externalBinaryNetworker struct {
	commandRunner    command_runner.CommandRunner
	configStore      kawasaki.ConfigStore
	portPool         kawasaki.PortPool
	externalIP       net.IP
	dnsServers       []net.IP
	resolvConfigurer kawasaki.DnsResolvConfigurer
	path             string
	extraArg         []string
}

func New(
	commandRunner command_runner.CommandRunner,
	configStore kawasaki.ConfigStore,
	portPool kawasaki.PortPool,
	externalIP net.IP,
	dnsServers []net.IP,
	resolvConfigurer kawasaki.DnsResolvConfigurer,
	path string,
	extraArg []string,
) ExternalNetworker {
	return &externalBinaryNetworker{
		commandRunner:    commandRunner,
		configStore:      configStore,
		portPool:         portPool,
		externalIP:       externalIP,
		dnsServers:       dnsServers,
		resolvConfigurer: resolvConfigurer,
		path:             path,
		extraArg:         extraArg,
	}
}

type ExternalNetworker interface {
	gardener.Networker
	gardener.Starter
}

func (p *externalBinaryNetworker) Start() error { return nil }

func networkProperties(containerProperties garden.Properties) garden.Properties {
	properties := garden.Properties{}

	for k, value := range containerProperties {
		if strings.HasPrefix(k, NetworkPropertyPrefix) {
			key := strings.TrimPrefix(k, NetworkPropertyPrefix)
			properties[key] = value
		}
	}

	return properties
}

type UpInputs struct {
	Pid        int
	Properties map[string]string
}
type UpOutputs struct {
	Properties map[string]string
}

func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden.ContainerSpec, pid int) error {
	p.configStore.Set(containerSpec.Handle, gardener.ExternalIPKey, p.externalIP.String())

	inputs := UpInputs{
		Pid:        pid,
		Properties: networkProperties(containerSpec.Properties),
	}
	outputs := UpOutputs{}
	err := p.exec2(log, "up", containerSpec.Handle, inputs, &outputs)
	if err != nil {
		return err
	}

	for k, v := range outputs.Properties {
		p.configStore.Set(containerSpec.Handle, k, v)
	}

	containerIP, ok := p.configStore.Get(containerSpec.Handle, gardener.ContainerIPKey)
	if !ok {
		return fmt.Errorf("plugin failed to set a container ip")
	}

	log.Info("external-binary-write-dns-to-config", lager.Data{
		"dnsServers": p.dnsServers,
	})
	cfg := kawasaki.NetworkConfig{
		ContainerIP:     net.ParseIP(containerIP),
		BridgeIP:        net.ParseIP(containerIP),
		ContainerHandle: containerSpec.Handle,
		DNSServers:      p.dnsServers,
	}

	err = p.resolvConfigurer.Configure(log, cfg, pid)
	if err != nil {
		return err
	}

	return nil
}

func (p *externalBinaryNetworker) Destroy(log lager.Logger, handle string) error {
	pathAndExtraArgs := append([]string{p.path}, p.extraArg...)

	networkPluginFlags := []string{
		"--handle", handle,
	}

	downArgs := append(pathAndExtraArgs, "--action", "down")
	downArgs = append(downArgs, networkPluginFlags...)

	cmd := exec.Command(p.path)
	cmd.Args = downArgs
	return p.commandRunner.Run(cmd)
}

func (p *externalBinaryNetworker) Restore(log lager.Logger, handle string) error {
	return nil
}

func (p *externalBinaryNetworker) Capacity() (m uint64) {
	return math.MaxUint64
}

type NetInInputs struct {
	HostIP        string
	HostPort      uint32
	ContainerIP   string
	ContainerPort uint32
}

type NetInOutputs struct {
	HostPort      uint32 `json:"host_port"`
	ContainerPort uint32 `json:"container_port"`
}

func (p *externalBinaryNetworker) NetIn(log lager.Logger, handle string, hostPort, containerPort uint32) (uint32, uint32, error) {
	containerIP, ok := p.configStore.Get(handle, gardener.ContainerIPKey)
	if !ok {
		return 0, 0, fmt.Errorf("cannot find container [%s]\n", handle)
	}

	inputs := NetInInputs{
		HostIP:        p.externalIP.String(),
		ContainerIP:   containerIP,
		HostPort:      hostPort,
		ContainerPort: containerPort,
	}

	var outputs NetInOutputs

	err := p.exec2(log, "net-in", handle, inputs, &outputs)
	if err != nil {
		return 0, 0, err
	}

	err = kawasaki.AddPortMapping(log, p.configStore, handle, garden.PortMapping{
		HostPort:      outputs.HostPort,
		ContainerPort: outputs.ContainerPort,
	})
	if err != nil {
		return 0, 0, err
	}

	return outputs.HostPort, outputs.ContainerPort, err
}

type NetOutInputs struct {
	ContainerIP string            `json:"container_ip"`
	NetOutRule  garden.NetOutRule `json:"netout_rule"`
}

func (p *externalBinaryNetworker) NetOut(log lager.Logger, handle string, rule garden.NetOutRule) error {
	containerIP, ok := p.configStore.Get(handle, gardener.ContainerIPKey)
	if !ok {
		return fmt.Errorf("cannot find container [%s]\n", handle)
	}

	inputs := NetOutInputs{
		ContainerIP: containerIP,
		NetOutRule:  rule,
	}

	err := p.exec2(log, "net-out", handle, inputs, nil)
	if err != nil {
		return err
	}

	return nil
}

func (p *externalBinaryNetworker) exec2(log lager.Logger, action, handle string,
	inputData interface{}, outputData interface{}) error {

	stdinBytes, err := json.Marshal(inputData)
	if err != nil {
		return err
	}

	args := append(p.extraArg, "--action", action, "--handle", handle)
	cmd := exec.Command(p.path, args...)
	stdout := &bytes.Buffer{}
	cmd.Stdout = stdout
	stderr := &bytes.Buffer{}
	cmd.Stderr = stderr
	cmd.Stdin = bytes.NewReader(stdinBytes)

	err = p.commandRunner.Run(cmd)

	logData := lager.Data{"stderr": stderr.String(), "stdout": stdout.String()}
	if err != nil {
		log.Error("external-networker-result", err, logData)
		return fmt.Errorf("external networker: %s", err)
	}

	if outputData != nil {
		err = json.Unmarshal(stdout.Bytes(), outputData)
		if err != nil {
			log.Error("external-networker-result", err, logData)
			return fmt.Errorf("unmarshaling result from external networker: %s", err)
		}
	}

	log.Debug("external-networker-result", logData)
	return nil
}
<|endoftext|>


<commit_before><commit_msg>ensure we can spend previously time-locked outputs<commit_after><|endoftext|>


<commit_before>/*
 * gomacro - A Go interpreter with Lisp-like macros
 *
 * Copyright (C) 2017-2019 Massimiliano Ghilardi
 *
 *     This Source Code Form is subject to the terms of the Mozilla Public
 *     License, v. 2.0. If a copy of the MPL was not distributed with this
 *     file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 *
 * genimport.go
 *
 *  Created on May 26, 2017
 *      Author Massimiliano Ghilardi
 */

package genimport

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/types"
	"math"
	"strconv"
	"strings"

	"github.com/cosmos72/gomacro/base/output"
	"github.com/cosmos72/gomacro/base/untyped"
)

type Output = output.Output

type genimport struct {
	output      *Output
	mode        ImportMode
	gpkg        *types.Package
	scope       *types.Scope
	names       []string
	pkgrenames  map[string]string // map[path]name of packages to import, where name:s are guaranteed to be unique
	out         *bytes.Buffer
	path        string
	name, name_ string
	proxyprefix string
	reflect     string
}

func writeImportFile(o *Output, out *bytes.Buffer, path string, gpkg *types.Package, mode ImportMode) (isEmpty bool) {

	gen := newGenImport(o, out, path, gpkg, mode)
	if gen == nil {
		return true
	}
	gen.write()
	return false
}

func newGenImport(o *Output, out *bytes.Buffer, path string, gpkg *types.Package, mode ImportMode) *genimport {
	scope := gpkg.Scope()
	names := scope.Names()

	isEmpty := true
	for _, name := range names {
		if obj := scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.Const, *types.Var, *types.Func, *types.TypeName:
				isEmpty = false
				break
			}
		}
	}
	if isEmpty {
		return nil
	}

	gen := &genimport{output: o, mode: mode, gpkg: gpkg, scope: scope, names: names, out: out, path: path}

	if mode == ImInception {
		gen.reflect = "r."
		gen.name = packageSanitizedName(gen.path)
	}
	if mode == ImPlugin {
		gen.proxyprefix = "P_"
	} else {
		gen.proxyprefix = fmt.Sprintf("P_%s_", sanitizeIdent(path))
	}
	return gen
}

func (gen *genimport) write() {

	gen.writePreamble()

	gen.writeBinds()
	gen.writeTypes()
	gen.writeProxies()
	gen.writeUntypeds()
	gen.writeWrappers()

	gen.out.WriteString("\n\t}\n}\n")
	gen.writeInterfaceProxies()
}

type mapdecl struct {
	out  *bytes.Buffer
	head string
	foot string
}

func (gen *genimport) mapdecl(head string) mapdecl {
	if strings.IndexByte(head, '%') >= 0 {
		head = fmt.Sprintf(head, gen.reflect)
	}
	return mapdecl{gen.out, head, ""}
}

func (d *mapdecl) header() {
	if len(d.head) != 0 {
		d.out.WriteString(d.head)
		d.out.WriteByte('{')
		d.head = ""
		d.foot = "\n\t}"
	}
}

func (d *mapdecl) footer() {
	if len(d.foot) != 0 {
		d.out.WriteString(d.foot)
		d.out.WriteString(", ")
	}
}

func (d *mapdecl) footer1(comma bool) {
	if len(d.foot) != 0 {
		d.out.WriteString(d.foot)
		if comma {
			d.out.WriteString(", ")
		}
	}
}

func (gen *genimport) collectPackageImportsWithRename(requireAllInterfaceMethodsExported bool) {
	gen.pkgrenames = collectPackageImportsWithRename(gen.output, gen.gpkg, requireAllInterfaceMethodsExported)
	gen.name = gen.pkgrenames[gen.path]
	if gen.name == "" {
		gen.name = packageSanitizedName(gen.path)
	}
	if gen.mode != ImInception {
		gen.name_ = gen.name + "."
	}
}

func (gen *genimport) writePreamble() {
	mode := gen.mode
	out := gen.out

	var alias, filepkg string
	switch mode {
	case ImBuiltin:
		alias = "_b "
		filepkg = "imports"
	case ImThirdParty:
		filepkg = "thirdparty"
	case ImPlugin:
		filepkg = "main"
	case ImInception:
		alias = "_i "
		filepkg = gen.name
	}

	fmt.Fprintf(gen.out, `// this file was generated by gomacro command: import %s%q
// DO NOT EDIT! Any change will be lost when the file is re-generated

package %s

import (`, alias, gen.path, filepkg)

	var imports string
	if mode == ImInception {
		fmt.Fprintf(gen.out, "\n\tr \"reflect\"\n\t\"github.com/cosmos72/gomacro/imports\"")
		imports = "imports."
	} else {
		fmt.Fprintf(out, "\n\t. \"reflect\"")
	}
	gen.collectPackageImportsWithRename(true)
	for path, name := range gen.pkgrenames {
		if mode == ImInception && path == gen.path {
			continue // writing inside the package: it should not import itself
		} else {
			// always name the imported package: its name may differ from paths.FileName(path)
			fmt.Fprintf(out, "\n\t%s %q", name, path)
		}
	}
	fmt.Fprintf(out, "\n)\n")

	if mode == ImInception {
		gen.pkgrenames[gen.path] = "" // writing inside the package: remove the package prefix
	}

	if mode == ImPlugin {
		fmt.Fprint(out, `
type Package = struct {
	Name     string
	Binds    map[string]Value
	Types    map[string]Type
	Proxies  map[string]Type
	Untypeds map[string]string
	Wrappers map[string][]string
}

var Packages = make(map[string]Package)

func main() {
}

`)
	}

	fmt.Fprintf(out, `
// reflection: allow interpreted code to import %q
func init() {
	%sPackages[%q] = %sPackage{
	Name: %q,
	`, gen.path, imports, gen.path, imports, gen.gpkg.Name())
}

func (gen *genimport) writeBinds() {
	d := gen.mapdecl("Binds: map[string]%sValue")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj := obj.(type) {
			case *types.Const:
				val := obj.Val()
				var conv1, conv2 string
				if t, ok := obj.Type().(*types.Basic); ok && t.Info()&types.IsUntyped != 0 {
					// untyped constants have arbitrary precision... they may overflow integers.
					// this is just an approximation, use Package.Untypeds for exact value
					if val.Kind() == constant.Int {
						str := val.ExactString()
						conv1, conv2 = detectIntKind(gen.output, gen.path, name, str)
					}
				}
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(%s%s%s%s),", name, gen.reflect, conv1, gen.name_, name, conv2)
			case *types.Var:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(&%s%s).Elem(),", name, gen.reflect, gen.name_, name)
			case *types.Func:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(%s%s),", name, gen.reflect, gen.name_, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeTypes() {
	d := gen.mapdecl("Types: map[string]%sType")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.TypeName:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sTypeOf((*%s%s)(nil)).Elem(),", name, gen.reflect, gen.name_, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeProxies() {
	d := gen.mapdecl("Proxies: map[string]%sType")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			if t := extractInterface(obj, true); t != nil {
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sTypeOf((*%s%s)(nil)).Elem(),", name, gen.reflect, gen.proxyprefix, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeUntypeds() {
	d := gen.mapdecl("Untypeds: map[string]string")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj := obj.(type) {
			case *types.Const:
				if t, ok := obj.Type().(*types.Basic); ok && t.Info()&types.IsUntyped != 0 {
					kind := untyped.GoUntypedToKind(t.Kind())
					str := untyped.Marshal(kind, obj.Val())
					if len(str) != 0 {
						d.header()
						fmt.Fprintf(gen.out, "\n\t\t%q:\t%q,", name, str)
					}
				}
			}
		}
	}
	d.footer()
}

// find wrapper methods and write them. needed for accurate method selection.
func (gen *genimport) writeWrappers() {
	d := gen.mapdecl("Wrappers: map[string][]string")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.TypeName:
				if t, ok := obj.Type().(*types.Named); ok {
					// only structs can have embedded fields, and thus wrapper methods for embedded fields
					if _, ok := t.Underlying().(*types.Struct); ok {
						wrappers := new(analyzer).Analyze(t)
						if len(wrappers) != 0 {
							d.header()
							fmt.Fprintf(gen.out, "\n\t\t%q:\t[]string{", obj.Name())
							for _, wrapper := range wrappers {
								fmt.Fprintf(gen.out, "%q,", wrapper)
							}
							fmt.Fprint(gen.out, "},")
						}
					}
				}
			}
		}
	}
	d.footer()
}

// write proxies that pre-implement package's interfaces
func (gen *genimport) writeInterfaceProxies() {
	path := gen.gpkg.Path()
	for _, name := range gen.names {
		obj := gen.scope.Lookup(name)
		if t := extractInterface(obj, true); t != nil {
			gen.writeInterfaceProxy(path, name, t)
		}
	}
}

func detectIntKind(o *Output, path, name, str string) (string, string) {
	i, err := strconv.ParseInt(str, 0, 64)
	if err == nil {
		if i == int64(int32(i)) {
			// constant fits int32. We can use the default (i.e. int)
			// on both 32-bit and 64-bit platforms
			return "", ""
		} else if i == int64(uint32(i)) {
			// constant fits uint32
			return "uint32(", ")"
		} else {
			return "int64(", ")"
		}
	}
	_, err = strconv.ParseUint(str, 0, 64)
	if err == nil {
		return "uint64(", ")"
	}
	f, err := strconv.ParseFloat(str, 64)
	if err != nil {
		// nothing fits... leave the default
		return "", ""
	} else {
		prefix := "float64"
		f = math.Abs(f)
		if f == float64(float32(f)) && f <= math.MaxFloat32 && f >= math.SmallestNonzeroFloat32 {
			// float32 loses no precision vs. float64
			prefix = "float32"
		}
		o.Warnf("package %q: integer constant %s = %s overflows both int64 and uint64, converting to %s", path, name, str, prefix)
		return prefix + "(", ")"
	}
}
<commit_msg>improve previous commit: ensure 'import _i "package/full/path"' i.e. ImInception mode writes the correct package name in the generated file $GOPATH/src/package/full/path/x_package.go<commit_after>/*
 * gomacro - A Go interpreter with Lisp-like macros
 *
 * Copyright (C) 2017-2019 Massimiliano Ghilardi
 *
 *     This Source Code Form is subject to the terms of the Mozilla Public
 *     License, v. 2.0. If a copy of the MPL was not distributed with this
 *     file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 *
 * genimport.go
 *
 *  Created on May 26, 2017
 *      Author Massimiliano Ghilardi
 */

package genimport

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/types"
	"math"
	"strconv"
	"strings"

	"github.com/cosmos72/gomacro/base/output"
	"github.com/cosmos72/gomacro/base/untyped"
)

type Output = output.Output

type genimport struct {
	output      *Output
	mode        ImportMode
	gpkg        *types.Package
	scope       *types.Scope
	names       []string
	pkgrenames  map[string]string // map[path]name of packages to import, where name:s are guaranteed to be unique
	out         *bytes.Buffer
	path        string
	name, name_ string
	proxyprefix string
	reflect     string
}

func writeImportFile(o *Output, out *bytes.Buffer, path string, gpkg *types.Package, mode ImportMode) (isEmpty bool) {

	gen := newGenImport(o, out, path, gpkg, mode)
	if gen == nil {
		return true
	}
	gen.write()
	return false
}

func newGenImport(o *Output, out *bytes.Buffer, path string, gpkg *types.Package, mode ImportMode) *genimport {
	scope := gpkg.Scope()
	names := scope.Names()

	isEmpty := true
	for _, name := range names {
		if obj := scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.Const, *types.Var, *types.Func, *types.TypeName:
				isEmpty = false
				break
			}
		}
	}
	if isEmpty {
		return nil
	}

	gen := &genimport{output: o, mode: mode, gpkg: gpkg, scope: scope, names: names, out: out, path: path}

	if mode == ImInception {
		gen.reflect = "r."
		gen.name = gpkg.Name()
	}
	if mode == ImPlugin {
		gen.proxyprefix = "P_"
	} else {
		gen.proxyprefix = fmt.Sprintf("P_%s_", sanitizeIdent(path))
	}
	return gen
}

func (gen *genimport) write() {

	gen.writePreamble()

	gen.writeBinds()
	gen.writeTypes()
	gen.writeProxies()
	gen.writeUntypeds()
	gen.writeWrappers()

	gen.out.WriteString("\n\t}\n}\n")
	gen.writeInterfaceProxies()
}

type mapdecl struct {
	out  *bytes.Buffer
	head string
	foot string
}

func (gen *genimport) mapdecl(head string) mapdecl {
	if strings.IndexByte(head, '%') >= 0 {
		head = fmt.Sprintf(head, gen.reflect)
	}
	return mapdecl{gen.out, head, ""}
}

func (d *mapdecl) header() {
	if len(d.head) != 0 {
		d.out.WriteString(d.head)
		d.out.WriteByte('{')
		d.head = ""
		d.foot = "\n\t}"
	}
}

func (d *mapdecl) footer() {
	if len(d.foot) != 0 {
		d.out.WriteString(d.foot)
		d.out.WriteString(", ")
	}
}

func (d *mapdecl) footer1(comma bool) {
	if len(d.foot) != 0 {
		d.out.WriteString(d.foot)
		if comma {
			d.out.WriteString(", ")
		}
	}
}

func (gen *genimport) collectPackageImportsWithRename(requireAllInterfaceMethodsExported bool) {
	gen.pkgrenames = collectPackageImportsWithRename(gen.output, gen.gpkg, requireAllInterfaceMethodsExported)
	gen.name = gen.pkgrenames[gen.path]
	if gen.name == "" {
		gen.name = packageSanitizedName(gen.path)
	}
	if gen.mode != ImInception {
		gen.name_ = gen.name + "."
	}
}

func (gen *genimport) writePreamble() {
	mode := gen.mode
	out := gen.out

	var alias, filepkg string
	switch mode {
	case ImBuiltin:
		alias = "_b "
		filepkg = "imports"
	case ImThirdParty:
		filepkg = "thirdparty"
	case ImPlugin:
		filepkg = "main"
	case ImInception:
		alias = "_i "
		filepkg = gen.name
	}

	fmt.Fprintf(gen.out, `// this file was generated by gomacro command: import %s%q
// DO NOT EDIT! Any change will be lost when the file is re-generated

package %s

import (`, alias, gen.path, filepkg)

	var imports string
	if mode == ImInception {
		fmt.Fprintf(gen.out, "\n\tr \"reflect\"\n\t\"github.com/cosmos72/gomacro/imports\"")
		imports = "imports."
	} else {
		fmt.Fprintf(out, "\n\t. \"reflect\"")
	}
	gen.collectPackageImportsWithRename(true)
	for path, name := range gen.pkgrenames {
		if mode == ImInception && path == gen.path {
			continue // writing inside the package: it should not import itself
		} else {
			// always name the imported package: its name may differ from paths.FileName(path)
			fmt.Fprintf(out, "\n\t%s %q", name, path)
		}
	}
	fmt.Fprintf(out, "\n)\n")

	if mode == ImInception {
		gen.pkgrenames[gen.path] = "" // writing inside the package: remove the package prefix
	}

	if mode == ImPlugin {
		fmt.Fprint(out, `
type Package = struct {
	Name     string
	Binds    map[string]Value
	Types    map[string]Type
	Proxies  map[string]Type
	Untypeds map[string]string
	Wrappers map[string][]string
}

var Packages = make(map[string]Package)

func main() {
}

`)
	}

	fmt.Fprintf(out, `
// reflection: allow interpreted code to import %q
func init() {
	%sPackages[%q] = %sPackage{
	Name: %q,
	`, gen.path, imports, gen.path, imports, gen.gpkg.Name())
}

func (gen *genimport) writeBinds() {
	d := gen.mapdecl("Binds: map[string]%sValue")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj := obj.(type) {
			case *types.Const:
				val := obj.Val()
				var conv1, conv2 string
				if t, ok := obj.Type().(*types.Basic); ok && t.Info()&types.IsUntyped != 0 {
					// untyped constants have arbitrary precision... they may overflow integers.
					// this is just an approximation, use Package.Untypeds for exact value
					if val.Kind() == constant.Int {
						str := val.ExactString()
						conv1, conv2 = detectIntKind(gen.output, gen.path, name, str)
					}
				}
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(%s%s%s%s),", name, gen.reflect, conv1, gen.name_, name, conv2)
			case *types.Var:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(&%s%s).Elem(),", name, gen.reflect, gen.name_, name)
			case *types.Func:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sValueOf(%s%s),", name, gen.reflect, gen.name_, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeTypes() {
	d := gen.mapdecl("Types: map[string]%sType")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.TypeName:
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sTypeOf((*%s%s)(nil)).Elem(),", name, gen.reflect, gen.name_, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeProxies() {
	d := gen.mapdecl("Proxies: map[string]%sType")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			if t := extractInterface(obj, true); t != nil {
				d.header()
				fmt.Fprintf(gen.out, "\n\t\t%q:\t%sTypeOf((*%s%s)(nil)).Elem(),", name, gen.reflect, gen.proxyprefix, name)
			}
		}
	}
	d.footer()
}

func (gen *genimport) writeUntypeds() {
	d := gen.mapdecl("Untypeds: map[string]string")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj := obj.(type) {
			case *types.Const:
				if t, ok := obj.Type().(*types.Basic); ok && t.Info()&types.IsUntyped != 0 {
					kind := untyped.GoUntypedToKind(t.Kind())
					str := untyped.Marshal(kind, obj.Val())
					if len(str) != 0 {
						d.header()
						fmt.Fprintf(gen.out, "\n\t\t%q:\t%q,", name, str)
					}
				}
			}
		}
	}
	d.footer()
}

// find wrapper methods and write them. needed for accurate method selection.
func (gen *genimport) writeWrappers() {
	d := gen.mapdecl("Wrappers: map[string][]string")

	for _, name := range gen.names {
		if obj := gen.scope.Lookup(name); obj.Exported() {
			switch obj.(type) {
			case *types.TypeName:
				if t, ok := obj.Type().(*types.Named); ok {
					// only structs can have embedded fields, and thus wrapper methods for embedded fields
					if _, ok := t.Underlying().(*types.Struct); ok {
						wrappers := new(analyzer).Analyze(t)
						if len(wrappers) != 0 {
							d.header()
							fmt.Fprintf(gen.out, "\n\t\t%q:\t[]string{", obj.Name())
							for _, wrapper := range wrappers {
								fmt.Fprintf(gen.out, "%q,", wrapper)
							}
							fmt.Fprint(gen.out, "},")
						}
					}
				}
			}
		}
	}
	d.footer()
}

// write proxies that pre-implement package's interfaces
func (gen *genimport) writeInterfaceProxies() {
	path := gen.gpkg.Path()
	for _, name := range gen.names {
		obj := gen.scope.Lookup(name)
		if t := extractInterface(obj, true); t != nil {
			gen.writeInterfaceProxy(path, name, t)
		}
	}
}

func detectIntKind(o *Output, path, name, str string) (string, string) {
	i, err := strconv.ParseInt(str, 0, 64)
	if err == nil {
		if i == int64(int32(i)) {
			// constant fits int32. We can use the default (i.e. int)
			// on both 32-bit and 64-bit platforms
			return "", ""
		} else if i == int64(uint32(i)) {
			// constant fits uint32
			return "uint32(", ")"
		} else {
			return "int64(", ")"
		}
	}
	_, err = strconv.ParseUint(str, 0, 64)
	if err == nil {
		return "uint64(", ")"
	}
	f, err := strconv.ParseFloat(str, 64)
	if err != nil {
		// nothing fits... leave the default
		return "", ""
	} else {
		prefix := "float64"
		f = math.Abs(f)
		if f == float64(float32(f)) && f <= math.MaxFloat32 && f >= math.SmallestNonzeroFloat32 {
			// float32 loses no precision vs. float64
			prefix = "float32"
		}
		o.Warnf("package %q: integer constant %s = %s overflows both int64 and uint64, converting to %s", path, name, str, prefix)
		return prefix + "(", ")"
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/gob"
	"github.com/Symantec/Dominator/lib/errors"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/objectserver"
	"io"
)

func addObjects(conn *srpc.Conn, adder ObjectAdder, logger log.Logger) error {
	defer conn.Flush()
	decoder := gob.NewDecoder(conn)
	encoder := gob.NewEncoder(conn)
	numAdded := 0
	numObj := 0
	for ; ; numObj++ {
		var request objectserver.AddObjectRequest
		var response objectserver.AddObjectResponse
		if err := decoder.Decode(&request); err != nil {
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				break
			}
			return errors.New("error decoding: " + err.Error())
		}
		if request.Length < 1 {
			break
		}
		var err error
		response.Hash, response.Added, err =
			adder.AddObject(conn, request.Length, request.ExpectedHash)
		response.ErrorString = errors.ErrorToString(err)
		if err := encoder.Encode(response); err != nil {
			return errors.New("error encoding: " + err.Error())
		}
		if response.ErrorString != "" {
			logger.Printf(
				"AddObjects(): failed, %d of %d so far are new objects: %s",
				numAdded, numObj+1, response.ErrorString)
			return nil
		}
		if response.Added {
			numAdded++
		}
	}
	logger.Printf("AddObjects(): %d of %d are new objects", numAdded, numObj)
	return nil
}
<commit_msg>Add log message at start of objectserver/rpcd/lib.addObjects().<commit_after>package lib

import (
	"encoding/gob"
	"github.com/Symantec/Dominator/lib/errors"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/objectserver"
	"io"
)

func addObjects(conn *srpc.Conn, adder ObjectAdder, logger log.Logger) error {
	defer conn.Flush()
	logger.Printf("AddObjects(%s) starting\n", conn.RemoteAddr())
	decoder := gob.NewDecoder(conn)
	encoder := gob.NewEncoder(conn)
	numAdded := 0
	numObj := 0
	for ; ; numObj++ {
		var request objectserver.AddObjectRequest
		var response objectserver.AddObjectResponse
		if err := decoder.Decode(&request); err != nil {
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				break
			}
			return errors.New("error decoding: " + err.Error())
		}
		if request.Length < 1 {
			break
		}
		var err error
		response.Hash, response.Added, err =
			adder.AddObject(conn, request.Length, request.ExpectedHash)
		response.ErrorString = errors.ErrorToString(err)
		if err := encoder.Encode(response); err != nil {
			return errors.New("error encoding: " + err.Error())
		}
		if response.ErrorString != "" {
			logger.Printf(
				"AddObjects(): failed, %d of %d so far are new objects: %s",
				numAdded, numObj+1, response.ErrorString)
			return nil
		}
		if response.Added {
			numAdded++
		}
	}
	logger.Printf("AddObjects(): %d of %d are new objects", numAdded, numObj)
	return nil
}
<|endoftext|>


<commit_before>package cloudpelican

// @author Robin Verlangen
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net/http"
    "log"
)

// Settings
const ENDPOINT string = "https://app.cloudpelican.com/api/push/pixel"

// Write a message
func LogMessage(token string, msg string) bool {
    // @todo Write seperate func for url assembly, encoding, etc
    // @todo Validate
    var res bool = requestAsync(ENDPOINT + "?t=" + token + "&f[msg]=" + msg)
    return res
}

// Request a sync
// @todo Make sure all are pushed to backend before application shuts down
func requestAsync(url string) bool {
    go func() {
        _, err := http.Get(url)
        log.Println(url)
	if err != nil {
            log.Printf("Error while forwarding data: %s\n", err)
        }
    }()
    return true
}
<commit_msg>Working on drain function<commit_after>package cloudpelican

// @author Robin Verlangen
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net/http"
    "log"
)

// Settings
const ENDPOINT string = "https://app.cloudpelican.com/api/push/pixel"

// Monitor drain status
routineQuit chan int = make(chan int)
startCounter uint64 = uint64(0)
startCounterMux sync.Mutex
doneCounter uint64 = uint64(0)
doneCounterMux sync.Mutex

// Write a message
func LogMessage(token string, msg string) bool {
    // @todo Write seperate func for url assembly, encoding, etc
    // @todo Validate
    var res bool = requestAsync(ENDPOINT + "?t=" + token + "&f[msg]=" + msg)
    return res
}

// Drain: wait for all data pushes to finish
func Drain() bool {
    if startCounter > doneCounter {
        <-routineQuit
    }
}

// Request a sync
// @todo Make sure all are pushed to backend before application shuts down
func requestAsync(url string) bool {
    // Add counter
    startCounterMux.Lock()
    startcounter++
    startCounterMux.Unlock()

    // Launch
    go func() {
        // Request url
        _, err := http.Get(url)
        log.Println(url)
	if err != nil {
            log.Printf("Error while forwarding data: %s\n", err)
        }

        // Done counter
        doneCounterMux.Lock()
        doneCounter++
        doneCounter.Unlock()
        // @todo Check whether dif between started and done is = 0, if so, drop a message in the routineQuit
    }()
    return true
}
<|endoftext|>


<commit_before>package remote

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/ivanilves/lstags/docker"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/remote/auth"
	"github.com/ivanilves/lstags/util"
)

// ConcurrentRequests defines maximum number of concurrent requests we could maintain against the registry
var ConcurrentRequests = 32

// RetryRequests is a number of retries we do in case of request failure
var RetryRequests = 0

// RetryDelay is a delay between retries of failed requests to the registry
var RetryDelay = 5 * time.Second

// TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
var TraceRequests = false

func getRequestID() string {
	data := make([]byte, 10)

	for i := range data {
		data[i] = byte(rand.Intn(256))
	}

	return fmt.Sprintf("%x", sha256.Sum256(data))[0:7]
}

func httpResponseBody(resp *http.Response) string {
	b, _ := ioutil.ReadAll(resp.Body)
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(b))

	return string(b)
}

func httpRequest(url, authorization, mode string) (*http.Response, error) {
	hc := &http.Client{}
	rid := getRequestID()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", authorization)
	req.Header.Set("Accept", "application/json")

	switch mode {
	case "v1":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v1+json")
	case "v2":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
	default:
		return nil, errors.New("Unknown request mode: " + mode)
	}

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return resp, errors.New("Bad response status: " + resp.Status + " >> " + url)
	}

	if TraceRequests {
		fmt.Printf("%s|@URL: %s\n", rid, url)
		for k, v := range resp.Header {
			fmt.Printf("%s|@HEADER: %-40s = %s\n", rid, k, v)
		}
		fmt.Printf("%s|--- BODY BEGIN ---\n", rid)
		for _, line := range strings.Split(httpResponseBody(resp), "\n") {
			fmt.Printf("%s|%s\n", rid, line)
		}
		fmt.Printf("%s|--- BODY END ---\n", rid)
	}

	return resp, nil
}

func httpRetriableRequest(url, authorization, mode string) (*http.Response, error) {
	tries := 1

	if RetryRequests > 0 {
		tries = tries + RetryRequests
	}

	var resp *http.Response
	var err error

	for try := 1; try <= tries; try++ {
		resp, err := httpRequest(url, authorization, mode)

		if err == nil {
			return resp, nil
		}

		if resp.StatusCode >= 400 && resp.StatusCode < 500 {
			return nil, err
		}

		if try < tries {
			fmt.Printf(
				"Will retry '%s' [%s] in a %v\n=> Error: %s\n",
				url,
				mode,
				RetryDelay,
				err.Error(),
			)

			time.Sleep(RetryDelay)
		}
	}

	return resp, err
}

type tagNameInfo struct {
	TagNames []string `json:"tags"`
}

func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
	tn := tagNameInfo{}

	err := json.NewDecoder(data).Decode(&tn)
	if err != nil {
		return nil, err
	}

	return tn.TagNames, nil
}

func fetchTagNames(registry, repoPath, authorization string) ([]string, error) {
	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/tags/list"

	resp, err := httpRetriableRequest(url, authorization, "v2")
	if err != nil {
		return nil, err
	}

	tagNames, err := parseTagNamesJSON(resp.Body)
	if err != nil {
		return make([]string, 0), nil
	}

	return tagNames, nil
}

type imageMetadata struct {
	Created     int64
	ContainerID string
}

func extractMetadataFromHistory(s string) (imageMetadata, error) {
	var history struct {
		Created     string `json:"created"`
		ContainerID string `json:"container"`
	}

	err := json.Unmarshal([]byte(s), &history)
	if err != nil {
		return imageMetadata{}, err
	}

	t, err := time.Parse(time.RFC3339, history.Created)

	return imageMetadata{t.Unix(), history.ContainerID}, nil
}

func fetchMetadata(url, authorization string) (imageMetadata, error) {
	resp, err := httpRetriableRequest(url, authorization, "v1")
	if err != nil {
		return imageMetadata{}, nil
	}

	var v1manifest struct {
		History []map[string]string `json:"history"`
	}

	decodingError := json.NewDecoder(resp.Body).Decode(&v1manifest)
	if decodingError != nil {
		return imageMetadata{}, decodingError
	}

	if len(v1manifest.History) > 0 {
		metadata, err := extractMetadataFromHistory(v1manifest.History[0]["v1Compatibility"])
		if err != nil {
			return imageMetadata{}, err
		}

		return metadata, nil
	}

	return imageMetadata{}, errors.New("no source to fetch image creation date/time from")
}

func fetchDigest(url, authorization string) (string, error) {
	resp, err := httpRetriableRequest(url, authorization, "v2")
	if err != nil {
		return "", err
	}

	digests, defined := resp.Header["Docker-Content-Digest"]
	if !defined {
		return "", errors.New("header 'Docker-Content-Digest' not found in HTTP response")
	}

	return digests[0], nil
}

func fetchDetails(registry, repoPath, tagName, authorization string) (string, imageMetadata, error) {
	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/manifests/" + tagName

	dc := make(chan string, 0)
	mc := make(chan imageMetadata, 0)
	ec := make(chan error, 0)

	go func(url, authorization string, dc chan string, ec chan error) {
		digest, err := fetchDigest(url, authorization)
		if err != nil {
			ec <- err
		}

		dc <- digest
	}(url, authorization, dc, ec)

	go func(url, authorization string, mc chan imageMetadata, ec chan error) {
		metadata, err := fetchMetadata(url, authorization)
		if err != nil {
			ec <- err
		}

		mc <- metadata
	}(url, authorization, mc, ec)

	var digest string
	var metadata imageMetadata

	waitForDigest := true
	waitForMetadata := true
	for waitForDigest || waitForMetadata {
		select {
		case digest = <-dc:
			waitForDigest = false
		case metadata = <-mc:
			waitForMetadata = false
		case err := <-ec:
			if err != nil {
				return "", imageMetadata{}, err
			}
		}
	}

	return digest, metadata, nil
}

type detailResponse struct {
	TagName     string
	Digest      string
	Created     int64
	ContainerID string
	Error       error
}

func validateConcurrentRequests() (int, error) {
	const min = 1
	const max = 128

	if ConcurrentRequests < min {
		return 0, errors.New("Concurrent requests limit could not be lower than " + strconv.Itoa(min))
	}

	if ConcurrentRequests > max {
		return 0, errors.New("Concurrent requests limit could not be higher than " + strconv.Itoa(max))
	}

	return ConcurrentRequests, nil
}

func calculateBatchSteps(count, limit int) (int, int) {
	total := count / limit
	remain := count % limit

	if remain == 0 {
		return total, 0
	}

	return total + 1, remain
}

func calculateBatchStepSize(stepNumber, stepsTotal, remain, limit int) int {
	if remain != 0 && stepNumber == stepsTotal {
		return remain
	}

	return limit
}

// FetchTags looks up Docker repoPath tags present on remote Docker registry
func FetchTags(registry, repoPath, filter, username, password string) (map[string]*tag.Tag, error) {
	batchLimit, err := validateConcurrentRequests()
	if err != nil {
		return nil, err
	}

	tr, err := auth.NewToken(registry, repoPath, username, password)
	if err != nil {
		return nil, err
	}

	authorization := tr.AuthHeader()

	allTagNames, err := fetchTagNames(registry, repoPath, authorization)
	if err != nil {
		return nil, err
	}

	tagNames := make([]string, 0)
	for _, tagName := range allTagNames {
		if util.DoesMatch(tagName, filter) {
			tagNames = append(tagNames, tagName)
		}
	}

	tags := make(map[string]*tag.Tag)

	batchSteps, batchRemain := calculateBatchSteps(len(tagNames), batchLimit)

	var stepSize int
	var tagIndex = 0
	for b := 1; b <= batchSteps; b++ {
		stepSize = calculateBatchStepSize(b, batchSteps, batchRemain, batchLimit)

		ch := make(chan detailResponse, stepSize)

		for s := 1; s <= stepSize; s++ {
			go func(registry, repoPath, tagName, authorization string, ch chan detailResponse) {
				digest, metadata, err := fetchDetails(registry, repoPath, tagName, authorization)

				ch <- detailResponse{TagName: tagName, Digest: digest, Created: metadata.Created, ContainerID: metadata.ContainerID, Error: err}
			}(registry, repoPath, tagNames[tagIndex], authorization, ch)

			tagIndex++
		}

		for s := 1; s <= stepSize; s++ {
			dr := <-ch

			if dr.Error != nil {
				if strings.Contains(dr.Error.Error(), "404 Not Found") {
					println(dr.Error.Error())

					continue
				}

				return nil, dr.Error
			}

			tt, err := tag.New(dr.TagName, dr.Digest)
			if err != nil {
				return nil, err
			}

			tt.SetCreated(dr.Created)
			tt.SetContainerID(dr.ContainerID)

			tags[tt.GetName()] = tt
		}
	}

	return tags, nil
}
<commit_msg>Added backoff delay to save the Internet!<commit_after>package remote

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/ivanilves/lstags/docker"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/remote/auth"
	"github.com/ivanilves/lstags/util"
)

// ConcurrentRequests defines maximum number of concurrent requests we could maintain against the registry
var ConcurrentRequests = 32

// RetryRequests is a number of retries we do in case of request failure
var RetryRequests = 0

// RetryDelay is a delay between retries of failed requests to the registry
var RetryDelay = 5 * time.Second

// TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
var TraceRequests = false

func getRequestID() string {
	data := make([]byte, 10)

	for i := range data {
		data[i] = byte(rand.Intn(256))
	}

	return fmt.Sprintf("%x", sha256.Sum256(data))[0:7]
}

func httpResponseBody(resp *http.Response) string {
	b, _ := ioutil.ReadAll(resp.Body)
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(b))

	return string(b)
}

func httpRequest(url, authorization, mode string) (*http.Response, error) {
	hc := &http.Client{}
	rid := getRequestID()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", authorization)
	req.Header.Set("Accept", "application/json")

	switch mode {
	case "v1":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v1+json")
	case "v2":
		req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
	default:
		return nil, errors.New("Unknown request mode: " + mode)
	}

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return resp, errors.New("Bad response status: " + resp.Status + " >> " + url)
	}

	if TraceRequests {
		fmt.Printf("%s|@URL: %s\n", rid, url)
		for k, v := range resp.Header {
			fmt.Printf("%s|@HEADER: %-40s = %s\n", rid, k, v)
		}
		fmt.Printf("%s|--- BODY BEGIN ---\n", rid)
		for _, line := range strings.Split(httpResponseBody(resp), "\n") {
			fmt.Printf("%s|%s\n", rid, line)
		}
		fmt.Printf("%s|--- BODY END ---\n", rid)
	}

	return resp, nil
}

func httpRetriableRequest(url, authorization, mode string) (*http.Response, error) {
	tries := 1

	if RetryRequests > 0 {
		tries = tries + RetryRequests
	}

	var resp *http.Response
	var err error

	for try := 1; try <= tries; try++ {
		resp, err := httpRequest(url, authorization, mode)

		if err == nil {
			return resp, nil
		}

		if resp.StatusCode >= 400 && resp.StatusCode < 500 {
			return nil, err
		}

		if try < tries {
			fmt.Printf(
				"Will retry '%s' [%s] in a %v\n=> Error: %s\n",
				url,
				mode,
				RetryDelay,
				err.Error(),
			)

			time.Sleep(RetryDelay)

			RetryDelay += RetryDelay
		}
	}

	return resp, err
}

type tagNameInfo struct {
	TagNames []string `json:"tags"`
}

func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
	tn := tagNameInfo{}

	err := json.NewDecoder(data).Decode(&tn)
	if err != nil {
		return nil, err
	}

	return tn.TagNames, nil
}

func fetchTagNames(registry, repoPath, authorization string) ([]string, error) {
	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/tags/list"

	resp, err := httpRetriableRequest(url, authorization, "v2")
	if err != nil {
		return nil, err
	}

	tagNames, err := parseTagNamesJSON(resp.Body)
	if err != nil {
		return make([]string, 0), nil
	}

	return tagNames, nil
}

type imageMetadata struct {
	Created     int64
	ContainerID string
}

func extractMetadataFromHistory(s string) (imageMetadata, error) {
	var history struct {
		Created     string `json:"created"`
		ContainerID string `json:"container"`
	}

	err := json.Unmarshal([]byte(s), &history)
	if err != nil {
		return imageMetadata{}, err
	}

	t, err := time.Parse(time.RFC3339, history.Created)

	return imageMetadata{t.Unix(), history.ContainerID}, nil
}

func fetchMetadata(url, authorization string) (imageMetadata, error) {
	resp, err := httpRetriableRequest(url, authorization, "v1")
	if err != nil {
		return imageMetadata{}, nil
	}

	var v1manifest struct {
		History []map[string]string `json:"history"`
	}

	decodingError := json.NewDecoder(resp.Body).Decode(&v1manifest)
	if decodingError != nil {
		return imageMetadata{}, decodingError
	}

	if len(v1manifest.History) > 0 {
		metadata, err := extractMetadataFromHistory(v1manifest.History[0]["v1Compatibility"])
		if err != nil {
			return imageMetadata{}, err
		}

		return metadata, nil
	}

	return imageMetadata{}, errors.New("no source to fetch image creation date/time from")
}

func fetchDigest(url, authorization string) (string, error) {
	resp, err := httpRetriableRequest(url, authorization, "v2")
	if err != nil {
		return "", err
	}

	digests, defined := resp.Header["Docker-Content-Digest"]
	if !defined {
		return "", errors.New("header 'Docker-Content-Digest' not found in HTTP response")
	}

	return digests[0], nil
}

func fetchDetails(registry, repoPath, tagName, authorization string) (string, imageMetadata, error) {
	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/manifests/" + tagName

	dc := make(chan string, 0)
	mc := make(chan imageMetadata, 0)
	ec := make(chan error, 0)

	go func(url, authorization string, dc chan string, ec chan error) {
		digest, err := fetchDigest(url, authorization)
		if err != nil {
			ec <- err
		}

		dc <- digest
	}(url, authorization, dc, ec)

	go func(url, authorization string, mc chan imageMetadata, ec chan error) {
		metadata, err := fetchMetadata(url, authorization)
		if err != nil {
			ec <- err
		}

		mc <- metadata
	}(url, authorization, mc, ec)

	var digest string
	var metadata imageMetadata

	waitForDigest := true
	waitForMetadata := true
	for waitForDigest || waitForMetadata {
		select {
		case digest = <-dc:
			waitForDigest = false
		case metadata = <-mc:
			waitForMetadata = false
		case err := <-ec:
			if err != nil {
				return "", imageMetadata{}, err
			}
		}
	}

	return digest, metadata, nil
}

type detailResponse struct {
	TagName     string
	Digest      string
	Created     int64
	ContainerID string
	Error       error
}

func validateConcurrentRequests() (int, error) {
	const min = 1
	const max = 128

	if ConcurrentRequests < min {
		return 0, errors.New("Concurrent requests limit could not be lower than " + strconv.Itoa(min))
	}

	if ConcurrentRequests > max {
		return 0, errors.New("Concurrent requests limit could not be higher than " + strconv.Itoa(max))
	}

	return ConcurrentRequests, nil
}

func calculateBatchSteps(count, limit int) (int, int) {
	total := count / limit
	remain := count % limit

	if remain == 0 {
		return total, 0
	}

	return total + 1, remain
}

func calculateBatchStepSize(stepNumber, stepsTotal, remain, limit int) int {
	if remain != 0 && stepNumber == stepsTotal {
		return remain
	}

	return limit
}

// FetchTags looks up Docker repoPath tags present on remote Docker registry
func FetchTags(registry, repoPath, filter, username, password string) (map[string]*tag.Tag, error) {
	batchLimit, err := validateConcurrentRequests()
	if err != nil {
		return nil, err
	}

	tr, err := auth.NewToken(registry, repoPath, username, password)
	if err != nil {
		return nil, err
	}

	authorization := tr.AuthHeader()

	allTagNames, err := fetchTagNames(registry, repoPath, authorization)
	if err != nil {
		return nil, err
	}

	tagNames := make([]string, 0)
	for _, tagName := range allTagNames {
		if util.DoesMatch(tagName, filter) {
			tagNames = append(tagNames, tagName)
		}
	}

	tags := make(map[string]*tag.Tag)

	batchSteps, batchRemain := calculateBatchSteps(len(tagNames), batchLimit)

	var stepSize int
	var tagIndex = 0
	for b := 1; b <= batchSteps; b++ {
		stepSize = calculateBatchStepSize(b, batchSteps, batchRemain, batchLimit)

		ch := make(chan detailResponse, stepSize)

		for s := 1; s <= stepSize; s++ {
			go func(registry, repoPath, tagName, authorization string, ch chan detailResponse) {
				digest, metadata, err := fetchDetails(registry, repoPath, tagName, authorization)

				ch <- detailResponse{TagName: tagName, Digest: digest, Created: metadata.Created, ContainerID: metadata.ContainerID, Error: err}
			}(registry, repoPath, tagNames[tagIndex], authorization, ch)

			tagIndex++
		}

		for s := 1; s <= stepSize; s++ {
			dr := <-ch

			if dr.Error != nil {
				if strings.Contains(dr.Error.Error(), "404 Not Found") {
					println(dr.Error.Error())

					continue
				}

				return nil, dr.Error
			}

			tt, err := tag.New(dr.TagName, dr.Digest)
			if err != nil {
				return nil, err
			}

			tt.SetCreated(dr.Created)
			tt.SetContainerID(dr.ContainerID)

			tags[tt.GetName()] = tt
		}
	}

	return tags, nil
}
<|endoftext|>


<commit_before>// Package shell implements the shell command.
package shell

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"time"

	isatty "github.com/mattn/go-isatty"
	"github.com/spf13/cobra"
	"github.com/taskcluster/taskcluster-cli/cmds/root"
	"github.com/taskcluster/taskcluster-cli/config"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
	"github.com/taskcluster/taskcluster-worker/engines"
	v2client "github.com/taskcluster/taskcluster-worker/plugins/interactive/shellclient"
	"github.com/taskcluster/taskcluster-worker/runtime/ioext"
)

var (
	// Command is the root of the shell sub-tree.
	Command = &cobra.Command{
		Use:   "shell <taskId>",
		Short: "Connect to the shell of a running interactive task.",
		RunE:  Execute,
	}
)

func init() {
	root.Command.AddCommand(Command)
}

// Execute runs the shell.
func Execute(cmd *cobra.Command, args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("%s expects argument <taskId>", cmd.Name())
	}

	taskID := args[0]

	var creds *tcclient.Credentials
	if config.Credentials != nil {
		creds = config.Credentials.ToClientCredentials()
	}

	q := queue.New(creds)

	err := checkTask(q, taskID)
	if err != nil {
		return err
	}

	// At this point we know we have a valid task with interactivity.
	sURL, err := q.GetLatestArtifact_SignedURL(taskID, "private/docker-worker/shell.html", 1*time.Minute)
	if err != nil {
		return err
	}

	// client is an HTTP client that doesn't follow redirects.
	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}

	resp, err := client.Get(sURL.String())
	if err != nil {
		return err
	}
	redirectURL, err := resp.Location()
	if err != nil {
		return err
	}

	var sockURL *url.URL
	var shell engines.Shell
	tty := isatty.IsTerminal(os.Stdout.Fd())

	switch redirectURL.Query().Get("v") {
	case "1":
		return errors.New("the shell client doens't yet support v1 shells")
	case "2":
		sockURL, _ = url.Parse(redirectURL.Query().Get("socketUrl"))
		shell, err = v2client.Dial(sockURL.String(), []string{"bash"}, tty)
		if err != nil {
			return fmt.Errorf("could not create the shell client: %v", err)
		}
	default:
		return errors.New("unknown shell version")
	}

	// Switch terminal to raw mode
	cleanup := func() {}
	if tty {
		cleanup = setupRawTerminal(shell.SetSize)
	}

	// Connect pipes
	go ioext.CopyAndClose(shell.StdinPipe(), os.Stdin)
	go io.Copy(os.Stdout, shell.StdoutPipe())
	go io.Copy(os.Stderr, shell.StderrPipe())

	// Wait for shell to be done
	_, err = shell.Wait()

	// If we were in a tty we let's restore state
	cleanup()

	return err
}

// checkTask makes sure that the given task is interactive and that we can connect.
func checkTask(q *queue.Queue, taskID string) error {
	task, err := q.Task(taskID)
	if err != nil {
		return fmt.Errorf("could not get the definition of task %s: %v", taskID, err)
	}
	var payload map[string]json.RawMessage
	json.Unmarshal(task.Payload, &payload)
	if _, ok := payload["features"]; !ok {
		return fmt.Errorf("task %s was created without features.interactive", taskID)
	}
	var features map[string]json.RawMessage
	json.Unmarshal(payload["features"], &features)
	if _, ok := features["interactive"]; !ok {
		return fmt.Errorf("task %s was created without features.interactive", taskID)
	}
	var interactive bool
	json.Unmarshal(features["interactive"], &interactive)
	if !interactive {
		return fmt.Errorf("task %s was created without features.interactive = true", taskID)
	}

	s, err := q.Status(taskID)
	if err != nil {
		return fmt.Errorf("could not get the status of task %s: %v", taskID, err)
	}
	lastRunState := s.Status.Runs[len(s.Status.Runs)-1].State
	lastRunDeadline := time.Time(s.Status.Runs[len(s.Status.Runs)-1].Resolved).Add(15 * time.Minute)
	if !(lastRunState == "running" || (lastRunState == "completed" && lastRunDeadline.After(time.Now().UTC()))) {
		return fmt.Errorf("task %s is not running and was not completed in the last 15 minutes", taskID)
	}

	return nil
}
<commit_msg>Added support for the v1 client in shell command<commit_after>// Package shell implements the shell command.
package shell

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"time"

	isatty "github.com/mattn/go-isatty"
	"github.com/spf13/cobra"
	"github.com/taskcluster/taskcluster-cli/cmds/root"
	"github.com/taskcluster/taskcluster-cli/config"
	v1client "github.com/taskcluster/taskcluster-cli/external/pkg/docker-exec-ws"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
	"github.com/taskcluster/taskcluster-worker/engines"
	v2client "github.com/taskcluster/taskcluster-worker/plugins/interactive/shellclient"
	"github.com/taskcluster/taskcluster-worker/runtime/ioext"
)

var (
	// Command is the root of the shell sub-tree.
	Command = &cobra.Command{
		Use:   "shell <taskId>",
		Short: "Connect to the shell of a running interactive task.",
		RunE:  Execute,
	}
)

func init() {
	root.Command.AddCommand(Command)
}

// Execute runs the shell.
func Execute(cmd *cobra.Command, args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("%s expects argument <taskId>", cmd.Name())
	}

	taskID := args[0]

	var creds *tcclient.Credentials
	if config.Credentials != nil {
		creds = config.Credentials.ToClientCredentials()
	}

	q := queue.New(creds)

	err := checkTask(q, taskID)
	if err != nil {
		return err
	}

	// At this point we know we have a valid task with interactivity.
	sURL, err := q.GetLatestArtifact_SignedURL(taskID, "private/docker-worker/shell.html", 1*time.Minute)
	if err != nil {
		return err
	}

	// client is an HTTP client that doesn't follow redirects.
	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}

	resp, err := client.Get(sURL.String())
	if err != nil {
		return err
	}
	redirectURL, err := resp.Location()
	if err != nil {
		return err
	}

	var sockURL *url.URL
	var shell engines.Shell
	tty := isatty.IsTerminal(os.Stdout.Fd())

	switch redirectURL.Query().Get("v") {
	case "1":
		sockURL, _ = url.Parse(redirectURL.Query().Get("socketUrl"))
		shell, err = v1client.Dial(sockURL.String(), []string{"bash"}, tty)
		if err != nil {
			return fmt.Errorf("could not create the shell client: %v", err)
		}
	case "2":
		sockURL, _ = url.Parse(redirectURL.Query().Get("socketUrl"))
		shell, err = v2client.Dial(sockURL.String(), []string{"bash"}, tty)
		if err != nil {
			return fmt.Errorf("could not create the shell client: %v", err)
		}
	default:
		return errors.New("unknown shell version")
	}

	// Switch terminal to raw mode
	cleanup := func() {}
	if tty {
		cleanup = setupRawTerminal(shell.SetSize)
	}

	// Connect pipes
	go ioext.CopyAndClose(shell.StdinPipe(), os.Stdin)
	go io.Copy(os.Stdout, shell.StdoutPipe())
	go io.Copy(os.Stderr, shell.StderrPipe())

	// Wait for shell to be done
	_, err = shell.Wait()

	// If we were in a tty we let's restore state
	cleanup()

	return err
}

// checkTask makes sure that the given task is interactive and that we can connect.
func checkTask(q *queue.Queue, taskID string) error {
	task, err := q.Task(taskID)
	if err != nil {
		return fmt.Errorf("could not get the definition of task %s: %v", taskID, err)
	}
	var payload map[string]json.RawMessage
	json.Unmarshal(task.Payload, &payload)
	if _, ok := payload["features"]; !ok {
		return fmt.Errorf("task %s was created without features.interactive", taskID)
	}
	var features map[string]json.RawMessage
	json.Unmarshal(payload["features"], &features)
	if _, ok := features["interactive"]; !ok {
		return fmt.Errorf("task %s was created without features.interactive", taskID)
	}
	var interactive bool
	json.Unmarshal(features["interactive"], &interactive)
	if !interactive {
		return fmt.Errorf("task %s was created without features.interactive = true", taskID)
	}

	s, err := q.Status(taskID)
	if err != nil {
		return fmt.Errorf("could not get the status of task %s: %v", taskID, err)
	}
	lastRunState := s.Status.Runs[len(s.Status.Runs)-1].State
	lastRunDeadline := time.Time(s.Status.Runs[len(s.Status.Runs)-1].Resolved).Add(15 * time.Minute)
	if !(lastRunState == "running" || (lastRunState == "completed" && lastRunDeadline.After(time.Now().UTC()))) {
		return fmt.Errorf("task %s is not running and was not completed in the last 15 minutes", taskID)
	}

	return nil
}
<|endoftext|>