<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/url"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
)

var (
	migConfigFlag           config.MigConfigFlag
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("experimental-scale-down-enabled", false, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	gceManager, err := gce.CreateGceManager(migConfigs)
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				if *verifyUnschedulablePods {
					unschedulablePodsToHelp = FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)
					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now())

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						kubeClient,
						predicateChecker)

					if !calculateUtilizationOnly {
						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result != ScaleDownNodeDeleted {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
			}
		}
	}
}
<commit_msg>Cluster-autoscaler: clear utilization map on node deletion<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/url"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
)

var (
	migConfigFlag           config.MigConfigFlag
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("experimental-scale-down-enabled", false, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	gceManager, err := gce.CreateGceManager(migConfigs)
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				if *verifyUnschedulablePods {
					unschedulablePodsToHelp = FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)
					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now())

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						kubeClient,
						predicateChecker)

					if !calculateUtilizationOnly {
						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"fmt"
	"testing"
	"time"

	MQTT "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/mocks"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	. "github.com/TheThingsNetwork/ttn/utils/errors/checks"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	"github.com/brocaar/lorawan"
)

const brokerURL = "0.0.0.0:1883"

func TestMQTTSend(t *testing.T) {
	timeout = 100 * time.Millisecond

	tests := []struct {
		Desc       string          // Test Description
		Packet     []byte          // Handy representation of the packet to send
		Recipients []testRecipient // List of recipient to send

		WantData     []byte  // Expected Data on the recipient
		WantResponse []byte  // Expected Response from the Send method
		WantError    *string // Expected error nature returned by the Send method
	}{
		{
			Desc:   "1 packet | 1 recipient | No response",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 1 recipient | No down topic",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "invalid packet | 1 recipient | No response",
			Packet: nil,
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
			},

			WantData:     nil,
			WantResponse: nil,
			WantError:    pointer.String(string(errors.Structural)),
		},
		{
			Desc:   "1 packet | 2 recipient | No response",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  nil,
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 2 recipients | #1 answer ",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  nil,
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: []byte("IoT Rocks"),
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 2 recipients | both answers ",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    pointer.String(string(errors.Behavioural)),
		},
	}

	for i, test := range tests {
		// Describe
		Desc(t, fmt.Sprintf("#%d: %s", i, test.Desc))

		// Build
		aclient, adapter := createAdapter(t)
		sclients, chresp := createServers(test.Recipients)
		<-time.After(time.Millisecond * 50)

		// Operate
		resp, err := trySend(adapter, test.Packet, test.Recipients)
		var data []byte
		select {
		case data = <-chresp:
		case <-time.After(time.Millisecond * 100):
		}

		// Check
		CheckErrors(t, test.WantError, err)
		checkData(t, test.WantData, data)
		checkResponses(t, test.WantResponse, resp)

		// Clean
		<-time.After(time.Millisecond * 500)
		aclient.Disconnect(0)
		for _, sclient := range sclients {
			sclient.Disconnect(0)
		}
	}
}

func TestSendErrorCases(t *testing.T) {
	tests := []struct {
		Desc       string          // Test Description
		Packet     []byte          // Handy representation of the packet to send
		Recipients []testRecipient // List of recipient to send
		Client     *MockClient     // A mocked version of the client

		WantData  []byte  // Expected Data on the recipient
		WantError *string // Expected error nature returned by the Send method
	}{
		{
			Desc:   "1 packet | 1 Recipient | Error on publish",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Publish"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  []byte("TheThingsNetwork"),
			WantError: pointer.String(string(errors.Operational)),
		},
		{
			Desc:   "1 packet | 1 Recipient | Error on Subscribe",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Subscribe"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  nil,
			WantError: pointer.String(string(errors.Operational)),
		},
		{
			Desc:   "1 packet | 1 Recipient | Error on Unsubscribe",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Unsubscribe"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  []byte("TheThingsNetwork"),
			WantError: nil,
		},
	}

	for i, test := range tests {
		// Describe
		Desc(t, fmt.Sprintf("#%d: %s", i, test.Desc))

		// Build
		adapter := NewAdapter(test.Client, GetLogger(t, "Adapter"))

		// Operate
		_, err := trySend(adapter, test.Packet, test.Recipients)

		// Check
		CheckErrors(t, test.WantError, err)
		checkData(t, test.WantData, test.Client.InPublish.Payload())
	}
}

func TestOtherMethods(t *testing.T) {
	{
		// Describe
		Desc(t, "Get Recipient | Wrong data")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))

		// Operate
		_, err := adapter.GetRecipient([]byte{})

		// Check
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Get Recipient | Valid Recipient")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))
		data, _ := NewRecipient("up", "down").MarshalBinary()

		// Operate
		recipient, err := adapter.GetRecipient(data)

		// Check
		CheckErrors(t, nil, err)
		checkRecipients(t, NewRecipient("up", "down"), recipient)

	}

	// --------------------

	{
		// Describe
		Desc(t, "Send invalid recipients")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))

		// Operate
		_, err := adapter.Send(mocks.NewMockPacket(), mocks.NewMockRecipient())

		// Check
		CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler")

		// Build
		client := NewMockClient()
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()
		msg := MockMessage{
			topic:   "MessageTopic",
			payload: []byte{1, 2, 3, 4},
		}

		// Operate
		err := adapter.Bind(handler)
		client.InSubscribeCallBack(client, msg)

		// Check
		CheckErrors(t, nil, err)
		checkMessages(t, msg, handler.InMessage)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler | fails to handle")

		// Build
		client := NewMockClient()
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()
		handler.Failures["Handle"] = errors.New(errors.Operational, "Mock Error")
		msg := MockMessage{
			topic:   "MessageTopic",
			payload: []byte{1, 2, 3, 4},
		}

		// Operate
		err := adapter.Bind(handler)
		client.InSubscribeCallBack(client, msg)

		// Check
		CheckErrors(t, nil, err)
		checkMessages(t, msg, handler.InMessage)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler | fails to subscribe")

		// Build
		client := NewMockClient("Subscribe")
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()

		// Operate
		err := adapter.Bind(handler)

		// Check
		CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}
}

func TestMQTTRecipient(t *testing.T) {
	{
		Desc(t, "Marshal / Unmarshal valid recipient")
		rm := NewRecipient("topicup", "topicdown")
		ru := new(recipient)
		data, err := rm.MarshalBinary()
		if err == nil {
			err = ru.UnmarshalBinary(data)
		}
		CheckErrors(t, nil, err)
	}

	{
		Desc(t, "Unmarshal from nil pointer")
		rm := NewRecipient("topicup", "topicdown")
		var ru *recipient
		data, err := rm.MarshalBinary()
		if err == nil {
			err = ru.UnmarshalBinary(data)
		}
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	{
		Desc(t, "Unmarshal nil data")
		ru := new(recipient)
		err := ru.UnmarshalBinary(nil)
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	{
		Desc(t, "Unmarshal wrong data")
		ru := new(recipient)
		err := ru.UnmarshalBinary([]byte{1, 2, 3, 4})
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}
}

// ----- TYPE utilities
type testRecipient struct {
	Response  []byte
	TopicUp   string
	TopicDown string
}

type testPacket struct {
	payload []byte
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p testPacket) MarshalBinary() ([]byte, error) {
	if p.payload == nil {
		return nil, errors.New(errors.Structural, "Fake error")
	}

	return p.payload, nil
}

// String implements the core.Packet interface
func (p testPacket) String() string {
	return string(p.payload)
}

// DevEUI implements the core.Packet interface
func (p testPacket) DevEUI() lorawan.EUI64 {
	return lorawan.EUI64{}
}

// ----- BUILD utilities
func createAdapter(t *testing.T) (Client, core.Adapter) {
	client, err := NewClient("testClient", brokerURL, TCP)
	if err != nil {
		panic(err)
	}

	adapter := NewAdapter(client, GetLogger(t, "adapter"))
	return client, adapter
}

func createServers(recipients []testRecipient) ([]Client, chan []byte) {
	var clients []Client
	chresp := make(chan []byte, len(recipients))
	for i, r := range recipients {
		client, err := NewClient(fmt.Sprintf("FakeServerClient%d", i), brokerURL, TCP)
		if err != nil {
			panic(err)
		}
		clients = append(clients, client)

		go func(r testRecipient, client Client) {
			token := client.Subscribe(r.TopicUp, 2, func(client Client, msg MQTT.Message) {
				if r.Response != nil {
					token := client.Publish(r.TopicDown, 2, false, r.Response)
					if token.Wait() && token.Error() != nil {
						panic(token.Error())
					}
				}
				chresp <- msg.Payload()
			})
			if token.Wait() && token.Error() != nil {
				panic(token.Error())
			}
		}(r, client)
	}
	return clients, chresp
}

// ----- OPERATE utilities
func trySend(adapter core.Adapter, packet []byte, recipients []testRecipient) ([]byte, error) {
	// Convert testRecipient to core.Recipient using the mqtt recipient
	var coreRecipients []core.Recipient
	for _, r := range recipients {
		coreRecipients = append(coreRecipients, NewRecipient(r.TopicUp, r.TopicDown))
	}

	// Try send the packet
	chresp := make(chan struct {
		Data  []byte
		Error error
	})
	go func() {
		data, err := adapter.Send(testPacket{packet}, coreRecipients...)
		chresp <- struct {
			Data  []byte
			Error error
		}{data, err}
	}()

	select {
	case resp := <-chresp:
		return resp.Data, resp.Error
	case <-time.After(timeout + time.Millisecond*100):
		return nil, nil
	}
}

// ----- CHECK utilities
func checkResponses(t *testing.T, want []byte, got []byte) {
	mocks.Check(t, want, got, "Responses")
}

func checkData(t *testing.T, want []byte, got []byte) {
	mocks.Check(t, want, got, "Data")
}

func checkRecipients(t *testing.T, want core.Recipient, got core.Recipient) {
	mocks.Check(t, want, got, "Recipients")
}

func checkMessages(t *testing.T, want MQTT.Message, got MQTT.Message) {
	mocks.Check(t, want, got, "Messages")
}
<commit_msg>[hotfix-paho] [hotfix] Use of unique IDs for mock Cient in MQTT<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"fmt"
	"testing"
	"time"

	MQTT "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/mocks"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	. "github.com/TheThingsNetwork/ttn/utils/errors/checks"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	"github.com/brocaar/lorawan"
)

const brokerURL = "0.0.0.0:1883"

func TestMQTTSend(t *testing.T) {
	timeout = 100 * time.Millisecond

	tests := []struct {
		Desc       string          // Test Description
		Packet     []byte          // Handy representation of the packet to send
		Recipients []testRecipient // List of recipient to send

		WantData     []byte  // Expected Data on the recipient
		WantResponse []byte  // Expected Response from the Send method
		WantError    *string // Expected error nature returned by the Send method
	}{
		{
			Desc:   "1 packet | 1 recipient | No response",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 1 recipient | No down topic",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "invalid packet | 1 recipient | No response",
			Packet: nil,
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
			},

			WantData:     nil,
			WantResponse: nil,
			WantError:    pointer.String(string(errors.Structural)),
		},
		{
			Desc:   "1 packet | 2 recipient | No response",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  nil,
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 2 recipients | #1 answer ",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  nil,
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: []byte("IoT Rocks"),
			WantError:    nil,
		},
		{
			Desc:   "1 packet | 2 recipients | both answers ",
			Packet: []byte("TheThingsNetwork"),
			Recipients: []testRecipient{
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up1",
					TopicDown: "down1",
				},
				{
					Response:  []byte("IoT Rocks"),
					TopicUp:   "up2",
					TopicDown: "down2",
				},
			},

			WantData:     []byte("TheThingsNetwork"),
			WantResponse: nil,
			WantError:    pointer.String(string(errors.Behavioural)),
		},
	}

	for i, test := range tests {
		// Describe
		Desc(t, fmt.Sprintf("#%d: %s", i, test.Desc))

		// Build
		aclient, adapter := createAdapter(t)
		sclients, chresp := createServers(test.Recipients)
		<-time.After(time.Millisecond * 50)

		// Operate
		resp, err := trySend(adapter, test.Packet, test.Recipients)
		var data []byte
		select {
		case data = <-chresp:
		case <-time.After(time.Millisecond * 100):
		}

		// Check
		CheckErrors(t, test.WantError, err)
		checkData(t, test.WantData, data)
		checkResponses(t, test.WantResponse, resp)

		// Clean
		aclient.Disconnect(0)
		for _, sclient := range sclients {
			sclient.Disconnect(0)
		}
	}
}

func TestSendErrorCases(t *testing.T) {
	tests := []struct {
		Desc       string          // Test Description
		Packet     []byte          // Handy representation of the packet to send
		Recipients []testRecipient // List of recipient to send
		Client     *MockClient     // A mocked version of the client

		WantData  []byte  // Expected Data on the recipient
		WantError *string // Expected error nature returned by the Send method
	}{
		{
			Desc:   "1 packet | 1 Recipient | Error on publish",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Publish"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  []byte("TheThingsNetwork"),
			WantError: pointer.String(string(errors.Operational)),
		},
		{
			Desc:   "1 packet | 1 Recipient | Error on Subscribe",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Subscribe"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  nil,
			WantError: pointer.String(string(errors.Operational)),
		},
		{
			Desc:   "1 packet | 1 Recipient | Error on Unsubscribe",
			Packet: []byte("TheThingsNetwork"),
			Client: NewMockClient("Unsubscribe"),
			Recipients: []testRecipient{
				{
					Response:  nil,
					TopicUp:   "up",
					TopicDown: "down",
				},
			},

			WantData:  []byte("TheThingsNetwork"),
			WantError: nil,
		},
	}

	for i, test := range tests {
		// Describe
		Desc(t, fmt.Sprintf("#%d: %s", i, test.Desc))

		// Build
		adapter := NewAdapter(test.Client, GetLogger(t, "Adapter"))

		// Operate
		_, err := trySend(adapter, test.Packet, test.Recipients)

		// Check
		CheckErrors(t, test.WantError, err)
		checkData(t, test.WantData, test.Client.InPublish.Payload())
	}
}

func TestOtherMethods(t *testing.T) {
	{
		// Describe
		Desc(t, "Get Recipient | Wrong data")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))

		// Operate
		_, err := adapter.GetRecipient([]byte{})

		// Check
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Get Recipient | Valid Recipient")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))
		data, _ := NewRecipient("up", "down").MarshalBinary()

		// Operate
		recipient, err := adapter.GetRecipient(data)

		// Check
		CheckErrors(t, nil, err)
		checkRecipients(t, NewRecipient("up", "down"), recipient)

	}

	// --------------------

	{
		// Describe
		Desc(t, "Send invalid recipients")

		// Build
		adapter := NewAdapter(NewMockClient(), GetLogger(t, "Adapter"))

		// Operate
		_, err := adapter.Send(mocks.NewMockPacket(), mocks.NewMockRecipient())

		// Check
		CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler")

		// Build
		client := NewMockClient()
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()
		msg := MockMessage{
			topic:   "MessageTopic",
			payload: []byte{1, 2, 3, 4},
		}

		// Operate
		err := adapter.Bind(handler)
		client.InSubscribeCallBack(client, msg)

		// Check
		CheckErrors(t, nil, err)
		checkMessages(t, msg, handler.InMessage)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler | fails to handle")

		// Build
		client := NewMockClient()
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()
		handler.Failures["Handle"] = errors.New(errors.Operational, "Mock Error")
		msg := MockMessage{
			topic:   "MessageTopic",
			payload: []byte{1, 2, 3, 4},
		}

		// Operate
		err := adapter.Bind(handler)
		client.InSubscribeCallBack(client, msg)

		// Check
		CheckErrors(t, nil, err)
		checkMessages(t, msg, handler.InMessage)
	}

	// --------------------

	{
		// Describe
		Desc(t, "Bind a new handler | fails to subscribe")

		// Build
		client := NewMockClient("Subscribe")
		adapter := NewAdapter(client, GetLogger(t, "Adapter"))
		handler := NewMockHandler()

		// Operate
		err := adapter.Bind(handler)

		// Check
		CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}
}

func TestMQTTRecipient(t *testing.T) {
	{
		Desc(t, "Marshal / Unmarshal valid recipient")
		rm := NewRecipient("topicup", "topicdown")
		ru := new(recipient)
		data, err := rm.MarshalBinary()
		if err == nil {
			err = ru.UnmarshalBinary(data)
		}
		CheckErrors(t, nil, err)
	}

	{
		Desc(t, "Unmarshal from nil pointer")
		rm := NewRecipient("topicup", "topicdown")
		var ru *recipient
		data, err := rm.MarshalBinary()
		if err == nil {
			err = ru.UnmarshalBinary(data)
		}
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	{
		Desc(t, "Unmarshal nil data")
		ru := new(recipient)
		err := ru.UnmarshalBinary(nil)
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}

	{
		Desc(t, "Unmarshal wrong data")
		ru := new(recipient)
		err := ru.UnmarshalBinary([]byte{1, 2, 3, 4})
		CheckErrors(t, pointer.String(string(errors.Structural)), err)
	}
}

// ----- TYPE utilities
type testRecipient struct {
	Response  []byte
	TopicUp   string
	TopicDown string
}

type testPacket struct {
	payload []byte
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (p testPacket) MarshalBinary() ([]byte, error) {
	if p.payload == nil {
		return nil, errors.New(errors.Structural, "Fake error")
	}

	return p.payload, nil
}

// String implements the core.Packet interface
func (p testPacket) String() string {
	return string(p.payload)
}

// DevEUI implements the core.Packet interface
func (p testPacket) DevEUI() lorawan.EUI64 {
	return lorawan.EUI64{}
}

// ----- BUILD utilities
func createAdapter(t *testing.T) (Client, core.Adapter) {
	client, err := NewClient("testClient", brokerURL, TCP)
	if err != nil {
		panic(err)
	}

	adapter := NewAdapter(client, GetLogger(t, "adapter"))
	return client, adapter
}

func createServers(recipients []testRecipient) ([]Client, chan []byte) {
	var clients []Client
	chresp := make(chan []byte, len(recipients))
	for i, r := range recipients {
		client, err := NewClient(fmt.Sprintf("Client%d%d", i, time.Now().UnixNano()), brokerURL, TCP)
		if err != nil {
			panic(err)
		}
		clients = append(clients, client)

		go func(r testRecipient, client Client) {
			token := client.Subscribe(r.TopicUp, 2, func(client Client, msg MQTT.Message) {
				if r.Response != nil {
					token := client.Publish(r.TopicDown, 2, false, r.Response)
					if token.Wait() && token.Error() != nil {
						panic(token.Error())
					}
				}
				chresp <- msg.Payload()
			})
			if token.Wait() && token.Error() != nil {
				panic(token.Error())
			}
		}(r, client)
	}
	return clients, chresp
}

// ----- OPERATE utilities
func trySend(adapter core.Adapter, packet []byte, recipients []testRecipient) ([]byte, error) {
	// Convert testRecipient to core.Recipient using the mqtt recipient
	var coreRecipients []core.Recipient
	for _, r := range recipients {
		coreRecipients = append(coreRecipients, NewRecipient(r.TopicUp, r.TopicDown))
	}

	// Try send the packet
	chresp := make(chan struct {
		Data  []byte
		Error error
	})
	go func() {
		data, err := adapter.Send(testPacket{packet}, coreRecipients...)
		chresp <- struct {
			Data  []byte
			Error error
		}{data, err}
	}()

	select {
	case resp := <-chresp:
		return resp.Data, resp.Error
	case <-time.After(timeout + time.Millisecond*100):
		return nil, nil
	}
}

// ----- CHECK utilities
func checkResponses(t *testing.T, want []byte, got []byte) {
	mocks.Check(t, want, got, "Responses")
}

func checkData(t *testing.T, want []byte, got []byte) {
	mocks.Check(t, want, got, "Data")
}

func checkRecipients(t *testing.T, want core.Recipient, got core.Recipient) {
	mocks.Check(t, want, got, "Recipients")
}

func checkMessages(t *testing.T, want MQTT.Message, got MQTT.Message) {
	mocks.Check(t, want, got, "Messages")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/log"
)

var (
	cmdUnloadUnit = &Command{
		Name:    "unload",
		Summary: "Unschedule one or more units in the cluster.",
		Usage:   "UNIT...",
		Run:     runUnloadUnit,
	}
)

func init() {
	cmdUnloadUnit.Flags.IntVar(&sharedFlags.BlockAttempts, "block-attempts", 0, "Wait until the units are inactive, performing up to N attempts before giving up. A value of 0 indicates no limit.")
	cmdUnloadUnit.Flags.BoolVar(&sharedFlags.NoBlock, "no-block", false, "Do not wait until the units have become inactive before exiting.")
}

func runUnloadUnit(args []string) (exit int) {
	units, err := findUnits(args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return 1
	}

	wait := make([]string, 0)
	for _, s := range units {
		if !suToGlobal(s) {
			if job.JobState(s.CurrentState) == job.JobStateInactive {
				log.V(1).Infof("Target state of Unit(%s) already %s, skipping.", s.Name, job.JobStateInactive)
				continue
			}
		}

		log.V(1).Infof("Setting target state of Unit(%s) to %s", s.Name, job.JobStateInactive)
		cAPI.SetUnitTargetState(s.Name, string(job.JobStateInactive))
		if suToGlobal(s) {
			fmt.Printf("Triggered global unit %s unload\n", s.Name)
		} else {
			wait = append(wait, s.Name)
		}
	}

	if !sharedFlags.NoBlock {
		errchan := waitForUnitStates(wait, job.JobStateInactive, sharedFlags.BlockAttempts, os.Stdout)
		for err := range errchan {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			exit = 1
		}
	}

	return
}
<commit_msg>fleetctl: update 'unload' output<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/log"
)

var (
	cmdUnloadUnit = &Command{
		Name:    "unload",
		Summary: "Unschedule one or more units in the cluster.",
		Usage:   "UNIT...",
		Run:     runUnloadUnit,
	}
)

func init() {
	cmdUnloadUnit.Flags.IntVar(&sharedFlags.BlockAttempts, "block-attempts", 0, "Wait until the units are inactive, performing up to N attempts before giving up. A value of 0 indicates no limit.")
	cmdUnloadUnit.Flags.BoolVar(&sharedFlags.NoBlock, "no-block", false, "Do not wait until the units have become inactive before exiting.")
}

func runUnloadUnit(args []string) (exit int) {
	units, err := findUnits(args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return 1
	}

	wait := make([]string, 0)
	for _, s := range units {
		if !suToGlobal(s) {
			if job.JobState(s.CurrentState) == job.JobStateInactive {
				log.V(1).Infof("Target state of Unit(%s) already %s, skipping.", s.Name, job.JobStateInactive)
				continue
			}
		}

		log.V(1).Infof("Setting target state of Unit(%s) to %s", s.Name, job.JobStateInactive)
		cAPI.SetUnitTargetState(s.Name, string(job.JobStateInactive))
		if suToGlobal(s) {
			fmt.Printf("Triggered global unit %s unload\n", s.Name)
		} else {
			wait = append(wait, s.Name)
		}
	}

	if !sharedFlags.NoBlock {
		errchan := waitForUnitStates(wait, job.JobStateInactive, sharedFlags.BlockAttempts, os.Stderr)
		for err := range errchan {
			fmt.Fprintf(os.Stderr, "Error waiting for units: %v\n", err)
			exit = 1
		}
	} else {
		for _, name := range wait {
			fmt.Printf("Triggered unit %s unload\n", name)
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"log"
	"strings"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var callbacks = make(map[string]map[string]func(chan irc.Message, irc.Message))

// addCallback registers callbacks that can be later dispatched by Dispatcher
func addCallback(command, name string, callback func(chan irc.Message, irc.Message)) {
	log.Println("adding callback", command, name)
	if _, ok := callbacks[command]; !ok {
		callbacks[command] = make(map[string]func(chan irc.Message, irc.Message))
	}
	callbacks[strings.ToUpper(command)][strings.ToUpper(name)] = callback
}

// Dispatcher takes irc messages and dispatches them to registered callbacks.
//
// It will take an input message, check (based on message context), if the
// message should be dispatched, and passes it to registered callback.
func Dispatcher(output chan irc.Message, input irc.Message) {
	if _, ok := cfg.LookupStringMap(input.Context, "Ignore")[input.Context["Source"]]; ok {
		log.Println("Context:", input.Context, "Ignoring", input.Context["Source"])
		return
	}

	if callbacks[input.Command] != nil {
		for i, f := range callbacks[input.Command] {
			if _, ok := cfg.LookupStringMap(input.Context, "DisabledPlugins")[i]; ok {
				log.Println("Context:", input.Context, "Plugin disabled", i)
				return
			}
			go f(output, input)
		}
	}
}
<commit_msg>Implement plugin whitelist.<commit_after>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"log"
	"strings"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var callbacks = make(map[string]map[string]func(chan irc.Message, irc.Message))

// addCallback registers callbacks that can be later dispatched by Dispatcher
func addCallback(command, name string, callback func(chan irc.Message, irc.Message)) {
	log.Println("adding callback", command, name)
	if _, ok := callbacks[command]; !ok {
		callbacks[command] = make(map[string]func(chan irc.Message, irc.Message))
	}
	callbacks[strings.ToUpper(command)][strings.ToUpper(name)] = callback
}

// Dispatcher takes irc messages and dispatches them to registered callbacks.
//
// It will take an input message, check (based on message context), if the
// message should be dispatched, and passes it to registered callback.
func Dispatcher(output chan irc.Message, input irc.Message) {
	if _, ok := cfg.LookupStringMap(input.Context, "Ignore")[input.Context["Source"]]; ok {
		log.Println("Context:", input.Context, "Ignoring", input.Context["Source"])
		return
	}

	if callbacks[input.Command] != nil {
		if cfg.LookupStringMap(input.Context, "WhitelistedPlugins") != nil {
			for i, f := range callbacks[input.Command] {
				if _, ok := cfg.LookupStringMap(input.Context, "DisabledPlugins")[i]; ok {
					log.Println("Context:", input.Context, "Plugin disabled", i)
					return
				}
				if _, ok := cfg.LookupStringMap(input.Context, "WhitelistedPlugins")[i]; ok {
					go f(output, input)
				} else {
					log.Println("Context:", input.Context, "Plugin not whitelisted", i)
				}
			}
		} else {
			for i, f := range callbacks[input.Command] {
				if _, ok := cfg.LookupStringMap(input.Context, "DisabledPlugins")[i]; ok {
					log.Println("Context:", input.Context, "Plugin disabled", i)
					return
				}
				go f(output, input)
			}
		}
	}
}
<|endoftext|>


<commit_before>package txbuilder

import (
	"github.com/bytom/consensus"
	"github.com/bytom/consensus/segwit"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/vm/vmutil"
)

const (
	baseSize       = int64(176) // inputSize(112) + outputSize(64)
	baseP2WPKHSize = int64(98)
	baseP2WPKHGas  = int64(1409)
)

var (
	//ChainTxUtxoNum maximum utxo quantity in a tx
	ChainTxUtxoNum = 5
	//ChainTxMergeGas chain tx gas
	ChainTxMergeGas = uint64(10000000)
)

// EstimateTxGasInfo estimate transaction consumed gas
type EstimateTxGasInfo struct {
	TotalNeu    int64 `json:"total_neu"`
	FlexibleNeu int64 `json:"flexible_neu"`
	StorageNeu  int64 `json:"storage_neu"`
	VMNeu       int64 `json:"vm_neu"`
	ChainTxNeu  int64 `json:"chain_tx_neu"`
}

func EstimateChainTxGas(templates []Template) (*EstimateTxGasInfo, error) {
	estimated, err := EstimateTxGas(templates[len(templates)-1])
	if err != nil {
		return nil, err
	}

	if len(templates) > 1 {
		estimated.ChainTxNeu = int64(ChainTxMergeGas) * int64(len(templates)-1)
	}
	return estimated, nil
}

// EstimateTxGas estimate consumed neu for transaction
func EstimateTxGas(template Template) (*EstimateTxGasInfo, error) {
	var baseP2WSHSize, totalWitnessSize, baseP2WSHGas, totalP2WPKHGas, totalP2WSHGas, totalIssueGas int64
	for pos, input := range template.Transaction.TxData.Inputs {
		switch input.InputType() {
		case types.SpendInputType:
			controlProgram := input.ControlProgram()
			if segwit.IsP2WPKHScript(controlProgram) {
				totalWitnessSize += baseP2WPKHSize
				totalP2WPKHGas += baseP2WPKHGas
			} else if segwit.IsP2WSHScript(controlProgram) {
				baseP2WSHSize, baseP2WSHGas = estimateP2WSHGas(template.SigningInstructions[pos])
				totalWitnessSize += baseP2WSHSize
				totalP2WSHGas += baseP2WSHGas
			}

		case types.IssuanceInputType:
			issuanceProgram := input.IssuanceProgram()
			if height := vmutil.GetIssuanceProgramRestrictHeight(issuanceProgram); height > 0 {
				// the gas for issue program with checking block height
				totalIssueGas += 5
			}
			baseIssueSize, baseIssueGas := estimateIssueGas(template.SigningInstructions[pos])
			totalWitnessSize += baseIssueSize
			totalIssueGas += baseIssueGas
		}
	}

	flexibleGas := int64(0)
	if totalP2WPKHGas > 0 {
		flexibleGas += baseP2WPKHGas + (baseSize+baseP2WPKHSize)*consensus.StorageGasRate
	} else if totalP2WSHGas > 0 {
		flexibleGas += baseP2WSHGas + (baseSize+baseP2WSHSize)*consensus.StorageGasRate
	} else if totalIssueGas > 0 {
		totalIssueGas += baseP2WPKHGas
		totalWitnessSize += baseSize + baseP2WPKHSize
	}

	// the total transaction storage gas
	totalTxSizeGas := (int64(template.Transaction.TxData.SerializedSize) + totalWitnessSize) * consensus.StorageGasRate

	// the total transaction gas is composed of storage and virtual machines
	totalGas := totalTxSizeGas + totalP2WPKHGas + totalP2WSHGas + totalIssueGas + flexibleGas
	return &EstimateTxGasInfo{
		TotalNeu:    totalGas * consensus.VMGasRate,
		FlexibleNeu: flexibleGas * consensus.VMGasRate,
		StorageNeu:  totalTxSizeGas * consensus.VMGasRate,
		VMNeu:       (totalP2WPKHGas + totalP2WSHGas + totalIssueGas) * consensus.VMGasRate,
	}, nil
}

// estimateP2WSH return the witness size and the gas consumed to execute the virtual machine for P2WSH program
func estimateP2WSHGas(sigInst *SigningInstruction) (int64, int64) {
	var witnessSize, gas int64
	for _, witness := range sigInst.WitnessComponents {
		switch t := witness.(type) {
		case *SignatureWitness:
			witnessSize += 33*int64(len(t.Keys)) + 65*int64(t.Quorum)
			gas += 1131*int64(len(t.Keys)) + 72*int64(t.Quorum) + 659
			if int64(len(t.Keys)) == 1 && int64(t.Quorum) == 1 {
				gas += 27
			}
		case *RawTxSigWitness:
			witnessSize += 33*int64(len(t.Keys)) + 65*int64(t.Quorum)
			gas += 1131*int64(len(t.Keys)) + 72*int64(t.Quorum) + 659
			if int64(len(t.Keys)) == 1 && int64(t.Quorum) == 1 {
				gas += 27
			}
		}
	}
	return witnessSize, gas
}

// estimateIssueGas return the witness size and the gas consumed to execute the virtual machine for issuance program
func estimateIssueGas(sigInst *SigningInstruction) (int64, int64) {
	var witnessSize, gas int64
	for _, witness := range sigInst.WitnessComponents {
		switch t := witness.(type) {
		case *SignatureWitness:
			witnessSize += 65 * int64(t.Quorum)
			gas += 1065*int64(len(t.Keys)) + 72*int64(t.Quorum) + 316
		case *RawTxSigWitness:
			witnessSize += 65 * int64(t.Quorum)
			gas += 1065*int64(len(t.Keys)) + 72*int64(t.Quorum) + 316
		}
	}
	return witnessSize, gas
}
<commit_msg>modify chain tx fee (#1793)<commit_after>package txbuilder

import (
	"github.com/bytom/consensus"
	"github.com/bytom/consensus/segwit"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/vm/vmutil"
)

const (
	baseSize       = int64(176) // inputSize(112) + outputSize(64)
	baseP2WPKHSize = int64(98)
	baseP2WPKHGas  = int64(1409)
)

var (
	//ChainTxUtxoNum maximum utxo quantity in a tx
	ChainTxUtxoNum = 5
	//ChainTxMergeGas chain tx gas
	ChainTxMergeGas = uint64(6000000)
)

// EstimateTxGasInfo estimate transaction consumed gas
type EstimateTxGasInfo struct {
	TotalNeu    int64 `json:"total_neu"`
	FlexibleNeu int64 `json:"flexible_neu"`
	StorageNeu  int64 `json:"storage_neu"`
	VMNeu       int64 `json:"vm_neu"`
	ChainTxNeu  int64 `json:"chain_tx_neu"`
}

func EstimateChainTxGas(templates []Template) (*EstimateTxGasInfo, error) {
	estimated, err := EstimateTxGas(templates[len(templates)-1])
	if err != nil {
		return nil, err
	}

	if len(templates) > 1 {
		estimated.ChainTxNeu = int64(ChainTxMergeGas) * int64(len(templates)-1)
	}
	return estimated, nil
}

// EstimateTxGas estimate consumed neu for transaction
func EstimateTxGas(template Template) (*EstimateTxGasInfo, error) {
	var baseP2WSHSize, totalWitnessSize, baseP2WSHGas, totalP2WPKHGas, totalP2WSHGas, totalIssueGas int64
	for pos, input := range template.Transaction.TxData.Inputs {
		switch input.InputType() {
		case types.SpendInputType:
			controlProgram := input.ControlProgram()
			if segwit.IsP2WPKHScript(controlProgram) {
				totalWitnessSize += baseP2WPKHSize
				totalP2WPKHGas += baseP2WPKHGas
			} else if segwit.IsP2WSHScript(controlProgram) {
				baseP2WSHSize, baseP2WSHGas = estimateP2WSHGas(template.SigningInstructions[pos])
				totalWitnessSize += baseP2WSHSize
				totalP2WSHGas += baseP2WSHGas
			}

		case types.IssuanceInputType:
			issuanceProgram := input.IssuanceProgram()
			if height := vmutil.GetIssuanceProgramRestrictHeight(issuanceProgram); height > 0 {
				// the gas for issue program with checking block height
				totalIssueGas += 5
			}
			baseIssueSize, baseIssueGas := estimateIssueGas(template.SigningInstructions[pos])
			totalWitnessSize += baseIssueSize
			totalIssueGas += baseIssueGas
		}
	}

	flexibleGas := int64(0)
	if totalP2WPKHGas > 0 {
		flexibleGas += baseP2WPKHGas + (baseSize+baseP2WPKHSize)*consensus.StorageGasRate
	} else if totalP2WSHGas > 0 {
		flexibleGas += baseP2WSHGas + (baseSize+baseP2WSHSize)*consensus.StorageGasRate
	} else if totalIssueGas > 0 {
		totalIssueGas += baseP2WPKHGas
		totalWitnessSize += baseSize + baseP2WPKHSize
	}

	// the total transaction storage gas
	totalTxSizeGas := (int64(template.Transaction.TxData.SerializedSize) + totalWitnessSize) * consensus.StorageGasRate

	// the total transaction gas is composed of storage and virtual machines
	totalGas := totalTxSizeGas + totalP2WPKHGas + totalP2WSHGas + totalIssueGas + flexibleGas
	return &EstimateTxGasInfo{
		TotalNeu:    totalGas * consensus.VMGasRate,
		FlexibleNeu: flexibleGas * consensus.VMGasRate,
		StorageNeu:  totalTxSizeGas * consensus.VMGasRate,
		VMNeu:       (totalP2WPKHGas + totalP2WSHGas + totalIssueGas) * consensus.VMGasRate,
	}, nil
}

// estimateP2WSH return the witness size and the gas consumed to execute the virtual machine for P2WSH program
func estimateP2WSHGas(sigInst *SigningInstruction) (int64, int64) {
	var witnessSize, gas int64
	for _, witness := range sigInst.WitnessComponents {
		switch t := witness.(type) {
		case *SignatureWitness:
			witnessSize += 33*int64(len(t.Keys)) + 65*int64(t.Quorum)
			gas += 1131*int64(len(t.Keys)) + 72*int64(t.Quorum) + 659
			if int64(len(t.Keys)) == 1 && int64(t.Quorum) == 1 {
				gas += 27
			}
		case *RawTxSigWitness:
			witnessSize += 33*int64(len(t.Keys)) + 65*int64(t.Quorum)
			gas += 1131*int64(len(t.Keys)) + 72*int64(t.Quorum) + 659
			if int64(len(t.Keys)) == 1 && int64(t.Quorum) == 1 {
				gas += 27
			}
		}
	}
	return witnessSize, gas
}

// estimateIssueGas return the witness size and the gas consumed to execute the virtual machine for issuance program
func estimateIssueGas(sigInst *SigningInstruction) (int64, int64) {
	var witnessSize, gas int64
	for _, witness := range sigInst.WitnessComponents {
		switch t := witness.(type) {
		case *SignatureWitness:
			witnessSize += 65 * int64(t.Quorum)
			gas += 1065*int64(len(t.Keys)) + 72*int64(t.Quorum) + 316
		case *RawTxSigWitness:
			witnessSize += 65 * int64(t.Quorum)
			gas += 1065*int64(len(t.Keys)) + 72*int64(t.Quorum) + 316
		}
	}
	return witnessSize, gas
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	kftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfapp/coordinator"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfupgrade"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()
var kfApp kftypes.KfApp
var err error

// KFDef example configs to be printed out from apply --help
const (
	arritkoConfig = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_existing_arrikto.yaml"
	awsConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_aws.yaml"
	gcpConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_gcp_iap.yaml"
	k8sConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_k8s_istio.yaml"
)

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply -f ${CONFIG}",
	Short: "deploys a kubeflow application.",
	Long: `'kfctl apply' builds and deploys a kubeflow application from a KFDef config.` + "\n" +
		`To install run -> ` + ColorPrint("kfctl apply -f ${CONFIG}") + "\n" +
		`For more information, run 'kfctl apply -h' or read the docs at www.kubeflow.org.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		log.SetLevel(log.InfoLevel)
		if applyCfg.GetBool(string(kftypes.VERBOSE)) != true {
			log.SetLevel(log.WarnLevel)
		}

		// Load config from exisiting app.yaml
		if configFilePath == "" {
			return fmt.Errorf("Must pass in -f configFile")
		}

		kind, err := utils.GetObjectKindFromUri(configFilePath)
		if err != nil {
			return fmt.Errorf("Cannot determine the object kind: %v", err)
		}
		switch kind {
		case string(kftypes.KFDEF):
			kfApp, err = coordinator.NewLoadKfAppFromURI(configFilePath)
			if err != nil {
				return fmt.Errorf("failed to build kfApp from URI %s: %v", configFilePath, err)
			}
			return kfApp.Apply(kftypes.ALL)
		case string(kftypes.KFUPGRADE):
			kfUpgrade, err := kfupgrade.NewKfUpgrade(configFilePath)
			if err != nil {
				return fmt.Errorf("couldn't load KfUpgrade: %v", err)
			}

			err = kfUpgrade.Apply()
			if err != nil {
				return fmt.Errorf("couldn't apply KfUpgrade: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("Unsupported object kind: %v", kind)
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// Config file option
	applyCmd.PersistentFlags().StringVarP(&configFilePath, string(kftypes.FILE), "f", "",
		`Static config file to use. Can be either a local path or a URL.
For example:
export CONFIG=`+arritkoConfig+`
export CONFIG=`+awsConfig+`
export CONFIG=`+gcpConfig+`
export CONFIG=`+k8sConfig+`
kfctl apply -V --file=${CONFIG}`)

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
}
<commit_msg>fix example path in help messages (#4395)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	kftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfapp/coordinator"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfupgrade"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()
var kfApp kftypes.KfApp
var err error

// KFDef example configs to be printed out from apply --help
const (
	arritkoConfig = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_existing_arrikto.0.7.0.yaml"
	awsConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_aws.0.7.0.yaml"
	gcpConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_gcp_iap.0.7.0.yaml"
	k8sConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_k8s_istio.0.7.0.yaml"
)

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply -f ${CONFIG}",
	Short: "deploys a kubeflow application.",
	Long: `'kfctl apply' builds and deploys a kubeflow application from a KFDef config.` + "\n" +
		`To install run -> ` + ColorPrint("kfctl apply -f ${CONFIG}") + "\n" +
		`For more information, run 'kfctl apply -h' or read the docs at www.kubeflow.org.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		log.SetLevel(log.InfoLevel)
		if applyCfg.GetBool(string(kftypes.VERBOSE)) != true {
			log.SetLevel(log.WarnLevel)
		}

		// Load config from exisiting app.yaml
		if configFilePath == "" {
			return fmt.Errorf("Must pass in -f configFile")
		}

		kind, err := utils.GetObjectKindFromUri(configFilePath)
		if err != nil {
			return fmt.Errorf("Cannot determine the object kind: %v", err)
		}
		switch kind {
		case string(kftypes.KFDEF):
			kfApp, err = coordinator.NewLoadKfAppFromURI(configFilePath)
			if err != nil {
				return fmt.Errorf("failed to build kfApp from URI %s: %v", configFilePath, err)
			}
			return kfApp.Apply(kftypes.ALL)
		case string(kftypes.KFUPGRADE):
			kfUpgrade, err := kfupgrade.NewKfUpgrade(configFilePath)
			if err != nil {
				return fmt.Errorf("couldn't load KfUpgrade: %v", err)
			}

			err = kfUpgrade.Apply()
			if err != nil {
				return fmt.Errorf("couldn't apply KfUpgrade: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("Unsupported object kind: %v", kind)
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// Config file option
	applyCmd.PersistentFlags().StringVarP(&configFilePath, string(kftypes.FILE), "f", "",
		`Static config file to use. Can be either a local path or a URL.
For example:
export CONFIG=`+arritkoConfig+`
export CONFIG=`+awsConfig+`
export CONFIG=`+gcpConfig+`
export CONFIG=`+k8sConfig+`
kfctl apply -V --file=${CONFIG}`)

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
}
<|endoftext|>


<commit_before>package pilosa_test

import (
	"reflect"
	"testing"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/internal"
)

// Ensure a message can be marshaled and unmarshaled.
func TestMessage_Marshal(t *testing.T) {

	testMessageMarshal(t, &internal.CreateSliceMessage{
		DB:    "d",
		Slice: 8,
	})

	testMessageMarshal(t, &internal.DeleteDBMessage{
		DB: "d",
	})
}

func testMessageMarshal(t *testing.T, m proto.Message) {
	marshalled, err := pilosa.MarshalMessage(m)
	if err != nil {
		t.Fatal(err)
	}
	unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(unmarshalled, m) {
		t.Fatalf("unexpected message marshalling: %s", unmarshalled)
	}
}
<commit_msg>adds a basic BroadcastReceiver test<commit_after>package pilosa_test

import (
	"reflect"
	"testing"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa"
	"github.com/pilosa/pilosa/internal"
)

// Ensure a message can be marshaled and unmarshaled.
func TestMessage_Marshal(t *testing.T) {

	testMessageMarshal(t, &internal.CreateSliceMessage{
		DB:    "d",
		Slice: 8,
	})

	testMessageMarshal(t, &internal.DeleteDBMessage{
		DB: "d",
	})
}

func testMessageMarshal(t *testing.T, m proto.Message) {
	marshalled, err := pilosa.MarshalMessage(m)
	if err != nil {
		t.Fatal(err)
	}
	unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(unmarshalled, m) {
		t.Fatalf("unexpected message marshalling: %s", unmarshalled)
	}
}

// Ensure that BroadcastReceiver can register a BroadcastHandler.
func TestBroadcast_BroadcastReceiver(t *testing.T) {

	s := pilosa.NewServer()

	sbr := NewSimpleBroadcastReceiver()
	sbh := NewSimpleBroadcastHandler()

	s.BroadcastReceiver = sbr
	s.BroadcastReceiver.Start(sbh)

	msg := &internal.DeleteDBMessage{
		DB: "d",
	}

	s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg)

	// Make sure the message received is what was sentd
	if !reflect.DeepEqual(sbh.receivedMessage, msg) {
		t.Fatalf("unexpected message: %s", sbh.receivedMessage)
	}
}

type SimpleBroadcastReceiver struct {
	broadcastHandler pilosa.BroadcastHandler
}

func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver {
	return &SimpleBroadcastReceiver{}
}

func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error {
	r.broadcastHandler = h
	return nil
}

func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error {
	r.broadcastHandler.ReceiveMessage(pb)
	return nil
}

type SimpleBroadcastHandler struct {
	receivedMessage proto.Message
}

func NewSimpleBroadcastHandler() *SimpleBroadcastHandler {
	return &SimpleBroadcastHandler{}
}

func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error {
	h.receivedMessage = pb.(proto.Message)
	return nil
}
<|endoftext|>


<commit_before>// Import this in all *_test.go files to initialize ~/.tendermint_test.

package tendermint_test

import (
	"os"
	"path"
	"strings"

	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
)

func init() {
	// Creates ~/.tendermint_test/*
	config := GetConfig("")
	cfg.ApplyConfig(config)
}

func getTMRoot(rootDir string) string {
	if rootDir == "" {
		rootDir = os.Getenv("HOME") + "/.tendermint_test"
	}
	return rootDir
}

func initTMRoot(rootDir string) {
	rootDir = getTMRoot(rootDir)

	// Remove ~/.tendermint_test_bak
	err := os.RemoveAll(rootDir + "_bak")
	if err != nil {
		PanicSanity(err.Error())
	}
	// Move ~/.tendermint_test to ~/.tendermint_test_bak
	err = os.Rename(rootDir, rootDir+"_bak")
	if err != nil {
		PanicSanity(err.Error())
	}
	// Create new dir
	EnsureDir(rootDir, 0700)

	configFilePath := path.Join(rootDir, "config.toml")
	genesisFilePath := path.Join(rootDir, "genesis.json")
	privFilePath := path.Join(rootDir, "priv_validator.json")

	// Write default config file if missing.
	if !FileExists(configFilePath) {
		// Ask user for moniker
		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
	}
	if !FileExists(genesisFilePath) {
		MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644)
	}
	if !FileExists(privFilePath) {
		MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644)
	}
}

func GetConfig(rootDir string) cfg.Config {
	rootDir = getTMRoot(rootDir)
	initTMRoot(rootDir)

	configFilePath := path.Join(rootDir, "config.toml")
	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
	if err != nil {
		Exit(Fmt("Could not read config: %v", err))
	}

	// Set defaults or panic
	if mapConfig.IsSet("chain_id") {
		Exit("Cannot set 'chain_id' via config.toml")
	}
	mapConfig.SetDefault("chain_id", "tendermint_test")
	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
	mapConfig.SetDefault("proxy_app", "local")
	mapConfig.SetDefault("moniker", "anonymous")
	mapConfig.SetDefault("node_laddr", "0.0.0.0:36656")
	mapConfig.SetDefault("fast_sync", false)
	mapConfig.SetDefault("skip_upnp", true)
	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
	mapConfig.SetDefault("db_backend", "memdb")
	mapConfig.SetDefault("db_dir", rootDir+"/data")
	mapConfig.SetDefault("log_level", "debug")
	mapConfig.SetDefault("vm_log", true)
	mapConfig.SetDefault("rpc_laddr", "0.0.0.0:36657")
	mapConfig.SetDefault("prof_laddr", "")
	mapConfig.SetDefault("revision_file", rootDir+"/revision")
	return mapConfig
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

proxy_app = "local"
moniker = "__MONIKER__"
node_laddr = "0.0.0.0:36656"
seeds = ""
fast_sync = false
db_backend = "memdb"
log_level = "debug"
rpc_laddr = "0.0.0.0:36657"
`

func defaultConfig(moniker string) (defaultConfig string) {
	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
	return
}

var defaultGenesis = `{
  "genesis_time": "0001-01-01T00:00:00.000Z",
  "chain_id": "tendermint_test",
  "validators": [
    {
      "pub_key": [
        1,
        "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
      ],
      "amount": 10,
      "name": ""
    }
  ],
  "app_hash": ""
}`

var defaultPrivValidator = `{
  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
  "pub_key": [
    1,
    "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "priv_key": [
    1,
    "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "last_height": 0,
  "last_round": 0,
  "last_step": 0
}`
<commit_msg>Fix broken test due to config requiring ~/.tendermint_test<commit_after>// Import this in all *_test.go files to initialize ~/.tendermint_test.

package tendermint_test

import (
	"os"
	"path"
	"strings"

	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
)

func init() {
	// Creates ~/.tendermint_test/*
	config := GetConfig("")
	cfg.ApplyConfig(config)
}

func getTMRoot(rootDir string) string {
	if rootDir == "" {
		rootDir = os.Getenv("HOME") + "/.tendermint_test"
	}
	return rootDir
}

func initTMRoot(rootDir string) {
	rootDir = getTMRoot(rootDir)

	// Remove ~/.tendermint_test_bak
	if FileExists(rootDir + "_bak") {
		err := os.RemoveAll(rootDir + "_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Move ~/.tendermint_test to ~/.tendermint_test_bak
	if FileExists(rootDir + "_bak") {
		err := os.Rename(rootDir, rootDir+"_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Create new dir
	EnsureDir(rootDir, 0700)

	configFilePath := path.Join(rootDir, "config.toml")
	genesisFilePath := path.Join(rootDir, "genesis.json")
	privFilePath := path.Join(rootDir, "priv_validator.json")

	// Write default config file if missing.
	if !FileExists(configFilePath) {
		// Ask user for moniker
		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
	}
	if !FileExists(genesisFilePath) {
		MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644)
	}
	if !FileExists(privFilePath) {
		MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644)
	}
}

func GetConfig(rootDir string) cfg.Config {
	rootDir = getTMRoot(rootDir)
	initTMRoot(rootDir)

	configFilePath := path.Join(rootDir, "config.toml")
	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
	if err != nil {
		Exit(Fmt("Could not read config: %v", err))
	}

	// Set defaults or panic
	if mapConfig.IsSet("chain_id") {
		Exit("Cannot set 'chain_id' via config.toml")
	}
	mapConfig.SetDefault("chain_id", "tendermint_test")
	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
	mapConfig.SetDefault("proxy_app", "local")
	mapConfig.SetDefault("moniker", "anonymous")
	mapConfig.SetDefault("node_laddr", "0.0.0.0:36656")
	mapConfig.SetDefault("fast_sync", false)
	mapConfig.SetDefault("skip_upnp", true)
	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
	mapConfig.SetDefault("db_backend", "memdb")
	mapConfig.SetDefault("db_dir", rootDir+"/data")
	mapConfig.SetDefault("log_level", "debug")
	mapConfig.SetDefault("vm_log", true)
	mapConfig.SetDefault("rpc_laddr", "0.0.0.0:36657")
	mapConfig.SetDefault("prof_laddr", "")
	mapConfig.SetDefault("revision_file", rootDir+"/revision")
	return mapConfig
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

proxy_app = "local"
moniker = "__MONIKER__"
node_laddr = "0.0.0.0:36656"
seeds = ""
fast_sync = false
db_backend = "memdb"
log_level = "debug"
rpc_laddr = "0.0.0.0:36657"
`

func defaultConfig(moniker string) (defaultConfig string) {
	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
	return
}

var defaultGenesis = `{
  "genesis_time": "0001-01-01T00:00:00.000Z",
  "chain_id": "tendermint_test",
  "validators": [
    {
      "pub_key": [
        1,
        "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
      ],
      "amount": 10,
      "name": ""
    }
  ],
  "app_hash": ""
}`

var defaultPrivValidator = `{
  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
  "pub_key": [
    1,
    "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "priv_key": [
    1,
    "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "last_height": 0,
  "last_round": 0,
  "last_step": 0
}`
<|endoftext|>


<commit_before>package transaction

import (
	. "GoOnchain/common"
	"GoOnchain/common/serialization"
	"GoOnchain/core/contract"
	"GoOnchain/core/contract/program"
	sig "GoOnchain/core/signature"
	msg "GoOnchain/node/message"
	"GoOnchain/core/transaction/payload"
	. "GoOnchain/errors"
	"crypto/sha256"
	"errors"
	"io"
	"sort"
)

//for different transaction types with different payload format
//and transaction process methods
type TransactionType byte

const (
	BookKeeping   TransactionType = 0x00
	RegisterAsset TransactionType = 0x40
	IssueAsset    TransactionType = 0x01
	TransferAsset TransactionType = 0x10
	Record        TransactionType = 0x11
)

//Payload define the func for loading the payload data
//base on payload type which have different struture
type Payload interface {
	//  Get payload data
	Data() []byte

	//Serialize payload data
	Serialize(w io.Writer)

	Deserialize(r io.Reader) error
}

//Transaction is used for carry information or action to Ledger
//validated transaction will be added to block and updates state correspondingly

var TxStore ILedgerStore

type Transaction struct {
	TxType         TransactionType
	PayloadVersion byte
	Payload        Payload
	Nonce          uint64
	Attributes     []*TxAttribute
	UTXOInputs     []*UTXOTxInput
	BalanceInputs  []*BalanceTxInput
	Outputs        []*TxOutput
	Programs       []*program.Program

	//Inputs/Outputs map base on Asset (needn't serialize)
	AssetOutputs      map[Uint256][]*TxOutput
	AssetInputAmount  map[Uint256]Fixed64
	AssetOutputAmount map[Uint256]Fixed64

	hash *Uint256
}

//Serialize the Transaction
func (tx *Transaction) Serialize(w io.Writer) {

	tx.SerializeUnsigned(w)

	//Serialize  Transaction's programs
	len := uint64(len(tx.Programs))
	serialization.WriteVarUint(w, len)
	for _, p := range tx.Programs {
		p.Serialize(w)
	}
}

//Serialize the Transaction data without contracts
func (tx *Transaction) SerializeUnsigned(w io.Writer) error {
	//txType
	w.Write([]byte{byte(tx.TxType)})
	//PayloadVersion
	w.Write([]byte{tx.PayloadVersion})
	//Payload
	if tx.Payload == nil {
		return errors.New("Transaction Payload is nil.")
	}
	tx.Payload.Serialize(w)
	//nonce
	//serialization.WriteVarUint(w, tx.Nonce)
	//[]*txAttribute
	err := serialization.WriteVarUint(w, uint64(len(tx.Attributes)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item txAttribute length serialization failed.")
	}
	for _, attr := range tx.Attributes {
		attr.Serialize(w)
	}
	//[]*UTXOInputs
	err = serialization.WriteVarUint(w, uint64(len(tx.UTXOInputs)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item UTXOInputs length serialization failed.")
	}
	for _, utxo := range tx.UTXOInputs {
		utxo.Serialize(w)
	}
	/*
		//[]*BalanceInputs
		err = serialization.WriteVarUint(w, uint64(len(tx.BalanceInputs)))
		if err != nil {
			return NewDetailErr(err, ErrNoCode, "Transaction item BalanceInputs length serialization failed.")
		}
		for _, balance := range tx.BalanceInputs {
			balance.Serialize(w)
		}
	*/
	//[]*Outputs
	err = serialization.WriteVarUint(w, uint64(len(tx.Outputs)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item Outputs length serialization failed.")
	}
	for _, output := range tx.Outputs {
		output.Serialize(w)
	}

	return nil
}

//deserialize the Transaction
func (tx *Transaction) Deserialize(r io.Reader) error {
	// tx deserialize
	tx.DeserializeUnsigned(r)

	// tx program
	len, _ := serialization.ReadVarUint(r, 0)

	programHashes := []*program.Program{}

	for i := 0; i < int(len); i++ {
		outputHashes := new(program.Program)
		outputHashes.Deserialize(r)
		programHashes = append(programHashes, outputHashes)
	}
	tx.Programs = programHashes

	return nil
}

func (tx *Transaction) DeserializeUnsigned(r io.Reader) error {
	var txType [1]byte
	_, err := io.ReadFull(r, txType[:])
	tx.TxType = TransactionType(txType[0])
	if err != nil {
		return err
	}
	/*
		if txType[0] != byte(tx.TxType) {
			return errors.New("Transaction Type is different.")
		}
	*/
	return tx.DeserializeUnsignedWithoutType(r)
}

func (tx *Transaction) DeserializeUnsignedWithoutType(r io.Reader) error {
	var payloadVersion [1]byte
	_, err := io.ReadFull(r, payloadVersion[:])
	tx.PayloadVersion = payloadVersion[0]
	if err != nil {
		return err
	}

	//payload
	//tx.Payload.Deserialize(r)
	if tx.TxType == RegisterAsset {
		// Asset Registration
		tx.Payload = new(payload.AssetRegistration)
		tx.Payload.Deserialize(r)
	} else if tx.TxType == IssueAsset {
		// Asset Issue
	}
	//	else if tx.TxType == 0x00 {
	//		// Miner
	//		tx.Payload = new(payload.Miner)
	//		tx.Payload.Deserialize(r)
	//	}
	//attributes
	Len, err := serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		attr := new(TxAttribute)
		err = attr.Deserialize(r)
		if err != nil {
			return err
		}
		tx.Attributes = append(tx.Attributes, attr)
	}

	//UTXOInputs
	Len, err = serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		utxo := new(UTXOTxInput)
		err = utxo.Deserialize(r)
		if err != nil {
			return err
		}
		tx.UTXOInputs = append(tx.UTXOInputs, utxo)
	}
	/*
		//balanceInputs
		Len, err = serialization.ReadVarUint(r, 0)
		if err != nil {
			return err
		}
		for i := uint64(0); i < Len; i++ {
			balanceInput := new(BalanceTxInput)
			err = balanceInput.Deserialize(r)
			if err != nil {
				return err
			}
			tx.BalanceInputs = append(tx.BalanceInputs, balanceInput)
		}
	*/
	//Outputs
	Len, err = serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		output := new(TxOutput)
		output.Deserialize(r)

		tx.Outputs = append(tx.Outputs, output)
	}
	return nil
}

func (tx *Transaction) GetProgramHashes() ([]Uint160, error) {
	if tx == nil {
		return []Uint160{}, errors.New("[Transaction],GetProgramHashes transaction is nil.")
	}
	hashs := []Uint160{}
	// add inputUTXO's transaction
	referenceWithUTXO_Output, err := tx.GetReference()
	if err != nil {
		return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes failed.")
	}
	for _, output := range referenceWithUTXO_Output {
		programHash := output.ProgramHash
		hashs = append(hashs, programHash)
	}
	for _, attribute := range tx.Attributes {
		if attribute.Usage == Script {
			dataHash, err := Uint160ParseFromBytes(attribute.Date)
			if err != nil {
				return nil, NewDetailErr(errors.New("[Transaction], GetProgramHashes err."), ErrNoCode, "")
			}
			hashs = append(hashs, Uint160(dataHash))
		}
	}
	switch tx.TxType {
	case RegisterAsset:
		issuer := tx.Payload.(*payload.AssetRegistration).Issuer
		signatureRedeemScript, err := contract.CreateSignatureRedeemScript(issuer)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes CreateSignatureRedeemScript failed.")
		}

		astHash, err := ToCodeHash(signatureRedeemScript)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes ToCodeHash failed.")
		}
		hashs = append(hashs, astHash)
	case IssueAsset:
		ast := *(tx.Payload.(*payload.AssetRegistration).Controller)
		hashs = append(hashs, ast)
	default:
	}
	sort.Sort(byProgramHashes(hashs))
	return hashs, nil
}

func (tx *Transaction) SetPrograms(programs []*program.Program) {
	tx.Programs = programs
}

func (tx *Transaction) GetPrograms() []*program.Program {
	return tx.Programs
}

func (tx *Transaction) GetOutputHashes() ([]Uint160, error) {
	//TODO: implement Transaction.GetOutputHashes()

	return []Uint160{}, nil
}

func (tx *Transaction) GenerateAssetMaps() {
	//TODO: implement Transaction.GenerateAssetMaps()
}

func (tx *Transaction) GetMessage() []byte {
	return sig.GetHashData(tx)
}

func (tx *Transaction) Hash() Uint256 {
	if tx.hash == nil {
		d := sig.GetHashData(tx)
		temp := sha256.Sum256([]byte(d))
		f := Uint256(sha256.Sum256(temp[:]))
		tx.hash = &f
	}
	return *tx.hash

}

func (tx *Transaction) SetHash(hash Uint256) {
	tx.hash = &hash
}

func (tx *Transaction) Type() InventoryType {
	return TRANSACTION
}
func (tx *Transaction) Verify() error {
	//TODO: Verify()
	return nil
}

func (tx *Transaction) GetReference() (map[UTXOTxInput]TxOutput, error) {
	if tx.TxType == RegisterAsset {
		return nil, nil
	}
	//UTXO input /  Outputs
	reference := make(map[UTXOTxInput]TxOutput)
	// key 顺序，v UTXOInput
	for _, utxo := range tx.UTXOInputs {
		transaction, err := TxStore.GetTransaction(utxo.ReferTxID)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetReference failed.")
		}
		index := utxo.ReferTxOutputIndex
		reference[*utxo] = *transaction.Outputs[index]
	}
	return reference, nil
}

type byProgramHashes []Uint160

func (a byProgramHashes) Len() int      { return len(a) }
func (a byProgramHashes) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byProgramHashes) Less(i, j int) bool {
	if a[i].CompareTo(a[j]) > 0 {
		return false
	} else {
		return true
	}
}
<commit_msg>fixed the getMessage func<commit_after>package transaction

import (
	. "GoOnchain/common"
	"GoOnchain/common/serialization"
	"GoOnchain/core/contract"
	"GoOnchain/core/contract/program"
	sig "GoOnchain/core/signature"
	msg "GoOnchain/node/message"
	"GoOnchain/core/transaction/payload"
	. "GoOnchain/errors"
	"crypto/sha256"
	"errors"
	"io"
	"sort"
)

//for different transaction types with different payload format
//and transaction process methods
type TransactionType byte

const (
	BookKeeping   TransactionType = 0x00
	RegisterAsset TransactionType = 0x40
	IssueAsset    TransactionType = 0x01
	TransferAsset TransactionType = 0x10
	Record        TransactionType = 0x11
)

//Payload define the func for loading the payload data
//base on payload type which have different struture
type Payload interface {
	//  Get payload data
	Data() []byte

	//Serialize payload data
	Serialize(w io.Writer)

	Deserialize(r io.Reader) error
}

//Transaction is used for carry information or action to Ledger
//validated transaction will be added to block and updates state correspondingly

var TxStore ILedgerStore

type Transaction struct {
	TxType         TransactionType
	PayloadVersion byte
	Payload        Payload
	Nonce          uint64
	Attributes     []*TxAttribute
	UTXOInputs     []*UTXOTxInput
	BalanceInputs  []*BalanceTxInput
	Outputs        []*TxOutput
	Programs       []*program.Program

	//Inputs/Outputs map base on Asset (needn't serialize)
	AssetOutputs      map[Uint256][]*TxOutput
	AssetInputAmount  map[Uint256]Fixed64
	AssetOutputAmount map[Uint256]Fixed64

	hash *Uint256
}

//Serialize the Transaction
func (tx *Transaction) Serialize(w io.Writer) {

	tx.SerializeUnsigned(w)

	//Serialize  Transaction's programs
	len := uint64(len(tx.Programs))
	serialization.WriteVarUint(w, len)
	for _, p := range tx.Programs {
		p.Serialize(w)
	}
}

//Serialize the Transaction data without contracts
func (tx *Transaction) SerializeUnsigned(w io.Writer) error {
	//txType
	w.Write([]byte{byte(tx.TxType)})
	//PayloadVersion
	w.Write([]byte{tx.PayloadVersion})
	//Payload
	if tx.Payload == nil {
		return errors.New("Transaction Payload is nil.")
	}
	tx.Payload.Serialize(w)
	//nonce
	//serialization.WriteVarUint(w, tx.Nonce)
	//[]*txAttribute
	err := serialization.WriteVarUint(w, uint64(len(tx.Attributes)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item txAttribute length serialization failed.")
	}
	for _, attr := range tx.Attributes {
		attr.Serialize(w)
	}
	//[]*UTXOInputs
	err = serialization.WriteVarUint(w, uint64(len(tx.UTXOInputs)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item UTXOInputs length serialization failed.")
	}
	for _, utxo := range tx.UTXOInputs {
		utxo.Serialize(w)
	}
	/*
		//[]*BalanceInputs
		err = serialization.WriteVarUint(w, uint64(len(tx.BalanceInputs)))
		if err != nil {
			return NewDetailErr(err, ErrNoCode, "Transaction item BalanceInputs length serialization failed.")
		}
		for _, balance := range tx.BalanceInputs {
			balance.Serialize(w)
		}
	*/
	//[]*Outputs
	err = serialization.WriteVarUint(w, uint64(len(tx.Outputs)))
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "Transaction item Outputs length serialization failed.")
	}
	for _, output := range tx.Outputs {
		output.Serialize(w)
	}

	return nil
}

//deserialize the Transaction
func (tx *Transaction) Deserialize(r io.Reader) error {
	// tx deserialize
	tx.DeserializeUnsigned(r)

	// tx program
	len, _ := serialization.ReadVarUint(r, 0)

	programHashes := []*program.Program{}

	for i := 0; i < int(len); i++ {
		outputHashes := new(program.Program)
		outputHashes.Deserialize(r)
		programHashes = append(programHashes, outputHashes)
	}
	tx.Programs = programHashes

	return nil
}

func (tx *Transaction) DeserializeUnsigned(r io.Reader) error {
	var txType [1]byte
	_, err := io.ReadFull(r, txType[:])
	tx.TxType = TransactionType(txType[0])
	if err != nil {
		return err
	}
	/*
		if txType[0] != byte(tx.TxType) {
			return errors.New("Transaction Type is different.")
		}
	*/
	return tx.DeserializeUnsignedWithoutType(r)
}

func (tx *Transaction) DeserializeUnsignedWithoutType(r io.Reader) error {
	var payloadVersion [1]byte
	_, err := io.ReadFull(r, payloadVersion[:])
	tx.PayloadVersion = payloadVersion[0]
	if err != nil {
		return err
	}

	//payload
	//tx.Payload.Deserialize(r)
	if tx.TxType == RegisterAsset {
		// Asset Registration
		tx.Payload = new(payload.AssetRegistration)
		tx.Payload.Deserialize(r)
	} else if tx.TxType == IssueAsset {
		// Asset Issue
	}
	//	else if tx.TxType == 0x00 {
	//		// Miner
	//		tx.Payload = new(payload.Miner)
	//		tx.Payload.Deserialize(r)
	//	}
	//attributes
	Len, err := serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		attr := new(TxAttribute)
		err = attr.Deserialize(r)
		if err != nil {
			return err
		}
		tx.Attributes = append(tx.Attributes, attr)
	}

	//UTXOInputs
	Len, err = serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		utxo := new(UTXOTxInput)
		err = utxo.Deserialize(r)
		if err != nil {
			return err
		}
		tx.UTXOInputs = append(tx.UTXOInputs, utxo)
	}
	/*
		//balanceInputs
		Len, err = serialization.ReadVarUint(r, 0)
		if err != nil {
			return err
		}
		for i := uint64(0); i < Len; i++ {
			balanceInput := new(BalanceTxInput)
			err = balanceInput.Deserialize(r)
			if err != nil {
				return err
			}
			tx.BalanceInputs = append(tx.BalanceInputs, balanceInput)
		}
	*/
	//Outputs
	Len, err = serialization.ReadVarUint(r, 0)
	if err != nil {
		return err
	}
	for i := uint64(0); i < Len; i++ {
		output := new(TxOutput)
		output.Deserialize(r)

		tx.Outputs = append(tx.Outputs, output)
	}
	return nil
}

func (tx *Transaction) GetProgramHashes() ([]Uint160, error) {
	if tx == nil {
		return []Uint160{}, errors.New("[Transaction],GetProgramHashes transaction is nil.")
	}
	hashs := []Uint160{}
	// add inputUTXO's transaction
	referenceWithUTXO_Output, err := tx.GetReference()
	if err != nil {
		return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes failed.")
	}
	for _, output := range referenceWithUTXO_Output {
		programHash := output.ProgramHash
		hashs = append(hashs, programHash)
	}
	for _, attribute := range tx.Attributes {
		if attribute.Usage == Script {
			dataHash, err := Uint160ParseFromBytes(attribute.Date)
			if err != nil {
				return nil, NewDetailErr(errors.New("[Transaction], GetProgramHashes err."), ErrNoCode, "")
			}
			hashs = append(hashs, Uint160(dataHash))
		}
	}
	switch tx.TxType {
	case RegisterAsset:
		issuer := tx.Payload.(*payload.AssetRegistration).Issuer
		signatureRedeemScript, err := contract.CreateSignatureRedeemScript(issuer)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes CreateSignatureRedeemScript failed.")
		}

		astHash, err := ToCodeHash(signatureRedeemScript)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetProgramHashes ToCodeHash failed.")
		}
		hashs = append(hashs, astHash)
	case IssueAsset:
		ast := *(tx.Payload.(*payload.AssetRegistration).Controller)
		hashs = append(hashs, ast)
	default:
	}
	sort.Sort(byProgramHashes(hashs))
	return hashs, nil
}

func (tx *Transaction) SetPrograms(programs []*program.Program) {
	tx.Programs = programs
}

func (tx *Transaction) GetPrograms() []*program.Program {
	return tx.Programs
}

func (tx *Transaction) GetOutputHashes() ([]Uint160, error) {
	//TODO: implement Transaction.GetOutputHashes()

	return []Uint160{}, nil
}

func (tx *Transaction) GenerateAssetMaps() {
	//TODO: implement Transaction.GenerateAssetMaps()
}

func (tx *Transaction) GetMessage() []byte {
	return sig.GetHashForSigning(tx)
}

func (tx *Transaction) Hash() Uint256 {
	if tx.hash == nil {
		d := sig.GetHashData(tx)
		temp := sha256.Sum256([]byte(d))
		f := Uint256(sha256.Sum256(temp[:]))
		tx.hash = &f
	}
	return *tx.hash

}

func (tx *Transaction) SetHash(hash Uint256) {
	tx.hash = &hash
}

func (tx *Transaction) Type() InventoryType {
	return TRANSACTION
}
func (tx *Transaction) Verify() error {
	//TODO: Verify()
	return nil
}

func (tx *Transaction) GetReference() (map[UTXOTxInput]TxOutput, error) {
	if tx.TxType == RegisterAsset {
		return nil, nil
	}
	//UTXO input /  Outputs
	reference := make(map[UTXOTxInput]TxOutput)
	// key 顺序，v UTXOInput
	for _, utxo := range tx.UTXOInputs {
		transaction, err := TxStore.GetTransaction(utxo.ReferTxID)
		if err != nil {
			return nil, NewDetailErr(err, ErrNoCode, "[Transaction], GetReference failed.")
		}
		index := utxo.ReferTxOutputIndex
		reference[*utxo] = *transaction.Outputs[index]
	}
	return reference, nil
}

type byProgramHashes []Uint160

func (a byProgramHashes) Len() int      { return len(a) }
func (a byProgramHashes) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byProgramHashes) Less(i, j int) bool {
	if a[i].CompareTo(a[j]) > 0 {
		return false
	} else {
		return true
	}
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
)

// AMIConfig is for common configuration related to creating AMIs.
type AMIConfig struct {
	AMIName         string   `mapstructure:"ami_name"`
	AMIDescription  string   `mapstructure:"ami_description"`
	AMIUsers        []string `mapstructure:"ami_users"`
	AMIGroups       []string `mapstructure:"ami_groups"`
	AMIProductCodes []string `mapstructure:"ami_product_codes"`
	AMIRegions      []string `mapstructure:"ami_regions"`
}

func (c *AMIConfig) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	templates := map[string]*string{
		"ami_name":        &c.AMIName,
		"ami_description": &c.AMIDescription,
	}

	errs := make([]error, 0)
	for n, ptr := range templates {
		var err error
		*ptr, err = t.Process(*ptr, nil)
		if err != nil {
			errs = append(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	sliceTemplates := map[string][]string{
		"ami_users":         c.AMIUsers,
		"ami_groups":        c.AMIGroups,
		"ami_product_codes": c.AMIProductCodes,
		"ami_regions":       c.AMIRegions,
	}

	for n, slice := range sliceTemplates {
		for i, elem := range slice {
			var err error
			slice[i], err = t.Process(elem, nil)
			if err != nil {
				errs = append(
					errs, fmt.Errorf("Error processing %s[%d]: %s", n, i, err))
			}
		}
	}

	if c.AMIName == "" {
		errs = append(errs, fmt.Errorf("ami_name must be specified"))
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<commit_msg>Make sure ami_regions are valid<commit_after>package common

import (
	"fmt"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/packer/packer"
)

// AMIConfig is for common configuration related to creating AMIs.
type AMIConfig struct {
	AMIName         string   `mapstructure:"ami_name"`
	AMIDescription  string   `mapstructure:"ami_description"`
	AMIUsers        []string `mapstructure:"ami_users"`
	AMIGroups       []string `mapstructure:"ami_groups"`
	AMIProductCodes []string `mapstructure:"ami_product_codes"`
	AMIRegions      []string `mapstructure:"ami_regions"`
}

func (c *AMIConfig) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	templates := map[string]*string{
		"ami_name":        &c.AMIName,
		"ami_description": &c.AMIDescription,
	}

	errs := make([]error, 0)
	for n, ptr := range templates {
		var err error
		*ptr, err = t.Process(*ptr, nil)
		if err != nil {
			errs = append(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	sliceTemplates := map[string][]string{
		"ami_users":         c.AMIUsers,
		"ami_groups":        c.AMIGroups,
		"ami_product_codes": c.AMIProductCodes,
		"ami_regions":       c.AMIRegions,
	}

	for n, slice := range sliceTemplates {
		for i, elem := range slice {
			var err error
			slice[i], err = t.Process(elem, nil)
			if err != nil {
				errs = append(
					errs, fmt.Errorf("Error processing %s[%d]: %s", n, i, err))
			}
		}
	}

	if c.AMIName == "" {
		errs = append(errs, fmt.Errorf("ami_name must be specified"))
	}

	if len(c.AMIRegions) > 0 {
		for _, region := range c.AMIRegions {
			if _, ok := aws.Regions[region]; !ok {
				errs = append(errs, fmt.Errorf("Unknown region: %s", region))
			}
		}
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<|endoftext|>