<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	servers := make([]hashic.Server, len(nodes)+1)
	servers[0] = hashic.Server{
		Suffrage: hashic.Voter,
		ID:       hashic.ServerID(selflis),
		Address:  hashic.ServerAddress(selflis),
	}
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		servers[i+1] = hashic.Server{
			Suffrage: hashic.Voter,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", selflis)
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(selflis, addr, len(ids), 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            hashic.ServerID(selflis),
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(selflis)

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	if *serverMetrics {
		_, port, err := net.SplitHostPort(nodes[id-1])

		if err != nil {
			logger.Fatal(err)
		}

		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(":5"+port, nil))
		}()
	}

	logger.Fatal(node.Run(grpcServer))
}

func selfIDsNodes(logger logrus.FieldLogger, id uint64, ids []uint64, nodes []string) (string, []uint64, []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	return selflis, ids, nodes
}
<commit_msg>rkvd/main.go: Change metrics port to 590[raftID]<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	if *serverMetrics {
		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(fmt.Sprintf(":590%d", *id), nil))
		}()
	}

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	servers := make([]hashic.Server, len(nodes)+1)
	servers[0] = hashic.Server{
		Suffrage: hashic.Voter,
		ID:       hashic.ServerID(selflis),
		Address:  hashic.ServerAddress(selflis),
	}
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		servers[i+1] = hashic.Server{
			Suffrage: hashic.Voter,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", selflis)
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(selflis, addr, len(ids), 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            hashic.ServerID(selflis),
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	var selflis string
	selflis, ids, nodes = selfIDsNodes(logger, id, ids, nodes)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	logger.Fatal(node.Run(grpcServer))
}

func selfIDsNodes(logger logrus.FieldLogger, id uint64, ids []uint64, nodes []string) (string, []uint64, []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	return selflis, ids, nodes
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package cmd

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"os"
	"testing"
	"time"

	"github.com/phayes/freeport"
	"github.com/stretchr/testify/assert"
)

var frontendPort, backendPort int

func init() {
	var err error
	frontendPort, err = freeport.GetFreePort()
	if err != nil {
		panic(err.Error())
	}

	backendPort, err = freeport.GetFreePort()
	if err != nil {
		panic(err.Error())
	}

	os.Setenv("PUBLIC_PORT", fmt.Sprintf("%d", frontendPort))
	os.Setenv("ADMIN_PORT", fmt.Sprintf("%d", backendPort))
	os.Setenv("DATABASE_URL", "memory")
	//os.Setenv("HYDRA_URL", fmt.Sprintf("https://localhost:%d/", frontendPort))
	os.Setenv("OAUTH2_ISSUER_URL", fmt.Sprintf("https://localhost:%d/", frontendPort))
}

func TestExecute(t *testing.T) {
	var osArgs = make([]string, len(os.Args))
	copy(osArgs, os.Args)

	frontend := fmt.Sprintf("https://localhost:%d/", frontendPort)
	backend := fmt.Sprintf("https://localhost:%d/", backendPort)

	for _, c := range []struct {
		args      []string
		wait      func() bool
		expectErr bool
	}{
		{
			args: []string{"serve", "all", "--disable-telemetry"},
			wait: func() bool {
				client := &http.Client{
					Transport: &transporter{
						FakeTLSTermination: true,
						Transport: &http.Transport{
							TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
						},
					},
				}

				for _, u := range []string{
					fmt.Sprintf("https://127.0.0.1:%d/.well-known/openid-configuration", frontendPort),
					fmt.Sprintf("https://127.0.0.1:%d/health/status", backendPort),
				} {
					if resp, err := client.Get(u); err != nil {
						t.Logf("HTTP request to %s failed: %s", u, err)
						return true
					} else if resp.StatusCode != http.StatusOK {
						t.Logf("HTTP request to %s got status code %d but expected was 200", u, resp.StatusCode)
						return true
					}
				}

				// Give a bit more time to initialize
				time.Sleep(time.Second * 5)
				return false
			},
		},
		{args: []string{"clients", "create", "--endpoint", backend, "--id", "foobarbaz", "--secret", "foobar", "-g", "client_credentials"}},
		{args: []string{"clients", "get", "--endpoint", backend, "foobarbaz"}},
		{args: []string{"clients", "create", "--endpoint", backend, "--id", "public-foo"}},
		{args: []string{"clients", "delete", "--endpoint", backend, "public-foo"}},
		{args: []string{"keys", "create", "foo", "--endpoint", backend, "-a", "HS256"}},
		{args: []string{"keys", "get", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "rotate", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "get", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "delete", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "import", "--endpoint", backend, "import-1", "../test/stub/ecdh.key", "../test/stub/ecdh.pub"}},
		{args: []string{"keys", "import", "--endpoint", backend, "import-2", "../test/stub/rsa.key", "../test/stub/rsa.pub"}},
		{args: []string{"token", "revoke", "--endpoint", frontend, "--client-secret", "foobar", "--client-id", "foobarbaz", "foo"}},
		{args: []string{"token", "client", "--endpoint", frontend, "--client-secret", "foobar", "--client-id", "foobarbaz"}},
		{args: []string{"help", "migrate", "sql"}},
		{args: []string{"version"}},
		{args: []string{"token", "flush", "--endpoint", backend}},
	} {
		c.args = append(c.args, []string{"--skip-tls-verify"}...)
		RootCmd.SetArgs(c.args)

		t.Run(fmt.Sprintf("command=%v", c.args), func(t *testing.T) {
			if c.wait != nil {
				go func() {
					assert.Nil(t, RootCmd.Execute())
				}()
			}

			if c.wait != nil {
				var count = 0
				for c.wait() {
					t.Logf("Ports are not yet open, retrying attempt #%d...", count)
					count++
					if count > 15 {
						t.FailNow()
					}
					time.Sleep(time.Second)
				}
			} else {
				err := RootCmd.Execute()
				if c.expectErr {
					assert.Error(t, err)
				} else {
					assert.NoError(t, err)
				}
			}
		})
	}
}
<commit_msg>cmd: Fix flaky port finder<commit_after>/*
 * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 */

package cmd

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"os"
	"testing"
	"time"

	"github.com/phayes/freeport"
	"github.com/stretchr/testify/assert"
)

var frontendPort, backendPort int

func freePort() (int, int) {
	var err error
	r := make([]int, 2)

	if r[0], err = freeport.GetFreePort(); err != nil {
		panic(err.Error())
	}

	tries := 0
	for {
		r[1], err = freeport.GetFreePort()
		if r[0] != r[1] {
			break
		}
		tries++
		if tries > 10 {
			panic("Unable to find free port")
		}
	}
	return r[0], r[1]
}

func init() {
	frontendPort, backendPort = freePort()

	os.Setenv("PUBLIC_PORT", fmt.Sprintf("%d", frontendPort))
	os.Setenv("ADMIN_PORT", fmt.Sprintf("%d", backendPort))
	os.Setenv("DATABASE_URL", "memory")
	//os.Setenv("HYDRA_URL", fmt.Sprintf("https://localhost:%d/", frontendPort))
	os.Setenv("OAUTH2_ISSUER_URL", fmt.Sprintf("https://localhost:%d/", frontendPort))
}

func TestExecute(t *testing.T) {
	var osArgs = make([]string, len(os.Args))
	copy(osArgs, os.Args)

	frontend := fmt.Sprintf("https://localhost:%d/", frontendPort)
	backend := fmt.Sprintf("https://localhost:%d/", backendPort)

	for _, c := range []struct {
		args      []string
		wait      func() bool
		expectErr bool
	}{
		{
			args: []string{"serve", "all", "--disable-telemetry"},
			wait: func() bool {
				client := &http.Client{
					Transport: &transporter{
						FakeTLSTermination: true,
						Transport: &http.Transport{
							TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
						},
					},
				}

				for _, u := range []string{
					fmt.Sprintf("https://127.0.0.1:%d/.well-known/openid-configuration", frontendPort),
					fmt.Sprintf("https://127.0.0.1:%d/health/status", backendPort),
				} {
					if resp, err := client.Get(u); err != nil {
						t.Logf("HTTP request to %s failed: %s", u, err)
						return true
					} else if resp.StatusCode != http.StatusOK {
						t.Logf("HTTP request to %s got status code %d but expected was 200", u, resp.StatusCode)
						return true
					}
				}

				// Give a bit more time to initialize
				time.Sleep(time.Second * 5)
				return false
			},
		},
		{args: []string{"clients", "create", "--endpoint", backend, "--id", "foobarbaz", "--secret", "foobar", "-g", "client_credentials"}},
		{args: []string{"clients", "get", "--endpoint", backend, "foobarbaz"}},
		{args: []string{"clients", "create", "--endpoint", backend, "--id", "public-foo"}},
		{args: []string{"clients", "delete", "--endpoint", backend, "public-foo"}},
		{args: []string{"keys", "create", "foo", "--endpoint", backend, "-a", "HS256"}},
		{args: []string{"keys", "get", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "rotate", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "get", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "delete", "--endpoint", backend, "foo"}},
		{args: []string{"keys", "import", "--endpoint", backend, "import-1", "../test/stub/ecdh.key", "../test/stub/ecdh.pub"}},
		{args: []string{"keys", "import", "--endpoint", backend, "import-2", "../test/stub/rsa.key", "../test/stub/rsa.pub"}},
		{args: []string{"token", "revoke", "--endpoint", frontend, "--client-secret", "foobar", "--client-id", "foobarbaz", "foo"}},
		{args: []string{"token", "client", "--endpoint", frontend, "--client-secret", "foobar", "--client-id", "foobarbaz"}},
		{args: []string{"help", "migrate", "sql"}},
		{args: []string{"version"}},
		{args: []string{"token", "flush", "--endpoint", backend}},
	} {
		c.args = append(c.args, []string{"--skip-tls-verify"}...)
		RootCmd.SetArgs(c.args)

		t.Run(fmt.Sprintf("command=%v", c.args), func(t *testing.T) {
			if c.wait != nil {
				go func() {
					assert.Nil(t, RootCmd.Execute())
				}()
			}

			if c.wait != nil {
				var count = 0
				for c.wait() {
					t.Logf("Ports are not yet open, retrying attempt #%d...", count)
					count++
					if count > 15 {
						t.FailNow()
					}
					time.Sleep(time.Second)
				}
			} else {
				err := RootCmd.Execute()
				if c.expectErr {
					assert.Error(t, err)
				} else {
					assert.NoError(t, err)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/getcarina/carina/console"
	"github.com/spf13/cobra"
)

func newTemplatesCommand() *cobra.Command {
	var options struct {
		name  string
	}

	var cmd = &cobra.Command{
		Use:               "templates",
		Short:             "List cluster templates",
		Long:              "List cluster templates",
		PersistentPreRunE: authenticatedPreRunE,
		RunE: func(cmd *cobra.Command, args []string) error {
			templates, err := cxt.Client.ListClusterTemplates(cxt.Account, options.name)
			if err != nil {
				return err
			}

			data := [][]string{[]string{"Name", "COE", "Host"}}
			for _, template := range templates {
				data = append(data, []string{template.GetName(), template.GetCOE(), template.GetHostType()})
			}
			console.WriteTable(data)

			return nil
		},
	}

	cmd.Flags().StringVar(&options.name, "name", "", "Filter by name, e.g. Kubernetes*")
	cmd.SetUsageTemplate(cmd.UsageTemplate())

	return cmd
}
<commit_msg>Make go fmt happy<commit_after>package cmd

import (
	"github.com/getcarina/carina/console"
	"github.com/spf13/cobra"
)

func newTemplatesCommand() *cobra.Command {
	var options struct {
		name string
	}

	var cmd = &cobra.Command{
		Use:               "templates",
		Short:             "List cluster templates",
		Long:              "List cluster templates",
		PersistentPreRunE: authenticatedPreRunE,
		RunE: func(cmd *cobra.Command, args []string) error {
			templates, err := cxt.Client.ListClusterTemplates(cxt.Account, options.name)
			if err != nil {
				return err
			}

			data := [][]string{[]string{"Name", "COE", "Host"}}
			for _, template := range templates {
				data = append(data, []string{template.GetName(), template.GetCOE(), template.GetHostType()})
			}
			console.WriteTable(data)

			return nil
		},
	}

	cmd.Flags().StringVar(&options.name, "name", "", "Filter by name, e.g. Kubernetes*")
	cmd.SetUsageTemplate(cmd.UsageTemplate())

	return cmd
}
<|endoftext|>


<commit_before>package arm

import (
	"encoding/json"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/Azure/azure-sdk-for-go/arm/compute"
	"github.com/Azure/azure-sdk-for-go/arm/disk"
	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/Azure/azure-sdk-for-go/arm/resources/resources"
	armStorage "github.com/Azure/azure-sdk-for-go/arm/storage"
	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/hashicorp/packer/builder/azure/common"
	"github.com/hashicorp/packer/helper/useragent"
)

const (
	EnvPackerLogAzureMaxLen = "PACKER_LOG_AZURE_MAXLEN"
)

type AzureClient struct {
	storage.BlobStorageClient
	resources.DeploymentsClient
	resources.DeploymentOperationsClient
	resources.GroupsClient
	network.PublicIPAddressesClient
	network.InterfacesClient
	network.SubnetsClient
	network.VirtualNetworksClient
	compute.ImagesClient
	compute.VirtualMachinesClient
	common.VaultClient
	armStorage.AccountsClient
	disk.DisksClient

	InspectorMaxLength int
	Template           *CaptureTemplate
	LastError          azureErrorResponse
	VaultClientDelete  common.VaultClient
}

func getCaptureResponse(body string) *CaptureTemplate {
	var operation CaptureOperation
	err := json.Unmarshal([]byte(body), &operation)
	if err != nil {
		return nil
	}

	if operation.Properties != nil && operation.Properties.Output != nil {
		return operation.Properties.Output
	}

	return nil
}

// HACK(chrboum): This method is a hack.  It was written to work around this issue
// (https://github.com/Azure/azure-sdk-for-go/issues/307) and to an extent this
// issue (https://github.com/Azure/azure-rest-api-specs/issues/188).
//
// Capturing a VM is a long running operation that requires polling.  There are
// couple different forms of polling, and the end result of a poll operation is
// discarded by the SDK.  It is expected that any discarded data can be re-fetched,
// so discarding it has minimal impact.  Unfortunately, there is no way to re-fetch
// the template returned by a capture call that I am aware of.
//
// If the second issue were fixed the VM ID would be included when GET'ing a VM.  The
// VM ID could be used to locate the captured VHD, and captured template.
// Unfortunately, the VM ID is not included so this method cannot be used either.
//
// This code captures the template and saves it to the client (the AzureClient type).
// It expects that the capture API is called only once, or rather you only care that the
// last call's value is important because subsequent requests are not persisted.  There
// is no care given to multiple threads writing this value because for our use case
// it does not matter.
func templateCapture(client *AzureClient) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			body, bodyString := handleBody(resp.Body, math.MaxInt64)
			resp.Body = body

			captureTemplate := getCaptureResponse(bodyString)
			if captureTemplate != nil {
				client.Template = captureTemplate
			}

			return r.Respond(resp)
		})
	}
}

func errorCapture(client *AzureClient) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			body, bodyString := handleBody(resp.Body, math.MaxInt64)
			resp.Body = body

			errorResponse := newAzureErrorResponse(bodyString)
			if errorResponse != nil {
				client.LastError = *errorResponse
			}

			return r.Respond(resp)
		})
	}
}

// WAITING(chrboum): I have logged https://github.com/Azure/azure-sdk-for-go/issues/311 to get this
// method included in the SDK.  It has been accepted, and I'll cut over to the official way
// once it ships.
func byConcatDecorators(decorators ...autorest.RespondDecorator) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.DecorateResponder(r, decorators...)
	}
}

func NewAzureClient(subscriptionID, resourceGroupName, storageAccountName string,
	cloud *azure.Environment,
	servicePrincipalToken, servicePrincipalTokenVault *adal.ServicePrincipalToken) (*AzureClient, error) {

	var azureClient = &AzureClient{}

	maxlen := getInspectorMaxLength()

	azureClient.DeploymentsClient = resources.NewDeploymentsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DeploymentsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DeploymentsClient.RequestInspector = withInspection(maxlen)
	azureClient.DeploymentsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DeploymentsClient.UserAgent = useragent.String()

	azureClient.DeploymentOperationsClient = resources.NewDeploymentOperationsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DeploymentOperationsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DeploymentOperationsClient.RequestInspector = withInspection(maxlen)
	azureClient.DeploymentOperationsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DeploymentOperationsClient.UserAgent = useragent.String()

	azureClient.DisksClient = disk.NewDisksClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DisksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DisksClient.RequestInspector = withInspection(maxlen)
	azureClient.DisksClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DisksClient.UserAgent = useragent.String()

	azureClient.GroupsClient = resources.NewGroupsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.GroupsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.GroupsClient.RequestInspector = withInspection(maxlen)
	azureClient.GroupsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.GroupsClient.UserAgent = useragent.String()

	azureClient.ImagesClient = compute.NewImagesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.ImagesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.ImagesClient.RequestInspector = withInspection(maxlen)
	azureClient.ImagesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.ImagesClient.UserAgent = useragent.String()

	azureClient.InterfacesClient = network.NewInterfacesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.InterfacesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.InterfacesClient.RequestInspector = withInspection(maxlen)
	azureClient.InterfacesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.InterfacesClient.UserAgent = useragent.String()

	azureClient.SubnetsClient = network.NewSubnetsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.SubnetsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.SubnetsClient.RequestInspector = withInspection(maxlen)
	azureClient.SubnetsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.SubnetsClient.UserAgent = useragent.String()

	azureClient.VirtualNetworksClient = network.NewVirtualNetworksClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VirtualNetworksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VirtualNetworksClient.RequestInspector = withInspection(maxlen)
	azureClient.VirtualNetworksClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VirtualNetworksClient.UserAgent = useragent.String()

	azureClient.PublicIPAddressesClient = network.NewPublicIPAddressesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.PublicIPAddressesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.PublicIPAddressesClient.RequestInspector = withInspection(maxlen)
	azureClient.PublicIPAddressesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.PublicIPAddressesClient.UserAgent = useragent.String()

	azureClient.VirtualMachinesClient = compute.NewVirtualMachinesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VirtualMachinesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VirtualMachinesClient.RequestInspector = withInspection(maxlen)
	azureClient.VirtualMachinesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), templateCapture(azureClient), errorCapture(azureClient))
	azureClient.VirtualMachinesClient.UserAgent = useragent.String()

	azureClient.AccountsClient = armStorage.NewAccountsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.AccountsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.AccountsClient.RequestInspector = withInspection(maxlen)
	azureClient.AccountsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.AccountsClient.UserAgent = useragent.String()

	keyVaultURL, err := url.Parse(cloud.KeyVaultEndpoint)
	if err != nil {
		return nil, err
	}

	azureClient.VaultClient = common.NewVaultClient(*keyVaultURL)
	azureClient.VaultClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalTokenVault)
	azureClient.VaultClient.RequestInspector = withInspection(maxlen)
	azureClient.VaultClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VaultClient.UserAgent = useragent.String()

	// TODO(boumenot) - SDK still does not have a full KeyVault client.
	// There are two ways that KeyVault has to be accessed, and each one has their own SPN.  An authenticated SPN
	// is tied to the URL, and the URL associated with getting the secret is different than the URL
	// associated with deleting the KeyVault.  As a result, I need to have *two* different clients to
	// access KeyVault.  I did not want to split it into two separate files, so I am starting with this.
	//
	// I do not like this implementation.  It is getting long in the tooth, and should be re-examined now
	// that we have a "working" solution.
	azureClient.VaultClientDelete = common.NewVaultClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VaultClientDelete.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VaultClientDelete.RequestInspector = withInspection(maxlen)
	azureClient.VaultClientDelete.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VaultClientDelete.UserAgent = useragent.String()

	// If this is a managed disk build, this should be ignored.
	if resourceGroupName != "" && storageAccountName != "" {
		accountKeys, err := azureClient.AccountsClient.ListKeys(resourceGroupName, storageAccountName)
		if err != nil {
			return nil, err
		}

		storageClient, err := storage.NewClient(
			storageAccountName,
			*(*accountKeys.Keys)[0].Value,
			cloud.StorageEndpointSuffix,
			storage.DefaultAPIVersion,
			true /*useHttps*/)

		if err != nil {
			return nil, err
		}

		azureClient.BlobStorageClient = storageClient.GetBlobService()
	}

	return azureClient, nil
}

func getInspectorMaxLength() int64 {
	value, ok := os.LookupEnv(EnvPackerLogAzureMaxLen)
	if !ok {
		return math.MaxInt64
	}

	i, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return 0
	}

	if i < 0 {
		return 0
	}

	return i
}
<commit_msg>Include existing azure user agent<commit_after>package arm

import (
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/Azure/azure-sdk-for-go/arm/compute"
	"github.com/Azure/azure-sdk-for-go/arm/disk"
	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/Azure/azure-sdk-for-go/arm/resources/resources"
	armStorage "github.com/Azure/azure-sdk-for-go/arm/storage"
	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/hashicorp/packer/builder/azure/common"
	"github.com/hashicorp/packer/helper/useragent"
)

const (
	EnvPackerLogAzureMaxLen = "PACKER_LOG_AZURE_MAXLEN"
)

type AzureClient struct {
	storage.BlobStorageClient
	resources.DeploymentsClient
	resources.DeploymentOperationsClient
	resources.GroupsClient
	network.PublicIPAddressesClient
	network.InterfacesClient
	network.SubnetsClient
	network.VirtualNetworksClient
	compute.ImagesClient
	compute.VirtualMachinesClient
	common.VaultClient
	armStorage.AccountsClient
	disk.DisksClient

	InspectorMaxLength int
	Template           *CaptureTemplate
	LastError          azureErrorResponse
	VaultClientDelete  common.VaultClient
}

func getCaptureResponse(body string) *CaptureTemplate {
	var operation CaptureOperation
	err := json.Unmarshal([]byte(body), &operation)
	if err != nil {
		return nil
	}

	if operation.Properties != nil && operation.Properties.Output != nil {
		return operation.Properties.Output
	}

	return nil
}

// HACK(chrboum): This method is a hack.  It was written to work around this issue
// (https://github.com/Azure/azure-sdk-for-go/issues/307) and to an extent this
// issue (https://github.com/Azure/azure-rest-api-specs/issues/188).
//
// Capturing a VM is a long running operation that requires polling.  There are
// couple different forms of polling, and the end result of a poll operation is
// discarded by the SDK.  It is expected that any discarded data can be re-fetched,
// so discarding it has minimal impact.  Unfortunately, there is no way to re-fetch
// the template returned by a capture call that I am aware of.
//
// If the second issue were fixed the VM ID would be included when GET'ing a VM.  The
// VM ID could be used to locate the captured VHD, and captured template.
// Unfortunately, the VM ID is not included so this method cannot be used either.
//
// This code captures the template and saves it to the client (the AzureClient type).
// It expects that the capture API is called only once, or rather you only care that the
// last call's value is important because subsequent requests are not persisted.  There
// is no care given to multiple threads writing this value because for our use case
// it does not matter.
func templateCapture(client *AzureClient) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			body, bodyString := handleBody(resp.Body, math.MaxInt64)
			resp.Body = body

			captureTemplate := getCaptureResponse(bodyString)
			if captureTemplate != nil {
				client.Template = captureTemplate
			}

			return r.Respond(resp)
		})
	}
}

func errorCapture(client *AzureClient) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.ResponderFunc(func(resp *http.Response) error {
			body, bodyString := handleBody(resp.Body, math.MaxInt64)
			resp.Body = body

			errorResponse := newAzureErrorResponse(bodyString)
			if errorResponse != nil {
				client.LastError = *errorResponse
			}

			return r.Respond(resp)
		})
	}
}

// WAITING(chrboum): I have logged https://github.com/Azure/azure-sdk-for-go/issues/311 to get this
// method included in the SDK.  It has been accepted, and I'll cut over to the official way
// once it ships.
func byConcatDecorators(decorators ...autorest.RespondDecorator) autorest.RespondDecorator {
	return func(r autorest.Responder) autorest.Responder {
		return autorest.DecorateResponder(r, decorators...)
	}
}

func NewAzureClient(subscriptionID, resourceGroupName, storageAccountName string,
	cloud *azure.Environment,
	servicePrincipalToken, servicePrincipalTokenVault *adal.ServicePrincipalToken) (*AzureClient, error) {

	var azureClient = &AzureClient{}

	maxlen := getInspectorMaxLength()

	azureClient.DeploymentsClient = resources.NewDeploymentsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DeploymentsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DeploymentsClient.RequestInspector = withInspection(maxlen)
	azureClient.DeploymentsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DeploymentsClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.DeploymentsClient.UserAgent)

	azureClient.DeploymentOperationsClient = resources.NewDeploymentOperationsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DeploymentOperationsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DeploymentOperationsClient.RequestInspector = withInspection(maxlen)
	azureClient.DeploymentOperationsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DeploymentOperationsClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.DeploymentOperationsClient.UserAgent)

	azureClient.DisksClient = disk.NewDisksClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.DisksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.DisksClient.RequestInspector = withInspection(maxlen)
	azureClient.DisksClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.DisksClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.DisksClient.UserAgent)

	azureClient.GroupsClient = resources.NewGroupsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.GroupsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.GroupsClient.RequestInspector = withInspection(maxlen)
	azureClient.GroupsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.GroupsClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.GroupsClient.UserAgent)

	azureClient.ImagesClient = compute.NewImagesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.ImagesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.ImagesClient.RequestInspector = withInspection(maxlen)
	azureClient.ImagesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.ImagesClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.ImagesClient.UserAgent)

	azureClient.InterfacesClient = network.NewInterfacesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.InterfacesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.InterfacesClient.RequestInspector = withInspection(maxlen)
	azureClient.InterfacesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.InterfacesClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.InterfacesClient.UserAgent)

	azureClient.SubnetsClient = network.NewSubnetsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.SubnetsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.SubnetsClient.RequestInspector = withInspection(maxlen)
	azureClient.SubnetsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.SubnetsClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.SubnetsClient.UserAgent)

	azureClient.VirtualNetworksClient = network.NewVirtualNetworksClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VirtualNetworksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VirtualNetworksClient.RequestInspector = withInspection(maxlen)
	azureClient.VirtualNetworksClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VirtualNetworksClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.VirtualNetworksClient.UserAgent)

	azureClient.PublicIPAddressesClient = network.NewPublicIPAddressesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.PublicIPAddressesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.PublicIPAddressesClient.RequestInspector = withInspection(maxlen)
	azureClient.PublicIPAddressesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.PublicIPAddressesClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.PublicIPAddressesClient.UserAgent)

	azureClient.VirtualMachinesClient = compute.NewVirtualMachinesClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VirtualMachinesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VirtualMachinesClient.RequestInspector = withInspection(maxlen)
	azureClient.VirtualMachinesClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), templateCapture(azureClient), errorCapture(azureClient))
	azureClient.VirtualMachinesClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.VirtualMachinesClient.UserAgent)

	azureClient.AccountsClient = armStorage.NewAccountsClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.AccountsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.AccountsClient.RequestInspector = withInspection(maxlen)
	azureClient.AccountsClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.AccountsClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.AccountsClient.UserAgent)

	keyVaultURL, err := url.Parse(cloud.KeyVaultEndpoint)
	if err != nil {
		return nil, err
	}

	azureClient.VaultClient = common.NewVaultClient(*keyVaultURL)
	azureClient.VaultClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalTokenVault)
	azureClient.VaultClient.RequestInspector = withInspection(maxlen)
	azureClient.VaultClient.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VaultClient.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.VaultClient.UserAgent)

	// TODO(boumenot) - SDK still does not have a full KeyVault client.
	// There are two ways that KeyVault has to be accessed, and each one has their own SPN.  An authenticated SPN
	// is tied to the URL, and the URL associated with getting the secret is different than the URL
	// associated with deleting the KeyVault.  As a result, I need to have *two* different clients to
	// access KeyVault.  I did not want to split it into two separate files, so I am starting with this.
	//
	// I do not like this implementation.  It is getting long in the tooth, and should be re-examined now
	// that we have a "working" solution.
	azureClient.VaultClientDelete = common.NewVaultClientWithBaseURI(cloud.ResourceManagerEndpoint, subscriptionID)
	azureClient.VaultClientDelete.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
	azureClient.VaultClientDelete.RequestInspector = withInspection(maxlen)
	azureClient.VaultClientDelete.ResponseInspector = byConcatDecorators(byInspecting(maxlen), errorCapture(azureClient))
	azureClient.VaultClientDelete.UserAgent = fmt.Sprintf("%s %s", useragent.String(), azureClient.VaultClientDelete.UserAgent)

	// If this is a managed disk build, this should be ignored.
	if resourceGroupName != "" && storageAccountName != "" {
		accountKeys, err := azureClient.AccountsClient.ListKeys(resourceGroupName, storageAccountName)
		if err != nil {
			return nil, err
		}

		storageClient, err := storage.NewClient(
			storageAccountName,
			*(*accountKeys.Keys)[0].Value,
			cloud.StorageEndpointSuffix,
			storage.DefaultAPIVersion,
			true /*useHttps*/)

		if err != nil {
			return nil, err
		}

		azureClient.BlobStorageClient = storageClient.GetBlobService()
	}

	return azureClient, nil
}

func getInspectorMaxLength() int64 {
	value, ok := os.LookupEnv(EnvPackerLogAzureMaxLen)
	if !ok {
		return math.MaxInt64
	}

	i, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return 0
	}

	if i < 0 {
		return 0
	}

	return i
}
<|endoftext|>


<commit_before>package keeper

import (
	"context"

	"github.com/cosmos/cosmos-sdk/store/prefix"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/cosmos/cosmos-sdk/types/query"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/vendorinfo/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func (k Keeper) VendorInfoAll(c context.Context, req *types.QueryAllVendorInfoRequest) (*types.QueryAllVendorInfoResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}

	var vendorInfos []types.VendorInfo
	ctx := sdk.UnwrapSDKContext(c)

	store := ctx.KVStore(k.storeKey)
	vendorInfoStore := prefix.NewStore(store, types.KeyPrefix(types.VendorInfoKeyPrefix))

	pageRes, err := query.Paginate(vendorInfoStore, req.Pagination, func(key []byte, value []byte) error {
		var vendorInfo types.VendorInfo
		if err := k.cdc.Unmarshal(value, &vendorInfo); err != nil {
			return err
		}

		vendorInfos = append(vendorInfos, vendorInfo)
		return nil
	})

	if err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &types.QueryAllVendorInfoResponse{VendorInfo: vendorInfos, Pagination: pageRes}, nil
}

func (k Keeper) VendorInfo(c context.Context, req *types.QueryGetVendorInfoRequest) (*types.QueryGetVendorInfoResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}
	ctx := sdk.UnwrapSDKContext(c)

	val, found := k.GetVendorInfo(
		ctx,
		req.VendorID,
	)
	if !found {
		return nil, status.Error(codes.InvalidArgument, "not found")
	}

	return &types.QueryGetVendorInfoResponse{VendorInfo: val}, nil
}
<commit_msg>updated error handling<commit_after>package keeper

import (
	"context"

	"github.com/cosmos/cosmos-sdk/store/prefix"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/cosmos/cosmos-sdk/types/query"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/vendorinfo/types"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func (k Keeper) VendorInfoAll(c context.Context, req *types.QueryAllVendorInfoRequest) (*types.QueryAllVendorInfoResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}

	var vendorInfos []types.VendorInfo
	ctx := sdk.UnwrapSDKContext(c)

	store := ctx.KVStore(k.storeKey)
	vendorInfoStore := prefix.NewStore(store, types.KeyPrefix(types.VendorInfoKeyPrefix))

	pageRes, err := query.Paginate(vendorInfoStore, req.Pagination, func(key []byte, value []byte) error {
		var vendorInfo types.VendorInfo
		if err := k.cdc.Unmarshal(value, &vendorInfo); err != nil {
			return err
		}

		vendorInfos = append(vendorInfos, vendorInfo)
		return nil
	})

	if err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &types.QueryAllVendorInfoResponse{VendorInfo: vendorInfos, Pagination: pageRes}, nil
}

func (k Keeper) VendorInfo(c context.Context, req *types.QueryGetVendorInfoRequest) (*types.QueryGetVendorInfoResponse, error) {
	if req == nil {
		return nil, status.Error(codes.InvalidArgument, "invalid request")
	}
	ctx := sdk.UnwrapSDKContext(c)

	val, found := k.GetVendorInfo(
		ctx,
		req.VendorID,
	)
	if !found {
		return nil, status.Error(codes.NotFound, "not found")
	}

	return &types.QueryGetVendorInfoResponse{VendorInfo: val}, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package csv

import (
	"bufio"
	"io"
	"strings"
	"unicode"
	"unicode/utf8"
)

// A Writer writes records to a CSV encoded file.
//
// As returned by NewWriter, a Writer writes records terminated by a
// newline and uses ',' as the field delimiter. The exported fields can be
// changed to customize the details before the first call to Write or WriteAll.
//
// Comma is the field delimiter.
//
// If UseCRLF is true, the Writer ends each output line with \r\n instead of \n.
type Writer struct {
	Comma   rune // Field delimiter (set to ',' by NewWriter)
	UseCRLF bool // True to use \r\n as the line terminator
	w       *bufio.Writer
}

// NewWriter returns a new Writer that writes to w.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Comma: ',',
		w:     bufio.NewWriter(w),
	}
}

// Writer writes a single CSV record to w along with any necessary quoting.
// A record is a slice of strings with each string being one field.
func (w *Writer) Write(record []string) error {
	if !validDelim(w.Comma) {
		return errInvalidDelim
	}

	for n, field := range record {
		if n > 0 {
			if _, err := w.w.WriteRune(w.Comma); err != nil {
				return err
			}
		}

		// If we don't have to have a quoted field then just
		// write out the field and continue to the next field.
		if !w.fieldNeedsQuotes(field) {
			if _, err := w.w.WriteString(field); err != nil {
				return err
			}
			continue
		}

		if err := w.w.WriteByte('"'); err != nil {
			return err
		}
		for len(field) > 0 {
			// Search for special characters.
			i := strings.IndexAny(field, "\"\r\n")
			if i < 0 {
				i = len(field)
			}

			// Copy verbatim everything before the special character.
			if _, err := w.w.WriteString(field[:i]); err != nil {
				return err
			}
			field = field[i:]

			// Encode the special character.
			if len(field) > 0 {
				var err error
				switch field[0] {
				case '"':
					_, err = w.w.WriteString(`""`)
				case '\r':
					if !w.UseCRLF {
						err = w.w.WriteByte('\r')
					}
				case '\n':
					if w.UseCRLF {
						_, err = w.w.WriteString("\r\n")
					} else {
						err = w.w.WriteByte('\n')
					}
				}
				field = field[1:]
				if err != nil {
					return err
				}
			}
		}
		if err := w.w.WriteByte('"'); err != nil {
			return err
		}
	}
	var err error
	if w.UseCRLF {
		_, err = w.w.WriteString("\r\n")
	} else {
		err = w.w.WriteByte('\n')
	}
	return err
}

// Flush writes any buffered data to the underlying io.Writer.
// To check if an error occurred during the Flush, call Error.
func (w *Writer) Flush() {
	w.w.Flush()
}

// Error reports any error that has occurred during a previous Write or Flush.
func (w *Writer) Error() error {
	_, err := w.w.Write(nil)
	return err
}

// WriteAll writes multiple CSV records to w using Write and then calls Flush.
func (w *Writer) WriteAll(records [][]string) error {
	for _, record := range records {
		err := w.Write(record)
		if err != nil {
			return err
		}
	}
	return w.w.Flush()
}

// fieldNeedsQuotes reports whether our field must be enclosed in quotes.
// Fields with a Comma, fields with a quote or newline, and
// fields which start with a space must be enclosed in quotes.
// We used to quote empty strings, but we do not anymore (as of Go 1.4).
// The two representations should be equivalent, but Postgres distinguishes
// quoted vs non-quoted empty string during database imports, and it has
// an option to force the quoted behavior for non-quoted CSV but it has
// no option to force the non-quoted behavior for quoted CSV, making
// CSV with quoted empty strings strictly less useful.
// Not quoting the empty string also makes this package match the behavior
// of Microsoft Excel and Google Drive.
// For Postgres, quote the data terminating string `\.`.
func (w *Writer) fieldNeedsQuotes(field string) bool {
	if field == "" {
		return false
	}
	if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") {
		return true
	}

	r1, _ := utf8.DecodeRuneInString(field)
	return unicode.IsSpace(r1)
}
<commit_msg>encoding/csv: document that Writer is buffered<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package csv

import (
	"bufio"
	"io"
	"strings"
	"unicode"
	"unicode/utf8"
)

// A Writer writes records using CSV encoding.
//
// As returned by NewWriter, a Writer writes records terminated by a
// newline and uses ',' as the field delimiter. The exported fields can be
// changed to customize the details before the first call to Write or WriteAll.
//
// Comma is the field delimiter.
//
// If UseCRLF is true, the Writer ends each output line with \r\n instead of \n.
//
// The writes of individual records are buffered.
// After all data has been written, the client should call the
// Flush method to guarantee all data has been forwarded to
// the underlying io.Writer.  Any errors that occurred should
// be checked by calling the Error method.
type Writer struct {
	Comma   rune // Field delimiter (set to ',' by NewWriter)
	UseCRLF bool // True to use \r\n as the line terminator
	w       *bufio.Writer
}

// NewWriter returns a new Writer that writes to w.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Comma: ',',
		w:     bufio.NewWriter(w),
	}
}

// Writer writes a single CSV record to w along with any necessary quoting.
// A record is a slice of strings with each string being one field.
// Writes are buffered, so Flush must eventually be called to ensure
// that the record is written to the underlying io.Writer.
func (w *Writer) Write(record []string) error {
	if !validDelim(w.Comma) {
		return errInvalidDelim
	}

	for n, field := range record {
		if n > 0 {
			if _, err := w.w.WriteRune(w.Comma); err != nil {
				return err
			}
		}

		// If we don't have to have a quoted field then just
		// write out the field and continue to the next field.
		if !w.fieldNeedsQuotes(field) {
			if _, err := w.w.WriteString(field); err != nil {
				return err
			}
			continue
		}

		if err := w.w.WriteByte('"'); err != nil {
			return err
		}
		for len(field) > 0 {
			// Search for special characters.
			i := strings.IndexAny(field, "\"\r\n")
			if i < 0 {
				i = len(field)
			}

			// Copy verbatim everything before the special character.
			if _, err := w.w.WriteString(field[:i]); err != nil {
				return err
			}
			field = field[i:]

			// Encode the special character.
			if len(field) > 0 {
				var err error
				switch field[0] {
				case '"':
					_, err = w.w.WriteString(`""`)
				case '\r':
					if !w.UseCRLF {
						err = w.w.WriteByte('\r')
					}
				case '\n':
					if w.UseCRLF {
						_, err = w.w.WriteString("\r\n")
					} else {
						err = w.w.WriteByte('\n')
					}
				}
				field = field[1:]
				if err != nil {
					return err
				}
			}
		}
		if err := w.w.WriteByte('"'); err != nil {
			return err
		}
	}
	var err error
	if w.UseCRLF {
		_, err = w.w.WriteString("\r\n")
	} else {
		err = w.w.WriteByte('\n')
	}
	return err
}

// Flush writes any buffered data to the underlying io.Writer.
// To check if an error occurred during the Flush, call Error.
func (w *Writer) Flush() {
	w.w.Flush()
}

// Error reports any error that has occurred during a previous Write or Flush.
func (w *Writer) Error() error {
	_, err := w.w.Write(nil)
	return err
}

// WriteAll writes multiple CSV records to w using Write and then calls Flush,
// returning any error from the Flush.
func (w *Writer) WriteAll(records [][]string) error {
	for _, record := range records {
		err := w.Write(record)
		if err != nil {
			return err
		}
	}
	return w.w.Flush()
}

// fieldNeedsQuotes reports whether our field must be enclosed in quotes.
// Fields with a Comma, fields with a quote or newline, and
// fields which start with a space must be enclosed in quotes.
// We used to quote empty strings, but we do not anymore (as of Go 1.4).
// The two representations should be equivalent, but Postgres distinguishes
// quoted vs non-quoted empty string during database imports, and it has
// an option to force the quoted behavior for non-quoted CSV but it has
// no option to force the non-quoted behavior for quoted CSV, making
// CSV with quoted empty strings strictly less useful.
// Not quoting the empty string also makes this package match the behavior
// of Microsoft Excel and Google Drive.
// For Postgres, quote the data terminating string `\.`.
func (w *Writer) fieldNeedsQuotes(field string) bool {
	if field == "" {
		return false
	}
	if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") {
		return true
	}

	r1, _ := utf8.DecodeRuneInString(field)
	return unicode.IsSpace(r1)
}
<|endoftext|>


<commit_before>package code_gen

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"yum/ast"
	"yum/code_gen/op"

	"github.com/alecthomas/repr"
)

func emitString(b *bytes.Buffer, address byte, v int32) {
	b.WriteByte(op.I32_CONST)
	b.WriteByte(address)
	b.WriteByte(op.I32_CONST)
	encodeSleb128(b, v)
	b.WriteByte(op.I32_STORE)
	b.WriteByte(byte(0x02))
	b.WriteByte(byte(0x00))
}

var typeMap = map[string]byte{
	"num": op.F64,
	"i32": op.I32,
}

type FuncData struct {
	Index      int
	Name       string
	Params     map[string]FuncParam
	ReturnType string
}

type FuncParam struct {
	Index int
	Name  string
	Type  string
}

func emitOperation(buffer *bytes.Buffer, funcName, operation string) error {
	switch operation {
	case "+":
		buffer.WriteByte(op.F64_ADD)
	case "-":
		buffer.WriteByte(op.F64_SUB)
	case "*":
		buffer.WriteByte(op.F64_MUL)
	case "/":
		buffer.WriteByte(op.F64_DIV)
	default:
		return fmt.Errorf("func '%s' operation '%s' is invalid", funcName, operation)
	}
	return nil
}

func emitExpression(buffer *bytes.Buffer, funcName string, funcSymbolTable map[string]FuncData, operation *string, l *ast.Literal, r *ast.Expression) error {
	if l != nil && l.Num != nil {
		buffer.WriteByte(op.F64_CONST)
		buffer.Write(float64ToByte(*l.Num))
	}
	if l != nil && l.Str != nil {
		// (i32.store (i32.const 0) (i32.const 8))  ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string
		// (i32.store (i32.const 4) (i32.const 12))  ;; iov.iov_len - The length of the 'hello world\n' string
		// (i32.store (i32.const 8) (i32.const 0x6c6c6568))
		// (i32.store (i32.const 12) (i32.const 0x6f77206f))
		// (i32.store (i32.const 16) (i32.const 0x0a646c72))
		// buffer.WriteByte(op.I32_CONST)
		// buffer.WriteByte(byte(0))
	}
	// TODO: remove r
	if r != nil && r.Left != nil && r.Left.Num != nil {
		buffer.WriteByte(op.F64_CONST)
		buffer.Write(float64ToByte(*r.Left.Num))
		emitOperation(buffer, funcName, *operation)
	}
	if l != nil && l.Reference != nil {
		if operation != nil {
			ref, ok := funcSymbolTable[funcName].Params[*l.Reference]
			if !ok {
				return fmt.Errorf("func '%s' parameter '%s' does not exist", funcName, *l.Reference)
			}
			buffer.WriteByte(op.GET_LOCAL)
			buffer.WriteByte(byte(ref.Index))
		} else {
			// funcCALL
			callFunc, ok := funcSymbolTable[*l.Reference]
			if !ok {
				return fmt.Errorf("func '%s' trying to call non-existing func '%s'", funcName, *l.Reference)
			}
			if len(callFunc.Params) != len(r.Left.Params) {
				return fmt.Errorf("func '%s' trying to call function '%s' with extra parameters", funcName, callFunc.Name)
			}
			for _, v := range r.Left.Params {
				if v.Str != nil {
					str := *v.Str
					data := []byte(str)
					if len(data)%4 == 1 {
						data = append(data, 0)
					}
					if len(data)%4 == 2 {
						data = append(data, 0)
					}
					if len(data)%4 == 3 {
						data = append(data, '\n')
					}

					// (i32.store (i32.const 0) (i32.const 8))  ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string
					// (i32.store (i32.const 4) (i32.const 12))  ;; iov.iov_len - The length of the 'hello world\n' string
					// (i32.store (i32.const 8) (i32.const 0x6c6c6568))
					// (i32.store (i32.const 12) (i32.const 0x6f77206f))
					// (i32.store (i32.const 16) (i32.const 0x0a646c72))
					emitString(buffer, 0, 8)
					emitString(buffer, 4, int32(len(string(data))))
					startAddress := 0
					startData := 0
					for i := range data {
						index := i + 1
						if index%4 == 0 {
							startAddress = index + 4
							startData = index - 4
							remainingData := data[startData:index]
							emitString(buffer, byte(startAddress), int32(binary.LittleEndian.Uint32(remainingData)))
						}
					}
				}
			}
			for _, v := range r.Left.Params {
				if v.Num != nil {
					buffer.WriteByte(0x41)
					// TODO: fix this
					// this has to be float64 or int32
					buffer.WriteByte(byte(int32(*v.Num)))
				}
				if v.Reference != nil {
					vref, ok := funcSymbolTable[funcName].Params[*v.Reference]
					if !ok {
						return fmt.Errorf("func'" + funcName + "' trying to call '" + callFunc.Name + "' with non-existing variable " + *v.Reference)
					}
					buffer.WriteByte(op.GET_LOCAL)
					buffer.WriteByte(byte(vref.Index))
				}
				if v.Str != nil {
					buffer.WriteByte(op.I32_CONST)
					buffer.WriteByte(byte(0))
				}
			}
			buffer.WriteByte(op.CALL)
			buffer.WriteByte(byte(callFunc.Index))
			if r.Operator != nil {
				err := emitExpression(buffer, funcName, funcSymbolTable, r.Operator, nil, r.Right)
				if err != nil {
					return err
				}
			}
		}
	}
	if r != nil && r.Left != nil && r.Left.Reference != nil {
		ref, ok := funcSymbolTable[funcName].Params[*r.Left.Reference]
		if !ok {
			return fmt.Errorf("func'" + funcName + "' parameter '" + *r.Left.Reference + "' does not exist")
		}
		buffer.WriteByte(op.GET_LOCAL)
		buffer.WriteByte(byte(ref.Index))
		emitOperation(buffer, funcName, *operation)
	}
	if r != nil && r.Operator != nil {
		err := emitExpression(buffer, funcName, funcSymbolTable, r.Operator, nil, r.Right)
		if err != nil {
			return err
		}
	}
	return nil
}

// _pointer = a: string -> i32
// (i32.store (i32.const 0) (i32.const 8)) // iov.iov_base - This is a pointer to the start of the 'hello world\n' string

// _length = a: string -> i32
// (i32.store (i32.const 4) (i32.const 12)) // iov.iov_len - The length of the 'hello world\n' string

type Emitter struct {
	TypesSection     *bytes.Buffer
	ImportsSection   *bytes.Buffer
	FuncsSection     *bytes.Buffer
	ExportsSection   *bytes.Buffer
	FuncsBodySection *bytes.Buffer
	Tree             *ast.Ast
	funcSymbolTable  map[string]FuncData
}

func NewEmitter(tree *ast.Ast) Emitter {
	return Emitter{
		TypesSection:     bytes.NewBuffer(nil),
		ImportsSection:   bytes.NewBuffer(nil),
		FuncsSection:     bytes.NewBuffer(nil),
		ExportsSection:   bytes.NewBuffer(nil),
		FuncsBodySection: bytes.NewBuffer(nil),
		funcSymbolTable:  map[string]FuncData{},
		Tree:             tree,
	}
}

func (e Emitter) EmitTypes(fun FuncData) error {
	e.TypesSection.WriteByte(0x60)                  // type func
	e.TypesSection.WriteByte(byte(len(fun.Params))) // num params
	for _, param := range fun.Params {              // i32, i32
		t, ok := typeMap[param.Type]
		if !ok {
			return fmt.Errorf("func'" + fun.Name + "' parameter '" + param.Name + "' type '" + param.Type + "' does not exist")
		}
		e.TypesSection.WriteByte(t)
	}
	if fun.ReturnType != "" {
		t, ok := typeMap[fun.ReturnType]
		if !ok {
			return fmt.Errorf("func'" + fun.Name + "' return type '" + fun.ReturnType + "' does not exist")
		}
		e.TypesSection.WriteByte(0x01) // num results
		e.TypesSection.WriteByte(t)    // i32
	} else {
		e.TypesSection.WriteByte(0x00) // num results
	}
	// panic(repr.String(buffer.Bytes()))
	return nil
}

func (e Emitter) EmitImports(moduleName, funcName string, typeIndex int) {
	e.ImportsSection.WriteByte(byte(len(moduleName))) // module name length
	e.ImportsSection.Write([]byte(moduleName))        // module name
	e.ImportsSection.WriteByte(byte(len(funcName)))   // fn name length
	e.ImportsSection.Write([]byte(funcName))          // fn name
	e.ImportsSection.WriteByte(0x00)                  // import kind
	e.ImportsSection.WriteByte(byte(typeIndex))       // import signature index
}

func (e Emitter) EmitAll() (*bytes.Buffer, error) {
	buffer := bytes.NewBuffer(nil)
	buffer.Write([]byte{0x00, 0x61, 0x73, 0x6d}) // WASM_BINARY_MAGIC
	buffer.Write([]byte{0x01, 0x00, 0x00, 0x00}) // WASM_BINARY_VERSION

	funcs := []byte{}
	exports := []byte{}
	funcBodys := []byte{}
	externFuncsCount := 0
	funcsCount := 0
	funcSymbolTable := map[string]FuncData{}
	for _, p := range e.Tree.Modules {
		for _, a := range p.ExternFuncs {
			funcSymbolTable[a.Name] = FuncData{
				Index:      externFuncsCount,
				Name:       a.Name,
				Params:     map[string]FuncParam{},
				ReturnType: a.ReturnType,
			}
			for pi, param := range a.Parameters {
				funcSymbolTable[a.Name].Params[param.Name] = FuncParam{
					Index: pi,
					Name:  param.Name,
					Type:  param.Type.Name,
				}
			}
			err := e.EmitTypes(funcSymbolTable[a.Name])
			if err != nil {
				return nil, fmt.Errorf("Failed to emitTypes %v", err)
			}
			e.EmitImports(p.Name, a.Name, externFuncsCount)
			externFuncsCount += 1
		}
		for _, fun := range p.Funs {
			returnType := ""
			if len(fun.ReturnTypes) > 1 {
				returnType = fun.ReturnTypes[0]
			}
			funcSymbolTable[fun.Name] = FuncData{
				Index:      externFuncsCount + funcsCount,
				Name:       fun.Name,
				Params:     map[string]FuncParam{},
				ReturnType: returnType,
			}
			for pi, param := range fun.Parameters {
				funcSymbolTable[fun.Name].Params[param.Name] = FuncParam{
					Index: pi,
					Name:  param.Name,
					Type:  param.Type.Name,
				}
			}
			err := e.EmitTypes(funcSymbolTable[fun.Name])
			if err != nil {
				return nil, fmt.Errorf("Failed to emitTypes %v", err)
			}
			funcs = append(funcs, byte(externFuncsCount+funcsCount))           // function 0 signature index
			exports = append(exports, byte(len(fun.Name)))                     // name length
			exports = append(exports, []byte(fun.Name)...)                     // name
			exports = append(exports, 0x00, byte(externFuncsCount+funcsCount)) // export kind, export funcindex

			funcbody := bytes.NewBuffer([]byte{
				0x00, // local decl count
			})
			for i, s := range fun.Body {
				err := emitExpression(funcbody, fun.Name, funcSymbolTable, s.Exp.Operator, s.Exp.Left, s.Exp.Right)
				if err != nil {
					return nil, err
				}
				if i != len(fun.Body)-1 {
					funcbody.WriteByte(op.DROP)
				}
			}
			if funcSymbolTable[fun.Name].ReturnType == "" {
				funcbody.WriteByte(op.DROP)
			}
			funcBodys = append(funcBodys, byte(len(funcbody.Bytes())+1)) // funcbody size
			funcBodys = append(funcBodys, funcbody.Bytes()...)
			funcBodys = append(funcBodys, op.FUNC_END)
			// if fnIndex == 1 {
			// 	return nil, fmt.Errorf(fmt.Sprintf("%+v", funcbody.Bytes()))
			// }
			funcsCount += 1
		}
	}
	repr.Println(funcSymbolTable)
	buffer.Write([]byte{0x01, byte(e.TypesSection.Len() + 1), byte(externFuncsCount + funcsCount)}) // Type section code, section size, num types, type data
	buffer.Write(e.TypesSection.Bytes())
	buffer.Write([]byte{0x02, byte(e.ImportsSection.Len() + 1), byte(externFuncsCount)}) // Imports section, section size, num imports, type data
	buffer.Write(e.ImportsSection.Bytes())
	buffer.Write([]byte{0x03, byte(len(funcs) + 1), byte(funcsCount)}) // Func Sig section code, section size, num types, type data
	buffer.Write(funcs)
	buffer.Write([]byte{0x05, 0x03, 0x01, 0x00, 0x01})                   // Memory section code, section size, num memories, flags, initial (1 page 64KB)
	buffer.Write([]byte{0x07, byte(len(exports) + 1), byte(funcsCount)}) // exports section code, section size, num exports
	buffer.Write(exports)
	buffer.Write([]byte{0x0a, byte(len(funcBodys) + 1), byte(funcsCount)}) // funcbody section code, section size, num functions
	buffer.Write(funcBodys)
	return buffer, nil
}
<commit_msg>emit funcs sections<commit_after>package code_gen

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"yum/ast"
	"yum/code_gen/op"

	"github.com/alecthomas/repr"
)

func emitString(b *bytes.Buffer, address byte, v int32) {
	b.WriteByte(op.I32_CONST)
	b.WriteByte(address)
	b.WriteByte(op.I32_CONST)
	encodeSleb128(b, v)
	b.WriteByte(op.I32_STORE)
	b.WriteByte(byte(0x02))
	b.WriteByte(byte(0x00))
}

var typeMap = map[string]byte{
	"num": op.F64,
	"i32": op.I32,
}

type FuncData struct {
	Index      int
	Name       string
	Params     map[string]FuncParam
	ReturnType string
}

type FuncParam struct {
	Index int
	Name  string
	Type  string
}

func emitOperation(buffer *bytes.Buffer, funcName, operation string) error {
	switch operation {
	case "+":
		buffer.WriteByte(op.F64_ADD)
	case "-":
		buffer.WriteByte(op.F64_SUB)
	case "*":
		buffer.WriteByte(op.F64_MUL)
	case "/":
		buffer.WriteByte(op.F64_DIV)
	default:
		return fmt.Errorf("func '%s' operation '%s' is invalid", funcName, operation)
	}
	return nil
}

func emitExpression(buffer *bytes.Buffer, funcName string, funcSymbolTable map[string]FuncData, operation *string, l *ast.Literal, r *ast.Expression) error {
	if l != nil && l.Num != nil {
		buffer.WriteByte(op.F64_CONST)
		buffer.Write(float64ToByte(*l.Num))
	}
	if l != nil && l.Str != nil {
		// (i32.store (i32.const 0) (i32.const 8))  ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string
		// (i32.store (i32.const 4) (i32.const 12))  ;; iov.iov_len - The length of the 'hello world\n' string
		// (i32.store (i32.const 8) (i32.const 0x6c6c6568))
		// (i32.store (i32.const 12) (i32.const 0x6f77206f))
		// (i32.store (i32.const 16) (i32.const 0x0a646c72))
		// buffer.WriteByte(op.I32_CONST)
		// buffer.WriteByte(byte(0))
	}
	// TODO: remove r
	if r != nil && r.Left != nil && r.Left.Num != nil {
		buffer.WriteByte(op.F64_CONST)
		buffer.Write(float64ToByte(*r.Left.Num))
		emitOperation(buffer, funcName, *operation)
	}
	if l != nil && l.Reference != nil {
		if operation != nil {
			ref, ok := funcSymbolTable[funcName].Params[*l.Reference]
			if !ok {
				return fmt.Errorf("func '%s' parameter '%s' does not exist", funcName, *l.Reference)
			}
			buffer.WriteByte(op.GET_LOCAL)
			buffer.WriteByte(byte(ref.Index))
		} else {
			// funcCALL
			callFunc, ok := funcSymbolTable[*l.Reference]
			if !ok {
				return fmt.Errorf("func '%s' trying to call non-existing func '%s'", funcName, *l.Reference)
			}
			if len(callFunc.Params) != len(r.Left.Params) {
				return fmt.Errorf("func '%s' trying to call function '%s' with extra parameters", funcName, callFunc.Name)
			}
			for _, v := range r.Left.Params {
				if v.Str != nil {
					str := *v.Str
					data := []byte(str)
					if len(data)%4 == 1 {
						data = append(data, 0)
					}
					if len(data)%4 == 2 {
						data = append(data, 0)
					}
					if len(data)%4 == 3 {
						data = append(data, '\n')
					}

					// (i32.store (i32.const 0) (i32.const 8))  ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string
					// (i32.store (i32.const 4) (i32.const 12))  ;; iov.iov_len - The length of the 'hello world\n' string
					// (i32.store (i32.const 8) (i32.const 0x6c6c6568))
					// (i32.store (i32.const 12) (i32.const 0x6f77206f))
					// (i32.store (i32.const 16) (i32.const 0x0a646c72))
					emitString(buffer, 0, 8)
					emitString(buffer, 4, int32(len(string(data))))
					startAddress := 0
					startData := 0
					for i := range data {
						index := i + 1
						if index%4 == 0 {
							startAddress = index + 4
							startData = index - 4
							remainingData := data[startData:index]
							emitString(buffer, byte(startAddress), int32(binary.LittleEndian.Uint32(remainingData)))
						}
					}
				}
			}
			for _, v := range r.Left.Params {
				if v.Num != nil {
					buffer.WriteByte(0x41)
					// TODO: fix this
					// this has to be float64 or int32
					buffer.WriteByte(byte(int32(*v.Num)))
				}
				if v.Reference != nil {
					vref, ok := funcSymbolTable[funcName].Params[*v.Reference]
					if !ok {
						return fmt.Errorf("func'" + funcName + "' trying to call '" + callFunc.Name + "' with non-existing variable " + *v.Reference)
					}
					buffer.WriteByte(op.GET_LOCAL)
					buffer.WriteByte(byte(vref.Index))
				}
				if v.Str != nil {
					buffer.WriteByte(op.I32_CONST)
					buffer.WriteByte(byte(0))
				}
			}
			buffer.WriteByte(op.CALL)
			buffer.WriteByte(byte(callFunc.Index))
			if r.Operator != nil {
				err := emitExpression(buffer, funcName, funcSymbolTable, r.Operator, nil, r.Right)
				if err != nil {
					return err
				}
			}
		}
	}
	if r != nil && r.Left != nil && r.Left.Reference != nil {
		ref, ok := funcSymbolTable[funcName].Params[*r.Left.Reference]
		if !ok {
			return fmt.Errorf("func'" + funcName + "' parameter '" + *r.Left.Reference + "' does not exist")
		}
		buffer.WriteByte(op.GET_LOCAL)
		buffer.WriteByte(byte(ref.Index))
		emitOperation(buffer, funcName, *operation)
	}
	if r != nil && r.Operator != nil {
		err := emitExpression(buffer, funcName, funcSymbolTable, r.Operator, nil, r.Right)
		if err != nil {
			return err
		}
	}
	return nil
}

// _pointer = a: string -> i32
// (i32.store (i32.const 0) (i32.const 8)) // iov.iov_base - This is a pointer to the start of the 'hello world\n' string

// _length = a: string -> i32
// (i32.store (i32.const 4) (i32.const 12)) // iov.iov_len - The length of the 'hello world\n' string

type Emitter struct {
	TypesSection     *bytes.Buffer
	ImportsSection   *bytes.Buffer
	FuncsSection     *bytes.Buffer
	ExportsSection   *bytes.Buffer
	FuncsBodySection *bytes.Buffer
	Tree             *ast.Ast
	funcSymbolTable  map[string]FuncData
}

func NewEmitter(tree *ast.Ast) Emitter {
	return Emitter{
		TypesSection:     bytes.NewBuffer(nil),
		ImportsSection:   bytes.NewBuffer(nil),
		FuncsSection:     bytes.NewBuffer(nil),
		ExportsSection:   bytes.NewBuffer(nil),
		FuncsBodySection: bytes.NewBuffer(nil),
		funcSymbolTable:  map[string]FuncData{},
		Tree:             tree,
	}
}

func (e Emitter) EmitTypes(fun FuncData) error {
	e.TypesSection.WriteByte(0x60)                  // type func
	e.TypesSection.WriteByte(byte(len(fun.Params))) // num params
	for _, param := range fun.Params {              // i32, i32
		t, ok := typeMap[param.Type]
		if !ok {
			return fmt.Errorf("func'" + fun.Name + "' parameter '" + param.Name + "' type '" + param.Type + "' does not exist")
		}
		e.TypesSection.WriteByte(t)
	}
	if fun.ReturnType != "" {
		t, ok := typeMap[fun.ReturnType]
		if !ok {
			return fmt.Errorf("func'" + fun.Name + "' return type '" + fun.ReturnType + "' does not exist")
		}
		e.TypesSection.WriteByte(0x01) // num results
		e.TypesSection.WriteByte(t)    // i32
	} else {
		e.TypesSection.WriteByte(0x00) // num results
	}
	// panic(repr.String(buffer.Bytes()))
	return nil
}

func (e Emitter) EmitImports(moduleName, funcName string, typeIndex int) {
	e.ImportsSection.WriteByte(byte(len(moduleName))) // module name length
	e.ImportsSection.Write([]byte(moduleName))        // module name
	e.ImportsSection.WriteByte(byte(len(funcName)))   // fn name length
	e.ImportsSection.Write([]byte(funcName))          // fn name
	e.ImportsSection.WriteByte(0x00)                  // import kind
	e.ImportsSection.WriteByte(byte(typeIndex))       // import signature index
}

func (e Emitter) EmitFuncs(moduleName, funcName string, typeIndex int) {
}

func (e Emitter) EmitAll() (*bytes.Buffer, error) {
	buffer := bytes.NewBuffer(nil)
	buffer.Write([]byte{0x00, 0x61, 0x73, 0x6d}) // WASM_BINARY_MAGIC
	buffer.Write([]byte{0x01, 0x00, 0x00, 0x00}) // WASM_BINARY_VERSION

	exports := []byte{}
	funcBodys := []byte{}
	externFuncsCount := 0
	funcsCount := 0
	funcSymbolTable := map[string]FuncData{}
	for _, p := range e.Tree.Modules {
		for _, a := range p.ExternFuncs {
			funcSymbolTable[a.Name] = FuncData{
				Index:      externFuncsCount,
				Name:       a.Name,
				Params:     map[string]FuncParam{},
				ReturnType: a.ReturnType,
			}
			for pi, param := range a.Parameters {
				funcSymbolTable[a.Name].Params[param.Name] = FuncParam{
					Index: pi,
					Name:  param.Name,
					Type:  param.Type.Name,
				}
			}
			err := e.EmitTypes(funcSymbolTable[a.Name])
			if err != nil {
				return nil, fmt.Errorf("Failed to emitTypes %v", err)
			}
			e.EmitImports(p.Name, a.Name, externFuncsCount)
			externFuncsCount += 1
		}
		for _, fun := range p.Funs {
			returnType := ""
			if len(fun.ReturnTypes) > 1 {
				returnType = fun.ReturnTypes[0]
			}
			funcSymbolTable[fun.Name] = FuncData{
				Index:      externFuncsCount + funcsCount,
				Name:       fun.Name,
				Params:     map[string]FuncParam{},
				ReturnType: returnType,
			}
			for pi, param := range fun.Parameters {
				funcSymbolTable[fun.Name].Params[param.Name] = FuncParam{
					Index: pi,
					Name:  param.Name,
					Type:  param.Type.Name,
				}
			}
			err := e.EmitTypes(funcSymbolTable[fun.Name])
			if err != nil {
				return nil, fmt.Errorf("Failed to emitTypes %v", err)
			}
			e.FuncsSection.WriteByte(byte(externFuncsCount + funcsCount))      // function 0 signature index
			exports = append(exports, byte(len(fun.Name)))                     // name length
			exports = append(exports, []byte(fun.Name)...)                     // name
			exports = append(exports, 0x00, byte(externFuncsCount+funcsCount)) // export kind, export funcindex

			funcbody := bytes.NewBuffer([]byte{
				0x00, // local decl count
			})
			for i, s := range fun.Body {
				err := emitExpression(funcbody, fun.Name, funcSymbolTable, s.Exp.Operator, s.Exp.Left, s.Exp.Right)
				if err != nil {
					return nil, err
				}
				if i != len(fun.Body)-1 {
					funcbody.WriteByte(op.DROP)
				}
			}
			if funcSymbolTable[fun.Name].ReturnType == "" {
				funcbody.WriteByte(op.DROP)
			}
			funcBodys = append(funcBodys, byte(len(funcbody.Bytes())+1)) // funcbody size
			funcBodys = append(funcBodys, funcbody.Bytes()...)
			funcBodys = append(funcBodys, op.FUNC_END)
			// if fnIndex == 1 {
			// 	return nil, fmt.Errorf(fmt.Sprintf("%+v", funcbody.Bytes()))
			// }
			funcsCount += 1
		}
	}
	repr.Println(funcSymbolTable)
	buffer.Write([]byte{0x01, byte(e.TypesSection.Len() + 1), byte(externFuncsCount + funcsCount)}) // Type section code, section size, num types, type data
	buffer.Write(e.TypesSection.Bytes())
	buffer.Write([]byte{0x02, byte(e.ImportsSection.Len() + 1), byte(externFuncsCount)}) // Imports section, section size, num imports, type data
	buffer.Write(e.ImportsSection.Bytes())
	buffer.Write([]byte{0x03, byte(e.FuncsSection.Len() + 1), byte(funcsCount)}) // Func Sig section code, section size, num types, type data
	buffer.Write(e.FuncsSection.Bytes())
	buffer.Write([]byte{0x05, 0x03, 0x01, 0x00, 0x01})                   // Memory section code, section size, num memories, flags, initial (1 page 64KB)
	buffer.Write([]byte{0x07, byte(len(exports) + 1), byte(funcsCount)}) // exports section code, section size, num exports
	buffer.Write(exports)
	buffer.Write([]byte{0x0a, byte(len(funcBodys) + 1), byte(funcsCount)}) // funcbody section code, section size, num functions
	buffer.Write(funcBodys)
	return buffer, nil
}
<|endoftext|>