<commit_before>package rpc

import (
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/rpc"
	"time"

	coModel "github.com/Cepave/open-falcon-backend/common/model"

	sjson "github.com/bitly/go-simplejson"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Test rpc call: Agent.ReportStatus", ginkgoJsonRpc.NeedJsonRpc(func() {
	var (
		request = coModel.AgentReportRequest{
			Hostname:      "test-g-01",
			IP:            "123.45.61.81",
			AgentVersion:  "4.5.31",
			PluginVersion: "1.2.12",
		}
		response   = coModel.SimpleRpcResponse{}
		ts         *httptest.Server
		receiveCnt int64
	)

	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer r.Body.Close()
			j, err := sjson.NewFromReader(r.Body)
			Expect(err).To(BeNil())
			arr, err := j.Array()
			Expect(err).To(BeNil())

			rowsAffectedCnt := int64(len(arr))
			defer func() {
				GinkgoT().Log("Mock rowsAffectedCnt:", rowsAffectedCnt)
				receiveCnt = rowsAffectedCnt
			}()

			resp := `{}`
			w.Write([]byte(resp))
		}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
		GinkgoT().Log("Please set 'mysql_api.host' in hbs to addr of mock server")
	})

	AfterEach(func() {
		ts.Close()
	})

	It("should get correct value", func() {
		ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
			err := client.Call("Agent.ReportStatus", request, &response)

			Expect(err).To(BeNil())
		})

		Eventually(func() int64 {
			return receiveCnt
		}, time.Second*4, time.Second/2).Should(Equal(int64(1)))
	})
}))

var _ = Describe("Test rpc call [Agent.BuiltinMetrics]", ginkgoJsonRpc.NeedJsonRpc(func() {
	request := coModel.AgentHeartbeatRequest{
		Hostname: "cnc-he-060-008-151-208",
		Checksum: "d94ad826797905118ca30ba11a0273ad4303a8ca",
	}
	response := coModel.BuiltinMetricResponse{}
	var ts *httptest.Server
	tsResp := `
	{
	   "metrics":[
	      {
	         "metric":"net.if.in.bits",
	         "tags":"iface=eth0"
	      }
	   ],
	   "checksum":"e2c569be17396eca2a2e3c11578123ed",
	   "timestamp":1501491450
	}
  `

	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(
			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte(tsResp))
			}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
	})

	AfterEach(func() {
		ts.Close()
	})

	It("should get correct value", func() {
		ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
			err := client.Call("Agent.BuiltinMetrics", request, &response)
			respStr := fmt.Sprintf("%v", response)
			GinkgoT().Logf("RPC Response(%.5000s)", respStr)
			Expect(err).To(BeNil())
			Expect(response.Checksum).To(Equal("e2c569be17396eca2a2e3c11578123ed"))
			Expect(response.Timestamp).To(Equal(int64(1501491450)))
			Expect(response.Metrics).To(HaveLen(1))
		})
	})
}))

var _ = Describe("[Intg] Test rpc call: Agent.MinePlugins", ginkgoJsonRpc.NeedJsonRpc(func() {
	tsResp := []string{
		`
		{
		   "plugins":[

		   ],
		   "timestamp":1502355463,
		   "git_repo":""
		}
		`,
		`
		{
		   "plugins":[
		      "FastCache",
		      "basic/chk",
		      "basic/cpu",
		      "basic/dev",
		      "basic/file",
		      "basic/gd",
		      "basic/net",
		      "basic/sys"
		   ],
		   "timestamp":1502271611,
		   "git_repo":"http://gitlab.owl.fastweb.com.cn/Cepave/OwlPlugin.git"
		}
		`,
	}
	cnt := 0
	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(
			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte(tsResp[cnt]))
				cnt++
			}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
	})

	AfterEach(func() {
		ts.Close()
	})

	DescribeTable("when parameter is",
		func(request coModel.AgentHeartbeatRequest) {
			response := &coModel.AgentPluginsResponse{}
			ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
				err := client.Call("Agent.MinePlugins", request, &response)
				GinkgoT().Logf("RPC Response(%v)", response)
				Expect(err).To(BeNil())
			})
		},
		Entry("Zero value, should get nil plugins", coModel.AgentHeartbeatRequest{}),
		Entry("Not found but valid value, shold get empty plugins",
			coModel.AgentHeartbeatRequest{
				"test-agent-mineplugins",
				"Not-checked-value",
			},
		),
		Entry("Found and valid value, should get its plugins",
			coModel.AgentHeartbeatRequest{
				"cnc-he-060-008-151-208",
				"Not-checked-value",
			},
		),
	)
}))
<commit_msg>[OWL-1728] Fix undefined server variable.<commit_after>package rpc

import (
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/rpc"
	"time"

	coModel "github.com/Cepave/open-falcon-backend/common/model"

	sjson "github.com/bitly/go-simplejson"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Test rpc call: Agent.ReportStatus", ginkgoJsonRpc.NeedJsonRpc(func() {
	var (
		request = coModel.AgentReportRequest{
			Hostname:      "test-g-01",
			IP:            "123.45.61.81",
			AgentVersion:  "4.5.31",
			PluginVersion: "1.2.12",
		}
		response   = coModel.SimpleRpcResponse{}
		ts         *httptest.Server
		receiveCnt int64
	)

	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer r.Body.Close()
			j, err := sjson.NewFromReader(r.Body)
			Expect(err).To(BeNil())
			arr, err := j.Array()
			Expect(err).To(BeNil())

			rowsAffectedCnt := int64(len(arr))
			defer func() {
				GinkgoT().Log("Mock rowsAffectedCnt:", rowsAffectedCnt)
				receiveCnt = rowsAffectedCnt
			}()

			resp := `{}`
			w.Write([]byte(resp))
		}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
		GinkgoT().Log("Please set 'mysql_api.host' in hbs to addr of mock server")
	})

	AfterEach(func() {
		ts.Close()
	})

	It("should get correct value", func() {
		ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
			err := client.Call("Agent.ReportStatus", request, &response)

			Expect(err).To(BeNil())
		})

		Eventually(func() int64 {
			return receiveCnt
		}, time.Second*4, time.Second/2).Should(Equal(int64(1)))
	})
}))

var _ = Describe("Test rpc call [Agent.BuiltinMetrics]", ginkgoJsonRpc.NeedJsonRpc(func() {
	request := coModel.AgentHeartbeatRequest{
		Hostname: "cnc-he-060-008-151-208",
		Checksum: "d94ad826797905118ca30ba11a0273ad4303a8ca",
	}
	response := coModel.BuiltinMetricResponse{}
	var ts *httptest.Server
	tsResp := `
	{
	   "metrics":[
	      {
	         "metric":"net.if.in.bits",
	         "tags":"iface=eth0"
	      }
	   ],
	   "checksum":"e2c569be17396eca2a2e3c11578123ed",
	   "timestamp":1501491450
	}
  `

	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(
			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte(tsResp))
			}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
	})

	AfterEach(func() {
		ts.Close()
	})

	It("should get correct value", func() {
		ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
			err := client.Call("Agent.BuiltinMetrics", request, &response)
			respStr := fmt.Sprintf("%v", response)
			GinkgoT().Logf("RPC Response(%.5000s)", respStr)
			Expect(err).To(BeNil())
			Expect(response.Checksum).To(Equal("e2c569be17396eca2a2e3c11578123ed"))
			Expect(response.Timestamp).To(Equal(int64(1501491450)))
			Expect(response.Metrics).To(HaveLen(1))
		})
	})
}))

var _ = Describe("[Intg] Test rpc call: Agent.MinePlugins", ginkgoJsonRpc.NeedJsonRpc(func() {
	var ts *httptest.Server
	tsResp := []string{
		`
		{
		   "plugins":[],
		   "timestamp":1502355463,
		   "git_repo":""
		}
		`,
		`
		{
		   "plugins":[
		      "FastCache",
		      "basic/chk",
		      "basic/cpu",
		      "basic/dev",
		      "basic/file",
		      "basic/gd",
		      "basic/net",
		      "basic/sys"
		   ],
		   "timestamp":1502271611,
		   "git_repo":"http://gitlab.owl.fastweb.com.cn/Cepave/OwlPlugin.git"
		}
		`,
	}
	cnt := 0
	BeforeEach(func() {
		ts = httptest.NewUnstartedServer(
			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte(tsResp[cnt]))
				cnt++
			}))
		l, err := net.Listen("tcp", MOCK_URL)
		Expect(err).To(BeNil())
		ts.Listener = l
		ts.Start()

		GinkgoT().Logf("Mock server at: %s", ts.URL)
	})

	AfterEach(func() {
		ts.Close()
	})

	DescribeTable("when parameter is",
		func(request coModel.AgentHeartbeatRequest) {
			response := &coModel.AgentPluginsResponse{}
			ginkgoJsonRpc.OpenClient(func(client *rpc.Client) {
				err := client.Call("Agent.MinePlugins", request, &response)
				GinkgoT().Logf("RPC Response(%v)", response)
				Expect(err).To(BeNil())
			})
		},
		Entry("Zero value, should get nil plugins", coModel.AgentHeartbeatRequest{}),
		Entry("Not found but valid value, shold get empty plugins",
			coModel.AgentHeartbeatRequest{
				"test-agent-mineplugins",
				"Not-checked-value",
			},
		),
		Entry("Found and valid value, should get its plugins",
			coModel.AgentHeartbeatRequest{
				"cnc-he-060-008-151-208",
				"Not-checked-value",
			},
		),
	)
}))
<|endoftext|>


<commit_before>package mariadb_start_manager

import (
	"fmt"
	"regexp"

	"../os_helper"
)

type MariaDBStartManager struct {
	osHelper          os_helper.OsHelper
	logFileLocation   string
	stateFileLocation string
	mysqlServerPath   string
	username          string
	password          string
	jobIndex          int
	numberOfNodes     int
	loggingOn         bool
	dbSeedScriptPath  string
}

func New(osHelper os_helper.OsHelper,
	logFileLocation string,
	stateFileLocation string,
	mysqlServerPath string,
	username string,
	password string,
	dbSeedScriptPath string,
	jobIndex int,
	numberOfNodes int,
	loggingOn bool) *MariaDBStartManager {
	return &MariaDBStartManager{
		osHelper:          osHelper,
		logFileLocation:   logFileLocation,
		stateFileLocation: stateFileLocation,
		username:          username,
		password:          password,
		jobIndex:          jobIndex,
		mysqlServerPath:   mysqlServerPath,
		numberOfNodes:     numberOfNodes,
		loggingOn:         loggingOn,
		dbSeedScriptPath:  dbSeedScriptPath,
	}
}

func (m *MariaDBStartManager) log(info string) {
	if m.loggingOn {
		fmt.Printf(info)
	}
}

func (m *MariaDBStartManager) Execute() {
	//We should NEVER bootstrap unless we are Index 0
	if m.jobIndex == 0 {

		//single-node deploy
		if m.numberOfNodes == 1 {
			m.log("Single node deploy")
			m.bootstrapUpgradeAndWriteState("SINGLE_NODE")
			return
		}

		//MULTI-NODE DEPLOYMENTS BELOW

		//intial deploy, state file does not exists
		if !m.osHelper.FileExists(m.stateFileLocation) {
			m.log("state file does not exist, creating with contents: JOIN\n")
			m.bootstrapUpgradeAndWriteState("JOIN")
			return
		}

		//state file exists
		orig_contents, _ := m.osHelper.ReadFile(m.stateFileLocation)
		m.log(fmt.Sprintf("state file exists and contains: '%s'\n", orig_contents))

		//scaling up from a single node cluster
		if orig_contents == "SINGLE_NODE" {
			m.bootstrapUpgradeAndWriteState("JOIN")
			return
		}
	}

	m.joinCluster()
}

func (m *MariaDBStartManager) bootstrapUpgradeAndWriteState(state string) {
	m.bootstrapAndWriteState(state)
	m.seedDatabases()
	m.upgradeAndRestartIfNecessary("bootstrap")
}

func (m *MariaDBStartManager) bootstrapAndWriteState(state string) {
	m.osHelper.WriteStringToFile(m.stateFileLocation, state)

	m.log("starting in bootstrap \n")
	err := m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "bootstrap")
	if err != nil {
		panic(err)
	}
}

func (m *MariaDBStartManager) joinCluster() {
	m.log("starting in join\n")
	err := m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "start")
	if err != nil {
		panic(err)
	}

	m.seedDatabases()
	m.upgradeAndRestartIfNecessary("start")

	m.log("updating file with contents: JOIN\n")
	m.osHelper.WriteStringToFile(m.stateFileLocation, "JOIN")
}

func (m *MariaDBStartManager) seedDatabases() {
	output, err := m.osHelper.RunCommand("bash", m.dbSeedScriptPath)
	if err != nil {
		m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "stop")
		m.log("Seeding databases failed:\n")
		m.log(output)
		panic(err)
	}
}

func (m *MariaDBStartManager) upgradeAndRestartIfNecessary(mode string) {
	m.log("performing upgrade\n")
	output, err := m.osHelper.RunCommand(
		"bash",
		"mysql_upgrade.sh",
		m.username,
		m.password,
		m.logFileLocation)

	if m.requiresRestart(output, err) {
		m.log("stopping mysql\n")
		err = m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "stop")
		if err != nil {
			panic(err)
		}
		m.log("starting mysql\n")
		err = m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, mode)
		if err != nil {
			panic(err)
		}
	} else if m.numberOfNodes != 1 {
		m.log("updating file with contents: JOIN\n")
		m.osHelper.WriteStringToFile(m.stateFileLocation, "JOIN")
	}
}

func (m *MariaDBStartManager) requiresRestart(output string, err error) bool {
	// No error indicates that the upgrade script performed an upgrade.
	if err == nil {
		m.log("upgrade sucessful - restart required\n")
		return true
	}
	m.log(fmt.Sprintf("upgrade output: %s\n", output))

	//known error messages where a restart should not occur, do not remove from
	acceptableErrorsCompiled, _ := regexp.Compile("already upgraded|Unknown command|WSREP has not yet prepared node")
	if acceptableErrorsCompiled.MatchString(output) {
		m.log("output string matches acceptable errors - skip restart\n")
		return false
	} else {
		m.log("output string does not match acceptable errors - restart required\n")
		return true
	}
}
<commit_msg>Remove unnecessary code<commit_after>package mariadb_start_manager

import (
	"fmt"
	"regexp"

	"../os_helper"
)

type MariaDBStartManager struct {
	osHelper          os_helper.OsHelper
	logFileLocation   string
	stateFileLocation string
	mysqlServerPath   string
	username          string
	password          string
	jobIndex          int
	numberOfNodes     int
	loggingOn         bool
	dbSeedScriptPath  string
}

func New(osHelper os_helper.OsHelper,
	logFileLocation string,
	stateFileLocation string,
	mysqlServerPath string,
	username string,
	password string,
	dbSeedScriptPath string,
	jobIndex int,
	numberOfNodes int,
	loggingOn bool) *MariaDBStartManager {
	return &MariaDBStartManager{
		osHelper:          osHelper,
		logFileLocation:   logFileLocation,
		stateFileLocation: stateFileLocation,
		username:          username,
		password:          password,
		jobIndex:          jobIndex,
		mysqlServerPath:   mysqlServerPath,
		numberOfNodes:     numberOfNodes,
		loggingOn:         loggingOn,
		dbSeedScriptPath:  dbSeedScriptPath,
	}
}

func (m *MariaDBStartManager) log(info string) {
	if m.loggingOn {
		fmt.Printf(info)
	}
}

func (m *MariaDBStartManager) Execute() {
	//We should NEVER bootstrap unless we are Index 0
	if m.jobIndex == 0 {

		//single-node deploy
		if m.numberOfNodes == 1 {
			m.log("Single node deploy")
			m.bootstrapUpgradeAndWriteState("SINGLE_NODE")
			return
		}

		//MULTI-NODE DEPLOYMENTS BELOW

		//intial deploy, state file does not exists
		if !m.osHelper.FileExists(m.stateFileLocation) {
			m.log("state file does not exist, creating with contents: JOIN\n")
			m.bootstrapUpgradeAndWriteState("JOIN")
			return
		}

		//state file exists
		orig_contents, _ := m.osHelper.ReadFile(m.stateFileLocation)
		m.log(fmt.Sprintf("state file exists and contains: '%s'\n", orig_contents))

		//scaling up from a single node cluster
		if orig_contents == "SINGLE_NODE" {
			m.bootstrapUpgradeAndWriteState("JOIN")
			return
		}
	}

	m.joinCluster()
}

func (m *MariaDBStartManager) bootstrapUpgradeAndWriteState(state string) {
	m.bootstrapAndWriteState(state)
	m.seedDatabases()
	m.upgradeAndRestartIfNecessary("bootstrap")
}

func (m *MariaDBStartManager) bootstrapAndWriteState(state string) {
	m.osHelper.WriteStringToFile(m.stateFileLocation, state)

	m.log("starting in bootstrap \n")
	err := m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "bootstrap")
	if err != nil {
		panic(err)
	}
}

func (m *MariaDBStartManager) joinCluster() {
	m.log("starting in join\n")
	err := m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "start")
	if err != nil {
		panic(err)
	}

	m.seedDatabases()
	m.upgradeAndRestartIfNecessary("start")

	m.log("updating file with contents: JOIN\n")
	m.osHelper.WriteStringToFile(m.stateFileLocation, "JOIN")
}

func (m *MariaDBStartManager) seedDatabases() {
	output, err := m.osHelper.RunCommand("bash", m.dbSeedScriptPath)
	if err != nil {
		m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "stop")
		m.log("Seeding databases failed:\n")
		m.log(output)
		panic(err)
	}
}

func (m *MariaDBStartManager) upgradeAndRestartIfNecessary(mode string) {
	m.log("performing upgrade\n")
	output, err := m.osHelper.RunCommand(
		"bash",
		"mysql_upgrade.sh",
		m.username,
		m.password,
		m.logFileLocation)

	if m.requiresRestart(output, err) {
		m.log("stopping mysql\n")
		err = m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, "stop")
		if err != nil {
			panic(err)
		}
		m.log("starting mysql\n")
		err = m.osHelper.RunCommandWithTimeout(300, m.logFileLocation, "bash", m.mysqlServerPath, mode)
		if err != nil {
			panic(err)
		}
	}
}

func (m *MariaDBStartManager) requiresRestart(output string, err error) bool {
	// No error indicates that the upgrade script performed an upgrade.
	if err == nil {
		m.log("upgrade sucessful - restart required\n")
		return true
	}
	m.log(fmt.Sprintf("upgrade output: %s\n", output))

	//known error messages where a restart should not occur, do not remove from
	acceptableErrorsCompiled, _ := regexp.Compile("already upgraded|Unknown command|WSREP has not yet prepared node")
	if acceptableErrorsCompiled.MatchString(output) {
		m.log("output string matches acceptable errors - skip restart\n")
		return false
	} else {
		m.log("output string does not match acceptable errors - restart required\n")
		return true
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package block

import (
	"encoding/binary"

	"github.com/bitmark-inc/bitmarkd/blockrecord"
	"github.com/bitmark-inc/bitmarkd/storage"
)

func doBlockHeaderHash() error {
	return storage.Pool.Blocks.NewFetchCursor().Map(recoverBlockHeaderHash)
}

func recoverBlockHeaderHash(blockNumberBytes []byte, packedBlock []byte) error {
	globalData.Lock()
	defer globalData.Unlock()

	// TODO: decide if we need to disable reservoir when migrating the block db
	// reservoir.Disable()
	// defer reservoir.Enable()

	// reservoir.ClearSpend()

	var blockNumber uint64
	binary.BigEndian.PutUint64(blockNumberBytes, blockNumber)

	globalData.log.Debugf("rebuilt block header hash: %d", blockNumber)

	blockHeaderHashBytes := storage.Pool.BlockHeaderHash.Get(blockNumberBytes)
	if blockHeaderHashBytes == nil {
		digest, err := blockrecord.ComputeHeaderHash(packedBlock)
		if nil != err {
			return err
		}

		storage.Pool.BlockHeaderHash.Put(blockNumberBytes, digest[:])
	}

	return nil
}

func doRecovery() error {
	return storage.Pool.Blocks.NewFetchCursor().Map(recoverRecord)
}

func recoverRecord(key []byte, value []byte) error {
	return StoreIncoming(value, NoRescanVerified)
}
<commit_msg>[block] fix reading blocknumer key from db<commit_after>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package block

import (
	"encoding/binary"

	"github.com/bitmark-inc/bitmarkd/blockrecord"
	"github.com/bitmark-inc/bitmarkd/storage"
)

func doBlockHeaderHash() error {
	return storage.Pool.Blocks.NewFetchCursor().Map(recoverBlockHeaderHash)
}

func recoverBlockHeaderHash(blockNumberBytes []byte, packedBlock []byte) error {
	globalData.Lock()
	defer globalData.Unlock()

	// TODO: decide if we need to disable reservoir when migrating the block db
	// reservoir.Disable()
	// defer reservoir.Enable()

	// reservoir.ClearSpend()

	blockNumber := binary.BigEndian.Uint64(blockNumberBytes)

	blockHeaderHashBytes := storage.Pool.BlockHeaderHash.Get(blockNumberBytes)
	if blockHeaderHashBytes == nil {
		digest, err := blockrecord.ComputeHeaderHash(packedBlock)
		if nil != err {
			return err
		}

		storage.Pool.BlockHeaderHash.Put(blockNumberBytes, digest[:])
	}

	globalData.log.Debugf("rebuilt block: %d", blockNumber)

	return nil
}

func doRecovery() error {
	return storage.Pool.Blocks.NewFetchCursor().Map(recoverRecord)
}

func recoverRecord(key []byte, value []byte) error {
	return StoreIncoming(value, NoRescanVerified)
}
<|endoftext|>


<commit_before>package kit

import (
	"time"
)

// Foreman ... TODO
type Foreman struct {
	leakyBucket *LeakyBucket
	halt        chan bool
	JobQueue    chan AssetEvent
	WorkerQueue chan AssetEvent
	OnIdle      func()
}

// NewForeman ... TODO
func NewForeman(leakyBucket *LeakyBucket) Foreman {
	return Foreman{
		leakyBucket: leakyBucket,
		halt:        make(chan bool),
		JobQueue:    make(chan AssetEvent),
		WorkerQueue: make(chan AssetEvent),
		OnIdle:      func() {},
	}
}

// IssueWork ... TODO
func (f Foreman) IssueWork() {
	f.leakyBucket.StartDripping()
	go func() {
		notifyProcessed := false
		for {
			select {
			case job := <-f.JobQueue:
				f.leakyBucket.GetDrop()
				notifyProcessed = true
				// TODO: this was a potential aliasing issue!
				go func(jobToAdd AssetEvent) {
					f.WorkerQueue <- jobToAdd
				}(job)
			case <-f.halt:
				return
			case <-time.Tick(1 * time.Second):
				if notifyProcessed {
					notifyProcessed = false
					f.OnIdle()
				}
			}
		}
	}()
}

// Halt ... TODO
func (f Foreman) Halt() {
	f.leakyBucket.StopDripping()
	go func() {
		f.halt <- true
	}()
}
<commit_msg>added docs to foreman and simplified the worker queue.<commit_after>package kit

import (
	"time"
)

// Foreman is a job queueing processor using a LeakyBucket throttler.
type Foreman struct {
	leakyBucket *LeakyBucket
	halt        chan bool
	JobQueue    chan AssetEvent
	WorkerQueue chan AssetEvent
	OnIdle      func()
}

// NewForeman will return a new Foreman using the bucket for throttling.
func NewForeman(leakyBucket *LeakyBucket) Foreman {
	return Foreman{
		leakyBucket: leakyBucket,
		halt:        make(chan bool),
		JobQueue:    make(chan AssetEvent),
		WorkerQueue: make(chan AssetEvent),
		OnIdle:      func() {},
	}
}

// IssueWork start the Foreman processing jobs that are in it's queue. It will call
// OnIdle every second when there is no jobs to process. If there are jobs in the queue
// then it will make sure there is a worker to process it from the bucket.
func (f Foreman) IssueWork() {
	f.leakyBucket.StartDripping()
	go func() {
		notifyProcessed := false
		for {
			select {
			case job := <-f.JobQueue:
				f.leakyBucket.GetDrop()
				notifyProcessed = true
				f.WorkerQueue <- job
			case <-f.halt:
				return
			case <-time.Tick(1 * time.Second):
				if notifyProcessed {
					notifyProcessed = false
					f.OnIdle()
				}
			}
		}
	}()
}

// Halt stops the Foreman from processing jobs in its queue.
func (f Foreman) Halt() {
	f.leakyBucket.StopDripping()
	go func() {
		f.halt <- true
	}()
}
<|endoftext|>


<commit_before>package config_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/fly/config"
)

// In this example our organization is named "podhub", and our project
// namespace is "canary".
//
// In this example we have a file located at /Users/jchen/.config/podhub/canary/config.yaml,
// with the following contents:
//  example:
//    - "a"
//    - "b"
//    - "c"
func ExampleNamespace() {
	type Config struct {
		Example []string `yaml:"example"`
	}

	var err error
	var cfg Config
	var path string
	var cfgNS = config.Namespace{
		Organization: "podhub",
		System:       "canary",
	}

	path = cfgNS.Path()
	fmt.Println("Path to config " + path)

	err = cfgNS.Load(&cfg)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Println("Contents of cfg " + fmt.Sprint(cfg))
}

func TestExpandUser(t *testing.T) {
	var homeDir string
	var path string

	if os.Getenv("TRAVIS") == "true" {
		homeDir = "/home/travis"
	} else {
		homeDir = os.Getenv("HOME")
	}
	var correctPath = homeDir + "/.config/fly/config/config.yaml"

	path = config.ExpandUser("~/.config/fly/config/config.yaml")

	// docs say not to trust /home/travis to be homedir. We'll need to
	// revisit this later.
	if path != correctPath {
		t.Error("Expected ", correctPath, ", got ", path)
	}

	path = config.ExpandUser("$HOME/.config/fly/config/config.yaml")

	// docs say not to trust /home/travis to be homedir. We'll need to
	// revisit this later.
	if path != correctPath {
		t.Error("Expected ", correctPath, ", got ", path)
	}
}

func TestLoad(t *testing.T) {
	const correctDir = "/etc/fly/config/"
	type configExample struct {
		Location string `yaml:"location"`
		Burritos bool   `yaml:"burritos"`
	}

	var correctCfgText = `location: Señor Sisig
burritos: true`
	var correctCfg = configExample{
		Location: "Señor Sisig",
		Burritos: true,
	}
	var err error
	var cfg configExample
	var homeDir string
	var dirMode os.FileMode = 0755
	var fileMode os.FileMode = 0644

	if os.Getenv("TRAVIS") == "true" {
		homeDir = "/home/travis"
	} else {
		homeDir = os.Getenv("HOME")
	}
	const correctPath = correctDir + "config.yaml"

	// Setup
	os.RemoveAll(homeDir + "/.config/fly/config/config.yaml")
	os.MkdirAll(correctDir, dirMode)
	err = ioutil.WriteFile(correctPath, []byte(correctCfgText), fileMode)
	if err != nil {
		t.Error("Got an error writing ", correctPath, ", got an error: ", err)
	}

	// Test
	err = config.Load(correctPath, &cfg)
	if err != nil {
		t.Error("Got an error: ", err, ", expecting nil")
	}

	if cfg != correctCfg {
		t.Error("Expecting ", correctCfg, ", got ", cfg)
	}

	err = config.Load(correctPath, cfg)
	if err.Error() != "config: not a pointer" {
		t.Error("Expecting error: config: not a pointer, instead got", err)
	}

	err = config.Load("rumpelstilzjchen", &cfg)
	if err == nil {
		t.Error("Expected read error, got nil")
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}

func TestUserBase(t *testing.T) {
	const correctUserBase = "~/.config/"
	var userBase string
	userBase = config.UserBase

	if userBase != correctUserBase {
		t.Error("Expecting ", correctUserBase, ", got ", userBase)
	}
}

func TestSystemBase(t *testing.T) {
	const correctSystemBase = "/etc/"
	var systemBase string
	systemBase = config.SystemBase

	if systemBase != correctSystemBase {
		t.Error("Expecting ", correctSystemBase, ", got ", systemBase)
	}
}

func TestNamespacePath(t *testing.T) {
	var correctDir = "/etc/fly/config/"
	var cfgNS = config.Namespace{
		Organization: "fly",
		System:       "config",
	}
	var err error
	var path string
	var homeDir string
	var dirMode os.FileMode = 0755
	if os.Getenv("TRAVIS") == "true" {
		homeDir = "/home/travis"
	} else {
		homeDir = os.Getenv("HOME")
	}
	var correctPath = correctDir + "config.yaml"

	// Setup
	os.RemoveAll(homeDir + "/.config/fly/config/config.yaml")
	os.MkdirAll(correctDir, dirMode)
	_, err = os.Create(correctPath)
	if err != nil {
		t.Error("Unable to create file ", correctPath, ", got an error: ", err)
	}

	// Test
	path = cfgNS.Path()
	if path != correctPath {
		t.Error("Expecting ", correctPath, ", got ", path)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}

	// test homedir
	// Setup
	correctDir = "/home/travis/.config/fly/config/"
	correctPath = correctDir + "config.yaml"
	os.RemoveAll("/etc/fly/config/")
	os.MkdirAll(correctDir, dirMode)
	_, err = os.Create(correctPath)
	if err != nil {
		t.Error("Unable to create file ", correctPath, ", got an error: ", err)
	}

	// Test
	path = cfgNS.Path()
	if path != correctPath {
		t.Error("Expecting ", correctPath, ", got ", path)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}

func TestNamespaceEnvVar(t *testing.T) {
	var cfgNS = config.Namespace{
		Organization: "fly",
		System:       "config",
	}
	const correctEnvVar string = "FLY_CONFIG_CONFIG_URI"

	envVarURI := cfgNS.EnvVar()
	if envVarURI != correctEnvVar {
		t.Error("Expecting ", correctEnvVar, ", got ", envVarURI)
	}
}

func TestNamespaceLoad(t *testing.T) {
	const correctDir = "/etc/fly/config/"
	type configExample struct {
		Location string
		Burritos bool
	}

	var correctCfgText = `location: Señor Sisig
burritos: true`
	var correctCfg = configExample{
		Location: "Señor Sisig",
		Burritos: true,
	}
	var cfgNS = config.Namespace{
		Organization: "fly",
		System:       "config",
	}
	var err error
	var cfg configExample
	var homeDir string
	var dirMode os.FileMode = 0755
	var fileMode os.FileMode = 0644

	if os.Getenv("TRAVIS") == "true" {
		homeDir = "/home/travis"
	} else {
		homeDir = os.Getenv("HOME")
	}
	const correctPath = correctDir + "config.yaml"

	// Setup
	os.RemoveAll(homeDir + "/.config/fly/config/config.yaml")
	os.MkdirAll(correctDir, dirMode)
	err = ioutil.WriteFile(correctPath, []byte(correctCfgText), fileMode)
	if err != nil {
		t.Error("Got an error writing ", correctPath, ", got an error: ", err)
	}

	// Test
	err = cfgNS.Load(&cfg)
	if err != nil {
		t.Error("Got an error: ", err, ", expecting nil")
	}

	if cfg != correctCfg {
		t.Error("Expecting ", correctCfg, ", got ", cfg)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}
<commit_msg>remove duplicated consts/vars<commit_after>package config_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/fly/config"
)

type testConfig struct {
	Example []string `yaml:"example"`
}

type burritoConfig struct {
	Location string `yaml:"location"`
	Burritos bool   `yaml:"burritos"`
}

// TODO: come up with a way to test homedir without hardcoding travis
const (
	correctEnvVar string = "TESTORG_TESTSYSTEM_CONFIG_URI"
	systemBaseDir string = "/etc/"
	systemDir     string = "/etc/testorg/testsystem/"
	systemPath    string = "/etc/testorg/testsystem/config.yaml"
	userBaseDir   string = "/home/travis/.config/"
	userDir       string = "/home/travis/"
	userPath      string = "/home/travis/.config/testorg/testsystem/config.yaml"
	organization  string = "testorg"
	system        string = "testsystem"
)

const (
	dirMode  os.FileMode = 0755
	fileMode os.FileMode = 0644
)

var cfgNS = config.Namespace{
	Organization: organization,
	System:       system,
}

// In this example our organization is named "podhub", and our project
// namespace is "canary".
//
// In this example we have a file located at /Users/jchen/.config/podhub/canary/config.yaml,
// with the following contents:
//  example:
//    - "a"
//    - "b"
//    - "c"
func ExampleNamespace() {
	var err error
	var cfg testConfig
	var path string

	path = cfgNS.Path()
	fmt.Println("Path to config " + path)

	err = cfgNS.Load(&cfg)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Println("Contents of cfg " + fmt.Sprint(cfg))
}

func TestExpandUser(t *testing.T) {
	correctPath := userPath

	path = config.ExpandUser("~/.config/fly/config/config.yaml")

	if path != correctPath {
		t.Error("Expected ", correctPath, ", got ", path)
	}

	path = config.ExpandUser("$HOME/.config/fly/config/config.yaml")

	if path != correctPath {
		t.Error("Expected ", correctPath, ", got ", path)
	}
}

func TestLoad(t *testing.T) {
	correctDir := systemDir
	var correctCfgText = `location: Señor Sisig
burritos: true`
	var correctCfg = burritoConfig{
		Location: "Señor Sisig",
		Burritos: true,
	}
	var err error
	var cfg burritoConfig

	// Setup
	os.RemoveAll(userPath)
	os.MkdirAll(correctDir, dirMode)
	err = ioutil.WriteFile(correctPath, []byte(correctCfgText), fileMode)
	if err != nil {
		t.Error("Got an error writing ", correctPath, ", got an error: ", err)
	}

	// Test
	err = config.Load(correctPath, &cfg)
	if err != nil {
		t.Error("Got an error: ", err, ", expecting nil")
	}

	if cfg != correctCfg {
		t.Error("Expecting ", correctCfg, ", got ", cfg)
	}

	err = config.Load(correctPath, cfg)
	if err.Error() != "config: not a pointer" {
		t.Error("Expecting error: config: not a pointer, instead got", err)
	}

	err = config.Load("rumpelstilzjchen", &cfg)
	if err == nil {
		t.Error("Expected read error, got nil")
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}

func TestUserBase(t *testing.T) {
	if config.UserBase != userBaseDir {
		t.Error("Expecting ", userBaseDir, ", got ", config.UserBase)
	}
}

func TestSystemBase(t *testing.T) {
	if config.SystemBase != systemBaseDir {
		t.Error("Expecting ", systemBaseDir, ", got ", config.SystemBase)
	}
}

func TestNamespacePath(t *testing.T) {
	var err error
	var path string

	correctDir := systemDir
	correctPath := systemPath

	// Setup
	os.RemoveAll(userPath)
	os.MkdirAll(correctDir, dirMode)
	_, err = os.Create(correctPath)
	if err != nil {
		t.Error("Unable to create file ", correctPath, ", got an error: ", err)
	}

	// Test
	path = cfgNS.Path()
	if path != correctPath {
		t.Error("Expecting ", correctPath, ", got ", path)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}

	// test homedir
	// Setup
	correctDir := userDir
	correctPath := userPath
	os.RemoveAll(systemDir)
	os.MkdirAll(correctDir, dirMode)
	_, err = os.Create(correctPath)
	if err != nil {
		t.Error("Unable to create file ", correctPath, ", got an error: ", err)
	}

	// Test
	path = cfgNS.Path()
	if path != correctPath {
		t.Error("Expecting ", correctPath, ", got ", path)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}

func TestNamespaceEnvVar(t *testing.T) {
	if cfgNS.EnvVar() != envVar {
		t.Error("Expecting ", envVar, ", got ", cfgNS.EnvVar())
	}
}

func TestNamespaceLoad(t *testing.T) {
	correctDir := systemDir
	correctPath := systemPath

	correctCfgText := `location: Señor Sisig
burritos: true`
	correctCfg := configExample{
		Location: "Señor Sisig",
		Burritos: true,
	}
	var err error
	var cfg configExample

	// Setup
	os.RemoveAll(userPath)
	os.MkdirAll(correctDir, dirMode)
	err = ioutil.WriteFile(correctPath, []byte(correctCfgText), fileMode)
	if err != nil {
		t.Error("Got an error writing ", correctPath, ", got an error: ", err)
	}

	// Test
	err = cfgNS.Load(&cfg)
	if err != nil {
		t.Error("Got an error: ", err, ", expecting nil")
	}

	if cfg != correctCfg {
		t.Error("Expecting ", correctCfg, ", got ", cfg)
	}

	// Teardown
	err = os.RemoveAll(correctPath)
	if err != nil {
		t.Error("Unable to remove file ", correctPath,
			" in teardown, got an error: ", err)
	}
}
<|endoftext|>


<commit_before>package gophernaut

import "testing"

func TestConfig(t *testing.T) {

}
<commit_msg>adding some tests<commit_after>package gophernaut

import "testing"

func TestConfig(t *testing.T) {
	t.Log(Config{
		Host: "foo.bar",
		Port: 1234,
	})
}

func TestRead(t *testing.T) {
	c := ReadConfig()
	t.Log(c)
}
<|endoftext|>


<commit_before>package kitsu

import (
	"fmt"
)

// The possible age rating values for media types like Anime, Manga and Drama.
const (
	AgeRatingG   = "G"   // General Audiences
	AgeRatingPG  = "PG"  // Parental Guidance Suggested
	AgeRatingR   = "R"   // Restricted
	AgeRatingR18 = "R18" // Explicit
)
const (
	AnimeStatusCurrent    = "current"
	AnimeStatusFinished   = "finished"
	AnimeStatusTBA        = "tba"
	AnimeStatusUnreleased = "unreleased"
	AnimeStatusUpcoming   = "upcoming"
)

// The possible anime subtypes. They are convenient for making comparisons
// with Anime.Subtype.
const (
	AnimeSubtypeONA     = "ONA"
	AnimeSubtypeOVA     = "OVA"
	AnimeSubtypeTV      = "TV"
	AnimeSubtypeMovie   = "movie"
	AnimeSubtypeMusic   = "music"
	AnimeSubtypeSpecial = "special"
)

// AnimeService handles communication with the anime related methods of the
// Kitsu API.
//
// Kitsu API docs:
// http://docs.kitsu17.apiary.io/#reference/media/anime/show-anime
type AnimeService service

// Anime represents a Kitsu anime.
//
// Additional filters: text, season, streamers
type Anime struct {
	ID string `jsonapi:"primary,anime"`

	// ISO 8601 date and time, e.g. 2017-07-27T22:21:26.824Z
	CreatedAt string `jsonapi:"attr,createdAt,omitempty"`

	// ISO 8601 of last modification, e.g. 2017-07-27T22:47:45.129Z
	UpdatedAt string `jsonapi:"attr,updatedAt,omitempty"`

	// Unique slug used for page URLs, e.g. cowboy-bebop
	Slug string `jsonapi:"attr,slug,omitempty"`

	// Synopsis of the anime, e.g.
	//
	// In the year 2071, humanity has colonoized several of the planets and
	// moons...
	Synopsis string `jsonapi:"attr,synopsis,omitempty"`

	// e.g. 400
	CoverImageTopOffset int `jsonapi:"attr,coverImageTopOffset,omitempty"`

	// Titles in different languages. Other languages will be listed if they
	// exist, e.g.
	//
	// "en": "Attack on Titan"
	//
	// "en_jp": "Shingeki no Kyojin"
	//
	// "ja_jp": "進撃の巨人"
	Titles map[string]interface{} `jsonapi:"attr,titles,omitempty"`

	// Canonical title for the anime, e.g. Attack on Titan
	CanonicalTitle string `jsonapi:"attr,canonical_title,omitempty"`

	// Shortened nicknames for the anime, e.g. COWBOY BEBOP
	AbbreviatedTitles []string `jsonapi:"attr,abbreviatedTitles,omitempty"`

	// The average of all user ratings for the anime, e.g. 88.65
	AverageRating string `jsonapi:"attr,averageRating,omitempty"`

	// How many times each rating has been given to the anime, e.g.
	//
	// "2": "72"
	//
	// "3": "0"
	//
	// ...
	//
	// "19": "40"
	//
	// "20": "13607"
	RatingFrequencies map[string]interface{} `jsonapi:"attr,ratingFrequencies,omitempty"`

	// e.g. 40405
	UserCount int `jsonapi:"attr,userCount,omitempty"`

	// e.g. 3277
	FavoritesCount int `jsonapi:"attr,favoritesCount,omitempty"`

	// Date the anime started airing/was released, e.g. 2013-04-07
	StartDate string `jsonapi:"attr,startDate,omitempty"`

	// Date the anime finished airing, e.g. 2013-09-28
	EndDate string `jsonapi:"attr,endDate,omitempty"`

	// e.g. 10
	PopularityRank int `jsonapi:"attr,popularityRank,omitempty"`

	// e.g. 10
	RatingRank int `jsonapi:"attr,ratingRank,omitempty"`

	// Possible values described by the AgeRating constants.
	AgeRating string `jsonapi:"attr,ageRating,omitempty"`

	// Description of the age rating, e.g. 17+ (violence & profanity)
	AgeRatingGuide string `jsonapi:"attr,ageRatingGuide,omitempty"`

	// Show format of the anime. Possible values described by the AnimeSubtype
	// constants.
	Subtype string `jsonapi:"attr,subtype,omitempty"`

	// Possible values described by the AnimeStatus constants.
	Status string `jsonapi:"attr,status,omitempty"`

	// The URL template for the poster, e.g.
	//
	// "tiny": "https://media.kitsu.io/anime/poster_images/1/tiny.jpg?1431697256"
	//
	// "small": "https://media.kitsu.io/anime/poster_images/1/small.jpg?1431697256"
	//
	// "medium": "https://media.kitsu.io/anime/poster_images/1/medium.jpg?1431697256"
	//
	// "large": "https://media.kitsu.io/anime/poster_images/1/large.jpg?1431697256"
	//
	// "original: "https://media.kitsu.io/anime/poster_images/1/original.jpg?1431697256"
	PosterImage map[string]interface{} `jsonapi:"attr,posterImage,omitempty"`

	// The URL template for the cover, e.g.
	//
	// "tiny": "https://media.kitsu.io/anime/cover_images/1/tiny.jpg?1416336000"
	//
	// "small": "https://media.kitsu.io/anime/cover_images/1/small.jpg?1416336000"
	//
	// "large": "https://media.kitsu.io/anime/cover_images/1/large.jpg?1416336000"
	//
	// "original": "https://media.kitsu.io/anime/cover_images/1/original.jpg?1416336000"
	CoverImage map[string]interface{} `jsonapi:"attr,coverImage,omitempty"`

	// How many episodes the anime has, e.g. 25
	EpisodeCount int `jsonapi:"attr,episodeCount,omitempty"`

	// How many minutes long each episode is, e.g. 24
	EpisodeLength int `jsonapi:"attr,episodeLength,omitempty"`

	// YouTube video id for Promotional Video, e.g. n4Nj6Y_SNYI
	YoutubeVideoID string `jsonapi:"attr,youtubeVideoId,omitempty"`

	// Relationships.

	Genres   []*Genre   `jsonapi:"relation,genres,omitempty"`
	Castings []*Casting `jsonapi:"relation,castings,omitempty"`
}

// Genre represents a Kitsu media genre. Genre is a relationship of Kitsu media
// types like Anime, Manga and Drama.
type Genre struct {
	ID          string `jsonapi:"primary,genres"`
	Name        string `jsonapi:"attr,name"`
	Slug        string `jsonapi:"attr,slug"`
	Description string `jsonapi:"attr,description"`
}

// Casting represents a Kitsu media casting. Casting is a relationship of Kitsu
// media types like Anime, Manga and Drama.
type Casting struct {
	ID         string     `jsonapi:"primary,castings"`
	Role       string     `jsonapi:"attr,role"`
	VoiceActor bool       `jsonapi:"attr,voiceActor"`
	Featured   bool       `jsonapi:"attr,featured"`
	Language   string     `jsonapi:"attr,language"`
	Character  *Character `jsonapi:"relation,character"`
	Person     *Person    `jsonapi:"relation,person"`
}

// BUG(google/jsonapi): Unmarshaling of fields which are of type struct or
// map[string]string is not supported by google/jsonapi. A workaround for
// fields such as Character.Image and User.Avatar is to use
// map[string]interface{} instead.
//
// See: https://github.com/google/jsonapi/issues/74
//
// Another limitation is being unable to unmarshal to custom types such as
// "enum" types like AnimeType, MangaType and LibraryEntryStatus. These are
// useful for doing comparisons and working with fields such as Anime.ShowType,
// Manga.ShowType and LibraryEntry.Status.
//
// Because of this limitation the string type is used for those fields instead.
// As such, instead of using those custom types, we keep the possible values as
// untyped string constants to avoid unnecessary conversions when working with
// those fields.

// Character represents a Kitsu character like the fictional characters that
// appear in anime, manga and drama. Character is a relationship of Casting.
type Character struct {
	ID          string                 `jsonapi:"primary,characters"`
	Slug        string                 `jsonapi:"attr,slug"`
	Name        string                 `jsonapi:"attr,name"`
	MALID       int                    `jsonapi:"attr,malId"`
	Description string                 `jsonapi:"attr,description"`
	Image       map[string]interface{} `jsonapi:"attr,image"`
}

// Person represents a person that is involved with a certain media. It can be
// voice actors, animators, etc. Person is a relationship of Casting.
type Person struct {
	ID    string `jsonapi:"primary,people"`
	Name  string `jsonapi:"attr,name"`
	MALID int    `jsonapi:"attr,malId"`
	Image string `jsonapi:"attr,image"`
}

// Show returns details for a specific Anime by providing a unique identifier
// of the anime e.g. 7442.
func (s *AnimeService) Show(animeID string, opts ...URLOption) (*Anime, *Response, error) {
	u := fmt.Sprintf(defaultAPIVersion+"anime/%s", animeID)

	req, err := s.client.NewRequest("GET", u, nil, opts...)
	if err != nil {
		return nil, nil, err
	}

	a := new(Anime)
	resp, err := s.client.Do(req, a)
	if err != nil {
		return nil, resp, err
	}

	return a, resp, nil
}

// List returns a list of Anime. Optional parameters can be specified to filter
// the search results and control pagination, sorting etc.
func (s *AnimeService) List(opts ...URLOption) ([]*Anime, *Response, error) {
	u := defaultAPIVersion + "anime"

	req, err := s.client.NewRequest("GET", u, nil, opts...)
	if err != nil {
		return nil, nil, err
	}

	var anime []*Anime
	resp, err := s.client.Do(req, &anime)
	if err != nil {
		return nil, resp, err
	}

	return anime, resp, nil
}
<commit_msg>Add doc comment for Anime status constants.<commit_after>package kitsu

import (
	"fmt"
)

// The possible age rating values for media types like Anime, Manga and Drama.
const (
	AgeRatingG   = "G"   // General Audiences
	AgeRatingPG  = "PG"  // Parental Guidance Suggested
	AgeRatingR   = "R"   // Restricted
	AgeRatingR18 = "R18" // Explicit
)

// Possible values for Anime.Status.
const (
	AnimeStatusCurrent    = "current"
	AnimeStatusFinished   = "finished"
	AnimeStatusTBA        = "tba"
	AnimeStatusUnreleased = "unreleased"
	AnimeStatusUpcoming   = "upcoming"
)

// The possible anime subtypes. They are convenient for making comparisons
// with Anime.Subtype.
const (
	AnimeSubtypeONA     = "ONA"
	AnimeSubtypeOVA     = "OVA"
	AnimeSubtypeTV      = "TV"
	AnimeSubtypeMovie   = "movie"
	AnimeSubtypeMusic   = "music"
	AnimeSubtypeSpecial = "special"
)

// AnimeService handles communication with the anime related methods of the
// Kitsu API.
//
// Kitsu API docs:
// http://docs.kitsu17.apiary.io/#reference/media/anime/show-anime
type AnimeService service

// Anime represents a Kitsu anime.
//
// Additional filters: text, season, streamers
type Anime struct {
	ID string `jsonapi:"primary,anime"`

	// --- Attributes ---

	// ISO 8601 date and time, e.g. 2017-07-27T22:21:26.824Z
	CreatedAt string `jsonapi:"attr,createdAt,omitempty"`

	// ISO 8601 of last modification, e.g. 2017-07-27T22:47:45.129Z
	UpdatedAt string `jsonapi:"attr,updatedAt,omitempty"`

	// Unique slug used for page URLs, e.g. cowboy-bebop
	Slug string `jsonapi:"attr,slug,omitempty"`

	// Synopsis of the anime, e.g.
	//
	// In the year 2071, humanity has colonoized several of the planets and
	// moons...
	Synopsis string `jsonapi:"attr,synopsis,omitempty"`

	// e.g. 400
	CoverImageTopOffset int `jsonapi:"attr,coverImageTopOffset,omitempty"`

	// Titles in different languages. Other languages will be listed if they
	// exist, e.g.
	//
	// "en": "Attack on Titan"
	//
	// "en_jp": "Shingeki no Kyojin"
	//
	// "ja_jp": "進撃の巨人"
	Titles map[string]interface{} `jsonapi:"attr,titles,omitempty"`

	// Canonical title for the anime, e.g. Attack on Titan
	CanonicalTitle string `jsonapi:"attr,canonical_title,omitempty"`

	// Shortened nicknames for the anime, e.g. COWBOY BEBOP
	AbbreviatedTitles []string `jsonapi:"attr,abbreviatedTitles,omitempty"`

	// The average of all user ratings for the anime, e.g. 88.65
	AverageRating string `jsonapi:"attr,averageRating,omitempty"`

	// How many times each rating has been given to the anime, e.g.
	//
	// "2": "72"
	//
	// "3": "0"
	//
	// ...
	//
	// "19": "40"
	//
	// "20": "13607"
	RatingFrequencies map[string]interface{} `jsonapi:"attr,ratingFrequencies,omitempty"`

	// e.g. 40405
	UserCount int `jsonapi:"attr,userCount,omitempty"`

	// e.g. 3277
	FavoritesCount int `jsonapi:"attr,favoritesCount,omitempty"`

	// Date the anime started airing/was released, e.g. 2013-04-07
	StartDate string `jsonapi:"attr,startDate,omitempty"`

	// Date the anime finished airing, e.g. 2013-09-28
	EndDate string `jsonapi:"attr,endDate,omitempty"`

	// e.g. 10
	PopularityRank int `jsonapi:"attr,popularityRank,omitempty"`

	// e.g. 10
	RatingRank int `jsonapi:"attr,ratingRank,omitempty"`

	// Possible values described by the AgeRating constants.
	AgeRating string `jsonapi:"attr,ageRating,omitempty"`

	// Description of the age rating, e.g. 17+ (violence & profanity)
	AgeRatingGuide string `jsonapi:"attr,ageRatingGuide,omitempty"`

	// Show format of the anime. Possible values described by the AnimeSubtype
	// constants.
	Subtype string `jsonapi:"attr,subtype,omitempty"`

	// Possible values described by the AnimeStatus constants.
	Status string `jsonapi:"attr,status,omitempty"`

	// The URL template for the poster, e.g.
	//
	// "tiny": "https://media.kitsu.io/anime/poster_images/1/tiny.jpg?1431697256"
	//
	// "small": "https://media.kitsu.io/anime/poster_images/1/small.jpg?1431697256"
	//
	// "medium": "https://media.kitsu.io/anime/poster_images/1/medium.jpg?1431697256"
	//
	// "large": "https://media.kitsu.io/anime/poster_images/1/large.jpg?1431697256"
	//
	// "original: "https://media.kitsu.io/anime/poster_images/1/original.jpg?1431697256"
	PosterImage map[string]interface{} `jsonapi:"attr,posterImage,omitempty"`

	// The URL template for the cover, e.g.
	//
	// "tiny": "https://media.kitsu.io/anime/cover_images/1/tiny.jpg?1416336000"
	//
	// "small": "https://media.kitsu.io/anime/cover_images/1/small.jpg?1416336000"
	//
	// "large": "https://media.kitsu.io/anime/cover_images/1/large.jpg?1416336000"
	//
	// "original": "https://media.kitsu.io/anime/cover_images/1/original.jpg?1416336000"
	CoverImage map[string]interface{} `jsonapi:"attr,coverImage,omitempty"`

	// How many episodes the anime has, e.g. 25
	EpisodeCount int `jsonapi:"attr,episodeCount,omitempty"`

	// How many minutes long each episode is, e.g. 24
	EpisodeLength int `jsonapi:"attr,episodeLength,omitempty"`

	// YouTube video id for Promotional Video, e.g. n4Nj6Y_SNYI
	YoutubeVideoID string `jsonapi:"attr,youtubeVideoId,omitempty"`

	// --- Relationships ---

	Genres   []*Genre   `jsonapi:"relation,genres,omitempty"`
	Castings []*Casting `jsonapi:"relation,castings,omitempty"`
}

// Genre represents a Kitsu media genre. Genre is a relationship of Kitsu media
// types like Anime, Manga and Drama.
type Genre struct {
	ID          string `jsonapi:"primary,genres"`
	Name        string `jsonapi:"attr,name"`
	Slug        string `jsonapi:"attr,slug"`
	Description string `jsonapi:"attr,description"`
}

// Casting represents a Kitsu media casting. Casting is a relationship of Kitsu
// media types like Anime, Manga and Drama.
type Casting struct {
	ID         string     `jsonapi:"primary,castings"`
	Role       string     `jsonapi:"attr,role"`
	VoiceActor bool       `jsonapi:"attr,voiceActor"`
	Featured   bool       `jsonapi:"attr,featured"`
	Language   string     `jsonapi:"attr,language"`
	Character  *Character `jsonapi:"relation,character"`
	Person     *Person    `jsonapi:"relation,person"`
}

// BUG(google/jsonapi): Unmarshaling of fields which are of type struct or
// map[string]string is not supported by google/jsonapi. A workaround for
// fields such as Character.Image and User.Avatar is to use
// map[string]interface{} instead.
//
// See: https://github.com/google/jsonapi/issues/74
//
// Another limitation is being unable to unmarshal to custom types such as
// "enum" types like AnimeType, MangaType and LibraryEntryStatus. These are
// useful for doing comparisons and working with fields such as Anime.ShowType,
// Manga.ShowType and LibraryEntry.Status.
//
// Because of this limitation the string type is used for those fields instead.
// As such, instead of using those custom types, we keep the possible values as
// untyped string constants to avoid unnecessary conversions when working with
// those fields.

// Character represents a Kitsu character like the fictional characters that
// appear in anime, manga and drama. Character is a relationship of Casting.
type Character struct {
	ID          string                 `jsonapi:"primary,characters"`
	Slug        string                 `jsonapi:"attr,slug"`
	Name        string                 `jsonapi:"attr,name"`
	MALID       int                    `jsonapi:"attr,malId"`
	Description string                 `jsonapi:"attr,description"`
	Image       map[string]interface{} `jsonapi:"attr,image"`
}

// Person represents a person that is involved with a certain media. It can be
// voice actors, animators, etc. Person is a relationship of Casting.
type Person struct {
	ID    string `jsonapi:"primary,people"`
	Name  string `jsonapi:"attr,name"`
	MALID int    `jsonapi:"attr,malId"`
	Image string `jsonapi:"attr,image"`
}

// Show returns details for a specific Anime by providing a unique identifier
// of the anime e.g. 7442.
func (s *AnimeService) Show(animeID string, opts ...URLOption) (*Anime, *Response, error) {
	u := fmt.Sprintf(defaultAPIVersion+"anime/%s", animeID)

	req, err := s.client.NewRequest("GET", u, nil, opts...)
	if err != nil {
		return nil, nil, err
	}

	a := new(Anime)
	resp, err := s.client.Do(req, a)
	if err != nil {
		return nil, resp, err
	}

	return a, resp, nil
}

// List returns a list of Anime. Optional parameters can be specified to filter
// the search results and control pagination, sorting etc.
func (s *AnimeService) List(opts ...URLOption) ([]*Anime, *Response, error) {
	u := defaultAPIVersion + "anime"

	req, err := s.client.NewRequest("GET", u, nil, opts...)
	if err != nil {
		return nil, nil, err
	}

	var anime []*Anime
	resp, err := s.client.Do(req, &anime)
	if err != nil {
		return nil, resp, err
	}

	return anime, resp, nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

type Foo struct {
	A string
}

// implements stringer Interface
func (f *Foo) String() string {
	return fmt.Sprintf("<Foo.A=%q>", f.A)
}

func main() {
	fo := &Foo{"another Foo"}
	fmt.Println(fo) // => <Foo.A="another Foo">
}
<commit_msg>fix signature for Stringer interface<commit_after>package main

import "fmt"

type Foo struct {
	A string
}

// implements stringer Interface
func (f Foo) String() string {
	return fmt.Sprintf("<Foo.A=%q>", f.A)
}

func main() {
	fo := &Foo{"another Foo"}
	fmt.Println(fo) // => <Foo.A="another Foo">
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protobuf

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	customreflect "k8s.io/kubernetes/third_party/forked/golang/reflect"
)

func rewriteFile(name string, header []byte, rewriteFn func(*token.FileSet, *ast.File) error) error {
	fset := token.NewFileSet()
	src, err := ioutil.ReadFile(name)
	if err != nil {
		return err
	}
	file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		return err
	}

	if err := rewriteFn(fset, file); err != nil {
		return err
	}

	b := &bytes.Buffer{}
	b.Write(header)
	if err := printer.Fprint(b, fset, file); err != nil {
		return err
	}

	body, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}

	f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := f.Write(body); err != nil {
		return err
	}
	return f.Close()
}

// ExtractFunc extracts information from the provided TypeSpec and returns true if the type should be
// removed from the destination file.
type ExtractFunc func(*ast.TypeSpec) bool

// OptionalFunc returns true if the provided local name is a type that has protobuf.nullable=true
// and should have its marshal functions adjusted to remove the 'Items' accessor.
type OptionalFunc func(name string) bool

func RewriteGeneratedGogoProtobufFile(name string, extractFn ExtractFunc, optionalFn OptionalFunc, header []byte) error {
	return rewriteFile(name, header, func(fset *token.FileSet, file *ast.File) error {
		cmap := ast.NewCommentMap(fset, file, file.Comments)

		// transform methods that point to optional maps or slices
		for _, d := range file.Decls {
			rewriteOptionalMethods(d, optionalFn)
		}

		// remove types that are already declared
		decls := []ast.Decl{}
		for _, d := range file.Decls {
			if dropExistingTypeDeclarations(d, extractFn) {
				continue
			}
			if dropEmptyImportDeclarations(d) {
				continue
			}
			decls = append(decls, d)
		}
		file.Decls = decls

		// remove unmapped comments
		file.Comments = cmap.Filter(file).Comments()
		return nil
	})
}

// rewriteOptionalMethods makes specific mutations to marshaller methods that belong to types identified
// as being "optional" (they may be nil on the wire). This allows protobuf to serialize a map or slice and
// properly discriminate between empty and nil (which is not possible in protobuf).
// TODO: move into upstream gogo-protobuf once https://github.com/gogo/protobuf/issues/181
//   has agreement
func rewriteOptionalMethods(decl ast.Decl, isOptional OptionalFunc) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		ident, ptr, ok := receiver(t)
		if !ok {
			return
		}

		// correct initialization of the form `m.Field = &OptionalType{}` to
		// `m.Field = OptionalType{}`
		if t.Name.Name == "Unmarshal" {
			ast.Walk(optionalAssignmentVisitor{fn: isOptional}, t.Body)
		}

		if !isOptional(ident.Name) {
			return
		}

		switch t.Name.Name {
		case "Unmarshal":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
		case "MarshalTo", "Size":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
			fallthrough
		case "Marshal":
			// if the method has a pointer receiver, set it back to a normal receiver
			if ptr {
				t.Recv.List[0].Type = ident
			}
		}
	}
}

type optionalAssignmentVisitor struct {
	fn OptionalFunc
}

// Visit walks the provided node, transforming field initializations of the form
// m.Field = &OptionalType{} -> m.Field = OptionalType{}
func (v optionalAssignmentVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			if !isFieldSelector(t.Lhs[0], "m", "") {
				return nil
			}
			unary, ok := t.Rhs[0].(*ast.UnaryExpr)
			if !ok || unary.Op != token.AND {
				return nil
			}
			composite, ok := unary.X.(*ast.CompositeLit)
			if !ok || composite.Type == nil || len(composite.Elts) != 0 {
				return nil
			}
			if ident, ok := composite.Type.(*ast.Ident); ok && v.fn(ident.Name) {
				t.Rhs[0] = composite
			}
		}
		return nil
	}
	return v
}

type optionalItemsVisitor struct{}

// Visit walks the provided node, looking for specific patterns to transform that match
// the effective outcome of turning struct{ map[x]y || []x } into map[x]y or []x.
func (v *optionalItemsVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.RangeStmt:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
		}
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			switch lhs := t.Lhs[0].(type) {
			case *ast.IndexExpr:
				if isFieldSelector(lhs.X, "m", "Items") {
					lhs.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			default:
				if isFieldSelector(t.Lhs[0], "m", "Items") {
					t.Lhs[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
			switch rhs := t.Rhs[0].(type) {
			case *ast.CallExpr:
				if ident, ok := rhs.Fun.(*ast.Ident); ok && ident.Name == "append" {
					ast.Walk(v, rhs)
					if len(rhs.Args) > 0 {
						switch arg := rhs.Args[0].(type) {
						case *ast.Ident:
							if arg.Name == "m" {
								rhs.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
							}
						}
					}
					return nil
				}
			}
		}
	case *ast.IfStmt:
		switch cond := t.Cond.(type) {
		case *ast.BinaryExpr:
			if cond.Op == token.EQL {
				if isFieldSelector(cond.X, "m", "Items") && isIdent(cond.Y, "nil") {
					cond.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
		}
		if t.Init != nil {
			// Find form:
			// if err := m[len(m.Items)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
			// 	return err
			// }
			switch s := t.Init.(type) {
			case *ast.AssignStmt:
				if call, ok := s.Rhs[0].(*ast.CallExpr); ok {
					if sel, ok := call.Fun.(*ast.SelectorExpr); ok {
						if x, ok := sel.X.(*ast.IndexExpr); ok {
							// m[] -> (*m)[]
							if sel2, ok := x.X.(*ast.SelectorExpr); ok {
								if ident, ok := sel2.X.(*ast.Ident); ok && ident.Name == "m" {
									x.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
								}
							}
							// len(m.Items) -> len(*m)
							if bin, ok := x.Index.(*ast.BinaryExpr); ok {
								if call2, ok := bin.X.(*ast.CallExpr); ok && len(call2.Args) == 1 {
									if isFieldSelector(call2.Args[0], "m", "Items") {
										call2.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
									}
								}
							}
						}
					}
				}
			}
		}
	case *ast.IndexExpr:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
			return nil
		}
	case *ast.CallExpr:
		changed := false
		for i := range t.Args {
			if isFieldSelector(t.Args[i], "m", "Items") {
				t.Args[i] = &ast.Ident{Name: "m"}
				changed = true
			}
		}
		if changed {
			return nil
		}
	}
	return v
}

func isFieldSelector(n ast.Expr, name, field string) bool {
	s, ok := n.(*ast.SelectorExpr)
	if !ok || s.Sel == nil || (field != "" && s.Sel.Name != field) {
		return false
	}
	return isIdent(s.X, name)
}

func isIdent(n ast.Expr, value string) bool {
	ident, ok := n.(*ast.Ident)
	return ok && ident.Name == value
}

func receiver(f *ast.FuncDecl) (ident *ast.Ident, pointer bool, ok bool) {
	if f.Recv == nil || len(f.Recv.List) != 1 {
		return nil, false, false
	}
	switch t := f.Recv.List[0].Type.(type) {
	case *ast.StarExpr:
		identity, ok := t.X.(*ast.Ident)
		if !ok {
			return nil, false, false
		}
		return identity, true, true
	case *ast.Ident:
		return t, false, true
	}
	return nil, false, false
}

// dropExistingTypeDeclarations removes any type declaration for which extractFn returns true. The function
// returns true if the entire declaration should be dropped.
func dropExistingTypeDeclarations(decl ast.Decl, extractFn ExtractFunc) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.TYPE {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.TypeSpec:
				if extractFn(spec) {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

// dropEmptyImportDeclarations strips any generated but no-op imports from the generated code
// to prevent generation from being able to define side-effects.  The function returns true
// if the entire declaration should be dropped.
func dropEmptyImportDeclarations(decl ast.Decl) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.IMPORT {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.ImportSpec:
				if spec.Name != nil && spec.Name.Name == "_" {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

func RewriteTypesWithProtobufStructTags(name string, structTags map[string]map[string]string) error {
	return rewriteFile(name, []byte{}, func(fset *token.FileSet, file *ast.File) error {
		allErrs := []error{}

		// set any new struct tags
		for _, d := range file.Decls {
			if errs := updateStructTags(d, structTags, []string{"protobuf"}); len(errs) > 0 {
				allErrs = append(allErrs, errs...)
			}
		}

		if len(allErrs) > 0 {
			var s string
			for _, err := range allErrs {
				s += err.Error() + "\n"
			}
			return errors.New(s)
		}
		return nil
	})
}

func updateStructTags(decl ast.Decl, structTags map[string]map[string]string, toCopy []string) []error {
	var errs []error
	t, ok := decl.(*ast.GenDecl)
	if !ok {
		return nil
	}
	if t.Tok != token.TYPE {
		return nil
	}

	for _, s := range t.Specs {
		spec, ok := s.(*ast.TypeSpec)
		if !ok {
			continue
		}
		typeName := spec.Name.Name
		fieldTags, ok := structTags[typeName]
		if !ok {
			continue
		}
		st, ok := spec.Type.(*ast.StructType)
		if !ok {
			continue
		}

		for i := range st.Fields.List {
			f := st.Fields.List[i]
			var name string
			if len(f.Names) == 0 {
				switch t := f.Type.(type) {
				case *ast.Ident:
					name = t.Name
				case *ast.SelectorExpr:
					name = t.Sel.Name
				default:
					errs = append(errs, fmt.Errorf("unable to get name for tag from struct %q, field %#v", spec.Name.Name, t))
					continue
				}
			} else {
				name = f.Names[0].Name
			}
			value, ok := fieldTags[name]
			if !ok {
				continue
			}
			var tags customreflect.StructTags
			if f.Tag != nil {
				oldTags, err := customreflect.ParseStructTags(strings.Trim(f.Tag.Value, "`"))
				if err != nil {
					errs = append(errs, fmt.Errorf("unable to read struct tag from struct %q, field %q: %v", spec.Name.Name, name, err))
					continue
				}
				tags = oldTags
			}
			for _, name := range toCopy {
				// don't overwrite existing tags
				if tags.Has(name) {
					continue
				}
				// append new tags
				if v := reflect.StructTag(value).Get(name); len(v) > 0 {
					tags = append(tags, customreflect.StructTag{Name: name, Value: v})
				}
			}
			if len(tags) == 0 {
				continue
			}
			if f.Tag == nil {
				f.Tag = &ast.BasicLit{}
			}
			f.Tag.Value = tags.String()
		}
	}
	return errs
}
<commit_msg>fix String protobuf generator<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protobuf

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	customreflect "k8s.io/kubernetes/third_party/forked/golang/reflect"
)

func rewriteFile(name string, header []byte, rewriteFn func(*token.FileSet, *ast.File) error) error {
	fset := token.NewFileSet()
	src, err := ioutil.ReadFile(name)
	if err != nil {
		return err
	}
	file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		return err
	}

	if err := rewriteFn(fset, file); err != nil {
		return err
	}

	b := &bytes.Buffer{}
	b.Write(header)
	if err := printer.Fprint(b, fset, file); err != nil {
		return err
	}

	body, err := format.Source(b.Bytes())
	if err != nil {
		return err
	}

	f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := f.Write(body); err != nil {
		return err
	}
	return f.Close()
}

// ExtractFunc extracts information from the provided TypeSpec and returns true if the type should be
// removed from the destination file.
type ExtractFunc func(*ast.TypeSpec) bool

// OptionalFunc returns true if the provided local name is a type that has protobuf.nullable=true
// and should have its marshal functions adjusted to remove the 'Items' accessor.
type OptionalFunc func(name string) bool

func RewriteGeneratedGogoProtobufFile(name string, extractFn ExtractFunc, optionalFn OptionalFunc, header []byte) error {
	return rewriteFile(name, header, func(fset *token.FileSet, file *ast.File) error {
		cmap := ast.NewCommentMap(fset, file, file.Comments)

		// transform methods that point to optional maps or slices
		for _, d := range file.Decls {
			rewriteOptionalMethods(d, optionalFn)
		}

		// remove types that are already declared
		decls := []ast.Decl{}
		for _, d := range file.Decls {
			if dropExistingTypeDeclarations(d, extractFn) {
				continue
			}
			if dropEmptyImportDeclarations(d) {
				continue
			}
			decls = append(decls, d)
		}
		file.Decls = decls

		// remove unmapped comments
		file.Comments = cmap.Filter(file).Comments()
		return nil
	})
}

// rewriteOptionalMethods makes specific mutations to marshaller methods that belong to types identified
// as being "optional" (they may be nil on the wire). This allows protobuf to serialize a map or slice and
// properly discriminate between empty and nil (which is not possible in protobuf).
// TODO: move into upstream gogo-protobuf once https://github.com/gogo/protobuf/issues/181
//   has agreement
func rewriteOptionalMethods(decl ast.Decl, isOptional OptionalFunc) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		ident, ptr, ok := receiver(t)
		if !ok {
			return
		}

		// correct initialization of the form `m.Field = &OptionalType{}` to
		// `m.Field = OptionalType{}`
		if t.Name.Name == "Unmarshal" {
			ast.Walk(optionalAssignmentVisitor{fn: isOptional}, t.Body)
		}

		if !isOptional(ident.Name) {
			return
		}

		switch t.Name.Name {
		case "Unmarshal":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
		case "MarshalTo", "Size", "String":
			ast.Walk(&optionalItemsVisitor{}, t.Body)
			fallthrough
		case "Marshal":
			// if the method has a pointer receiver, set it back to a normal receiver
			if ptr {
				t.Recv.List[0].Type = ident
			}
		}
	}
}

type optionalAssignmentVisitor struct {
	fn OptionalFunc
}

// Visit walks the provided node, transforming field initializations of the form
// m.Field = &OptionalType{} -> m.Field = OptionalType{}
func (v optionalAssignmentVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			if !isFieldSelector(t.Lhs[0], "m", "") {
				return nil
			}
			unary, ok := t.Rhs[0].(*ast.UnaryExpr)
			if !ok || unary.Op != token.AND {
				return nil
			}
			composite, ok := unary.X.(*ast.CompositeLit)
			if !ok || composite.Type == nil || len(composite.Elts) != 0 {
				return nil
			}
			if ident, ok := composite.Type.(*ast.Ident); ok && v.fn(ident.Name) {
				t.Rhs[0] = composite
			}
		}
		return nil
	}
	return v
}

type optionalItemsVisitor struct{}

// Visit walks the provided node, looking for specific patterns to transform that match
// the effective outcome of turning struct{ map[x]y || []x } into map[x]y or []x.
func (v *optionalItemsVisitor) Visit(n ast.Node) ast.Visitor {
	switch t := n.(type) {
	case *ast.RangeStmt:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
		}
	case *ast.AssignStmt:
		if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
			switch lhs := t.Lhs[0].(type) {
			case *ast.IndexExpr:
				if isFieldSelector(lhs.X, "m", "Items") {
					lhs.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			default:
				if isFieldSelector(t.Lhs[0], "m", "Items") {
					t.Lhs[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
			switch rhs := t.Rhs[0].(type) {
			case *ast.CallExpr:
				if ident, ok := rhs.Fun.(*ast.Ident); ok && ident.Name == "append" {
					ast.Walk(v, rhs)
					if len(rhs.Args) > 0 {
						switch arg := rhs.Args[0].(type) {
						case *ast.Ident:
							if arg.Name == "m" {
								rhs.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
							}
						}
					}
					return nil
				}
			}
		}
	case *ast.IfStmt:
		switch cond := t.Cond.(type) {
		case *ast.BinaryExpr:
			if cond.Op == token.EQL {
				if isFieldSelector(cond.X, "m", "Items") && isIdent(cond.Y, "nil") {
					cond.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
				}
			}
		}
		if t.Init != nil {
			// Find form:
			// if err := m[len(m.Items)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
			// 	return err
			// }
			switch s := t.Init.(type) {
			case *ast.AssignStmt:
				if call, ok := s.Rhs[0].(*ast.CallExpr); ok {
					if sel, ok := call.Fun.(*ast.SelectorExpr); ok {
						if x, ok := sel.X.(*ast.IndexExpr); ok {
							// m[] -> (*m)[]
							if sel2, ok := x.X.(*ast.SelectorExpr); ok {
								if ident, ok := sel2.X.(*ast.Ident); ok && ident.Name == "m" {
									x.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
								}
							}
							// len(m.Items) -> len(*m)
							if bin, ok := x.Index.(*ast.BinaryExpr); ok {
								if call2, ok := bin.X.(*ast.CallExpr); ok && len(call2.Args) == 1 {
									if isFieldSelector(call2.Args[0], "m", "Items") {
										call2.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
									}
								}
							}
						}
					}
				}
			}
		}
	case *ast.IndexExpr:
		if isFieldSelector(t.X, "m", "Items") {
			t.X = &ast.Ident{Name: "m"}
			return nil
		}
	case *ast.CallExpr:
		changed := false
		for i := range t.Args {
			if isFieldSelector(t.Args[i], "m", "Items") {
				t.Args[i] = &ast.Ident{Name: "m"}
				changed = true
			}
		}
		if changed {
			return nil
		}
	}
	return v
}

func isFieldSelector(n ast.Expr, name, field string) bool {
	s, ok := n.(*ast.SelectorExpr)
	if !ok || s.Sel == nil || (field != "" && s.Sel.Name != field) {
		return false
	}
	return isIdent(s.X, name)
}

func isIdent(n ast.Expr, value string) bool {
	ident, ok := n.(*ast.Ident)
	return ok && ident.Name == value
}

func receiver(f *ast.FuncDecl) (ident *ast.Ident, pointer bool, ok bool) {
	if f.Recv == nil || len(f.Recv.List) != 1 {
		return nil, false, false
	}
	switch t := f.Recv.List[0].Type.(type) {
	case *ast.StarExpr:
		identity, ok := t.X.(*ast.Ident)
		if !ok {
			return nil, false, false
		}
		return identity, true, true
	case *ast.Ident:
		return t, false, true
	}
	return nil, false, false
}

// dropExistingTypeDeclarations removes any type declaration for which extractFn returns true. The function
// returns true if the entire declaration should be dropped.
func dropExistingTypeDeclarations(decl ast.Decl, extractFn ExtractFunc) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.TYPE {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.TypeSpec:
				if extractFn(spec) {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

// dropEmptyImportDeclarations strips any generated but no-op imports from the generated code
// to prevent generation from being able to define side-effects.  The function returns true
// if the entire declaration should be dropped.
func dropEmptyImportDeclarations(decl ast.Decl) bool {
	switch t := decl.(type) {
	case *ast.GenDecl:
		if t.Tok != token.IMPORT {
			return false
		}
		specs := []ast.Spec{}
		for _, s := range t.Specs {
			switch spec := s.(type) {
			case *ast.ImportSpec:
				if spec.Name != nil && spec.Name.Name == "_" {
					continue
				}
				specs = append(specs, spec)
			}
		}
		if len(specs) == 0 {
			return true
		}
		t.Specs = specs
	}
	return false
}

func RewriteTypesWithProtobufStructTags(name string, structTags map[string]map[string]string) error {
	return rewriteFile(name, []byte{}, func(fset *token.FileSet, file *ast.File) error {
		allErrs := []error{}

		// set any new struct tags
		for _, d := range file.Decls {
			if errs := updateStructTags(d, structTags, []string{"protobuf"}); len(errs) > 0 {
				allErrs = append(allErrs, errs...)
			}
		}

		if len(allErrs) > 0 {
			var s string
			for _, err := range allErrs {
				s += err.Error() + "\n"
			}
			return errors.New(s)
		}
		return nil
	})
}

func updateStructTags(decl ast.Decl, structTags map[string]map[string]string, toCopy []string) []error {
	var errs []error
	t, ok := decl.(*ast.GenDecl)
	if !ok {
		return nil
	}
	if t.Tok != token.TYPE {
		return nil
	}

	for _, s := range t.Specs {
		spec, ok := s.(*ast.TypeSpec)
		if !ok {
			continue
		}
		typeName := spec.Name.Name
		fieldTags, ok := structTags[typeName]
		if !ok {
			continue
		}
		st, ok := spec.Type.(*ast.StructType)
		if !ok {
			continue
		}

		for i := range st.Fields.List {
			f := st.Fields.List[i]
			var name string
			if len(f.Names) == 0 {
				switch t := f.Type.(type) {
				case *ast.Ident:
					name = t.Name
				case *ast.SelectorExpr:
					name = t.Sel.Name
				default:
					errs = append(errs, fmt.Errorf("unable to get name for tag from struct %q, field %#v", spec.Name.Name, t))
					continue
				}
			} else {
				name = f.Names[0].Name
			}
			value, ok := fieldTags[name]
			if !ok {
				continue
			}
			var tags customreflect.StructTags
			if f.Tag != nil {
				oldTags, err := customreflect.ParseStructTags(strings.Trim(f.Tag.Value, "`"))
				if err != nil {
					errs = append(errs, fmt.Errorf("unable to read struct tag from struct %q, field %q: %v", spec.Name.Name, name, err))
					continue
				}
				tags = oldTags
			}
			for _, name := range toCopy {
				// don't overwrite existing tags
				if tags.Has(name) {
					continue
				}
				// append new tags
				if v := reflect.StructTag(value).Get(name); len(v) > 0 {
					tags = append(tags, customreflect.StructTag{Name: name, Value: v})
				}
			}
			if len(tags) == 0 {
				continue
			}
			if f.Tag == nil {
				f.Tag = &ast.BasicLit{}
			}
			f.Tag.Value = tags.String()
		}
	}
	return errs
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/pkg/multiec2"
	"koding/kites/kloud/plans"
	awsprovider "koding/kites/kloud/provider/aws"
	"koding/kites/kloud/provider/koding"
	"koding/kites/kloud/userdata"

	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/kloud"

	"koding/kites/kloud/kloudctl/command"

	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/multiconfig"
	"github.com/mitchellh/goamz/aws"
)

var Name = "kloud"

// Config defines the configuration that Kloud needs to operate.
type Config struct {
	// ---  KLOUD SPECIFIC ---
	IP          string
	Port        int
	Region      string
	Environment string

	// Connect to Koding mongodb
	MongoURL string `required:"true"`

	// Endpoint for fetching plans
	PlanEndpoint string `required:"true"`

	// Endpoint for fetching user machine network usage
	NetworkUsageEndpoint string `required:"true"`

	// --- DEVELOPMENT CONFIG ---
	// Show version and exit if enabled
	Version bool

	// Enable debug log mode
	DebugMode bool

	// Enable production mode, operates on production channel
	ProdMode bool

	// Enable test mode, disabled some authentication checks
	TestMode bool

	// Defines the base domain for domain creation
	HostedZone string `required:"true"`

	// Defines the default AMI Tag to use for koding provider
	AMITag string

	// --- KLIENT DEVELOPMENT ---
	// KontrolURL to connect and to de deployed with klient
	KontrolURL string `required:"true"`

	// Private key to create kite.key
	PrivateKey string `required:"true"`

	// Public key to create kite.key
	PublicKey string `required:"true"`

	// Private and public key to put a ssh key into the users VM's so we can
	// have access to it. Note that these are different then from the Kontrol
	// keys.
	UserPublicKey  string `required:"true"`
	UserPrivateKey string `required:"true"`

	// --- KONTROL CONFIGURATION ---
	Public      bool   // Try to register with a public ip
	RegisterURL string // Explicitly register with this given url

	AWSAccessKeyId     string
	AWSSecretAccessKey string

	JanitorSecretKey        string
	VmwatcherSecretKey      string
	PaymentwebhookSecretKey string
}

func main() {
	conf := new(Config)

	// Load the config, it's reads environment variables or from flags
	multiconfig.New().MustLoad(conf)

	if conf.Version {
		fmt.Println(kloud.VERSION)
		os.Exit(0)
	}

	k := newKite(conf)

	if conf.DebugMode {
		// This should be actually debug level 2. It outputs every single Kite
		// message and enables the kite debugging system. So enable it only if
		// you need it.
		// k.SetLogLevel(kite.DEBUG)
		k.Log.Info("Debug mode enabled")
	}

	if conf.TestMode {
		k.Log.Info("Test mode enabled")
	}

	registerURL := k.RegisterURL(!conf.Public)
	if conf.RegisterURL != "" {
		u, err := url.Parse(conf.RegisterURL)
		if err != nil {
			k.Log.Fatal("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if err := k.RegisterForever(registerURL); err != nil {
		k.Log.Fatal(err.Error())
	}

	// DataDog listens to it
	go func() {
		err := http.ListenAndServe("0.0.0.0:6060", nil)
		k.Log.Error(err.Error())
	}()

	k.Run()
}

func newKite(conf *Config) *kite.Kite {
	k := kite.New(kloud.NAME, kloud.VERSION)
	k.Config = kiteconfig.MustGet()
	k.Config.Port = conf.Port

	if conf.Region != "" {
		k.Config.Region = conf.Region
	}

	if conf.Environment != "" {
		k.Config.Environment = conf.Environment
	}

	if conf.AMITag != "" {
		k.Log.Warning("Default AMI Tag changed from %s to %s", koding.DefaultCustomAMITag, conf.AMITag)
		koding.DefaultCustomAMITag = conf.AMITag
	}

	klientFolder := "development/latest"
	checkInterval := time.Second * 5
	if conf.ProdMode {
		k.Log.Info("Prod mode enabled")
		klientFolder = "production/latest"
		checkInterval = time.Millisecond * 500
	}
	k.Log.Info("Klient distribution channel is: %s", klientFolder)

	modelhelper.Initialize(conf.MongoURL)
	db := modelhelper.Mongo

	kontrolPrivateKey, kontrolPublicKey := kontrolKeys(conf)

	// Credential belongs to the `koding-kloud` user in AWS IAM's
	auth := aws.Auth{
		AccessKey: conf.AWSAccessKeyId,
		SecretKey: conf.AWSSecretAccessKey,
	}

	dnsInstance := dnsclient.NewRoute53Client(conf.HostedZone, auth)
	dnsStorage := dnsstorage.NewMongodbStorage(db)
	userdata := &userdata.Userdata{
		Keycreator: &keycreator.Key{
			KontrolURL:        getKontrolURL(conf.KontrolURL),
			KontrolPrivateKey: kontrolPrivateKey,
			KontrolPublicKey:  kontrolPublicKey,
		},
		Bucket: userdata.NewBucket("koding-klient", klientFolder, auth),
	}
	ec2clients := multiec2.New(auth, []string{
		"us-east-1",
		"ap-southeast-1",
		"us-west-2",
		"eu-west-1",
	})

	authorizedUsers := map[string]string{
		"kloudctl":       command.KloudSecretKey,
		"janitor":        conf.JanitorSecretKey,
		"vmwatcher":      conf.VmwatcherSecretKey,
		"paymentwebhook": conf.PaymentwebhookSecretKey,
	}

	/// KODING PROVIDER ///

	kodingProvider := &koding.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-koding", conf.DebugMode),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       k,
		EC2Clients: ec2clients,
		Userdata:   userdata,
		PaymentFetcher: &plans.Payment{
			PaymentEndpoint: conf.PlanEndpoint,
		},
		CheckerFetcher: &plans.KodingChecker{
			NetworkUsageEndpoint: conf.NetworkUsageEndpoint,
		},
		AuthorizedUsers: authorizedUsers,
	}

	go kodingProvider.RunChecker(checkInterval)
	go kodingProvider.RunCleaners(time.Minute * 60)

	/// AWS PROVIDER ///

	awsProvider := &awsprovider.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-aws", conf.DebugMode),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       k,
		Userdata:   userdata,
	}

	// KLOUD DISPATCHER ///
	kloudLogger := common.NewLogger("kloud", conf.DebugMode)
	sess := &session.Session{
		DB:         db,
		Kite:       k,
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		AWSClients: ec2clients,
		Userdata:   userdata,
		Log:        kloudLogger,
	}

	stats := common.MustInitMetrics(Name)

	kld := kloud.New()
	kld.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}
	kld.Metrics = stats

	userPrivateKey, userPublicKey := userMachinesKeys(conf.UserPublicKey, conf.UserPrivateKey)

	// RSA key pair that we add to the newly created machine for
	// provisioning.
	kld.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kld.DomainStorage = dnsStorage
	kld.Domainer = dnsInstance
	kld.Locker = kodingProvider
	kld.Log = kloudLogger

	err := kld.AddProvider("koding", kodingProvider)
	if err != nil {
		panic(err)
	}

	err = kld.AddProvider("aws", awsProvider)
	if err != nil {
		panic(err)
	}

	// Machine handling methods
	k.HandleFunc("plan", kld.Plan)
	k.HandleFunc("apply", kld.Apply)
	k.HandleFunc("status", kld.Status)
	k.HandleFunc("authenticate", kld.Authenticate)
	k.HandleFunc("bootstrap", kld.Bootstrap)

	k.HandleFunc("build", kld.Build)
	k.HandleFunc("destroy", kld.Destroy)
	k.HandleFunc("stop", kld.Stop)
	k.HandleFunc("start", kld.Start)
	k.HandleFunc("reinit", kld.Reinit)
	k.HandleFunc("restart", kld.Restart)
	k.HandleFunc("info", kld.Info)
	k.HandleFunc("event", kld.Event)
	k.HandleFunc("resize", kld.Resize)

	// Snapshot functionality
	k.HandleFunc("createSnapshot", kld.CreateSnapshot)
	k.HandleFunc("deleteSnapshot", kld.DeleteSnapshot)

	// Domain records handling methods
	k.HandleFunc("domain.set", kld.DomainSet)
	k.HandleFunc("domain.unset", kld.DomainUnset)
	k.HandleFunc("domain.add", kld.DomainAdd)
	k.HandleFunc("domain.remove", kld.DomainRemove)

	k.HandleHTTPFunc("/healthCheck", artifact.HealthCheckHandler(Name))
	k.HandleHTTPFunc("/version", artifact.VersionHandler())

	for w, s := range authorizedUsers {
		func(worker, secretKey string) {
			k.Authenticators[worker] = func(r *kite.Request) error {
				if r.Auth.Key != secretKey {
					return errors.New("wrong secret key passed, you are not authenticated")
				}
				return nil
			}
		}(w, s)
	}

	return k
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

func kontrolKeys(conf *Config) (string, string) {
	pubKey, err := ioutil.ReadFile(conf.PublicKey)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(conf.PrivateKey)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return privateKey, publicKey
}

func getKontrolURL(ownURL string) string {
	// read kontrolURL from kite.key if it doesn't exist.
	kontrolURL := kiteconfig.MustGet().KontrolURL

	if ownURL != "" {
		u, err := url.Parse(ownURL)
		if err != nil {
			log.Fatalln(err)
		}

		kontrolURL = u.String()
	}

	return kontrolURL
}
<commit_msg>kloud: change method to describeStack for future addition informations<commit_after>package main

import (
	"errors"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"koding/artifact"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/pkg/multiec2"
	"koding/kites/kloud/plans"
	awsprovider "koding/kites/kloud/provider/aws"
	"koding/kites/kloud/provider/koding"
	"koding/kites/kloud/userdata"

	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/kloud"

	"koding/kites/kloud/kloudctl/command"

	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/multiconfig"
	"github.com/mitchellh/goamz/aws"
)

var Name = "kloud"

// Config defines the configuration that Kloud needs to operate.
type Config struct {
	// ---  KLOUD SPECIFIC ---
	IP          string
	Port        int
	Region      string
	Environment string

	// Connect to Koding mongodb
	MongoURL string `required:"true"`

	// Endpoint for fetching plans
	PlanEndpoint string `required:"true"`

	// Endpoint for fetching user machine network usage
	NetworkUsageEndpoint string `required:"true"`

	// --- DEVELOPMENT CONFIG ---
	// Show version and exit if enabled
	Version bool

	// Enable debug log mode
	DebugMode bool

	// Enable production mode, operates on production channel
	ProdMode bool

	// Enable test mode, disabled some authentication checks
	TestMode bool

	// Defines the base domain for domain creation
	HostedZone string `required:"true"`

	// Defines the default AMI Tag to use for koding provider
	AMITag string

	// --- KLIENT DEVELOPMENT ---
	// KontrolURL to connect and to de deployed with klient
	KontrolURL string `required:"true"`

	// Private key to create kite.key
	PrivateKey string `required:"true"`

	// Public key to create kite.key
	PublicKey string `required:"true"`

	// Private and public key to put a ssh key into the users VM's so we can
	// have access to it. Note that these are different then from the Kontrol
	// keys.
	UserPublicKey  string `required:"true"`
	UserPrivateKey string `required:"true"`

	// --- KONTROL CONFIGURATION ---
	Public      bool   // Try to register with a public ip
	RegisterURL string // Explicitly register with this given url

	AWSAccessKeyId     string
	AWSSecretAccessKey string

	JanitorSecretKey        string
	VmwatcherSecretKey      string
	PaymentwebhookSecretKey string
}

func main() {
	conf := new(Config)

	// Load the config, it's reads environment variables or from flags
	multiconfig.New().MustLoad(conf)

	if conf.Version {
		fmt.Println(kloud.VERSION)
		os.Exit(0)
	}

	k := newKite(conf)

	if conf.DebugMode {
		// This should be actually debug level 2. It outputs every single Kite
		// message and enables the kite debugging system. So enable it only if
		// you need it.
		// k.SetLogLevel(kite.DEBUG)
		k.Log.Info("Debug mode enabled")
	}

	if conf.TestMode {
		k.Log.Info("Test mode enabled")
	}

	registerURL := k.RegisterURL(!conf.Public)
	if conf.RegisterURL != "" {
		u, err := url.Parse(conf.RegisterURL)
		if err != nil {
			k.Log.Fatal("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if err := k.RegisterForever(registerURL); err != nil {
		k.Log.Fatal(err.Error())
	}

	// DataDog listens to it
	go func() {
		err := http.ListenAndServe("0.0.0.0:6060", nil)
		k.Log.Error(err.Error())
	}()

	k.Run()
}

func newKite(conf *Config) *kite.Kite {
	k := kite.New(kloud.NAME, kloud.VERSION)
	k.Config = kiteconfig.MustGet()
	k.Config.Port = conf.Port

	if conf.Region != "" {
		k.Config.Region = conf.Region
	}

	if conf.Environment != "" {
		k.Config.Environment = conf.Environment
	}

	if conf.AMITag != "" {
		k.Log.Warning("Default AMI Tag changed from %s to %s", koding.DefaultCustomAMITag, conf.AMITag)
		koding.DefaultCustomAMITag = conf.AMITag
	}

	klientFolder := "development/latest"
	checkInterval := time.Second * 5
	if conf.ProdMode {
		k.Log.Info("Prod mode enabled")
		klientFolder = "production/latest"
		checkInterval = time.Millisecond * 500
	}
	k.Log.Info("Klient distribution channel is: %s", klientFolder)

	modelhelper.Initialize(conf.MongoURL)
	db := modelhelper.Mongo

	kontrolPrivateKey, kontrolPublicKey := kontrolKeys(conf)

	// Credential belongs to the `koding-kloud` user in AWS IAM's
	auth := aws.Auth{
		AccessKey: conf.AWSAccessKeyId,
		SecretKey: conf.AWSSecretAccessKey,
	}

	dnsInstance := dnsclient.NewRoute53Client(conf.HostedZone, auth)
	dnsStorage := dnsstorage.NewMongodbStorage(db)
	userdata := &userdata.Userdata{
		Keycreator: &keycreator.Key{
			KontrolURL:        getKontrolURL(conf.KontrolURL),
			KontrolPrivateKey: kontrolPrivateKey,
			KontrolPublicKey:  kontrolPublicKey,
		},
		Bucket: userdata.NewBucket("koding-klient", klientFolder, auth),
	}
	ec2clients := multiec2.New(auth, []string{
		"us-east-1",
		"ap-southeast-1",
		"us-west-2",
		"eu-west-1",
	})

	authorizedUsers := map[string]string{
		"kloudctl":       command.KloudSecretKey,
		"janitor":        conf.JanitorSecretKey,
		"vmwatcher":      conf.VmwatcherSecretKey,
		"paymentwebhook": conf.PaymentwebhookSecretKey,
	}

	/// KODING PROVIDER ///

	kodingProvider := &koding.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-koding", conf.DebugMode),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       k,
		EC2Clients: ec2clients,
		Userdata:   userdata,
		PaymentFetcher: &plans.Payment{
			PaymentEndpoint: conf.PlanEndpoint,
		},
		CheckerFetcher: &plans.KodingChecker{
			NetworkUsageEndpoint: conf.NetworkUsageEndpoint,
		},
		AuthorizedUsers: authorizedUsers,
	}

	go kodingProvider.RunChecker(checkInterval)
	go kodingProvider.RunCleaners(time.Minute * 60)

	/// AWS PROVIDER ///

	awsProvider := &awsprovider.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-aws", conf.DebugMode),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       k,
		Userdata:   userdata,
	}

	// KLOUD DISPATCHER ///
	kloudLogger := common.NewLogger("kloud", conf.DebugMode)
	sess := &session.Session{
		DB:         db,
		Kite:       k,
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		AWSClients: ec2clients,
		Userdata:   userdata,
		Log:        kloudLogger,
	}

	stats := common.MustInitMetrics(Name)

	kld := kloud.New()
	kld.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}
	kld.Metrics = stats

	userPrivateKey, userPublicKey := userMachinesKeys(conf.UserPublicKey, conf.UserPrivateKey)

	// RSA key pair that we add to the newly created machine for
	// provisioning.
	kld.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kld.DomainStorage = dnsStorage
	kld.Domainer = dnsInstance
	kld.Locker = kodingProvider
	kld.Log = kloudLogger

	err := kld.AddProvider("koding", kodingProvider)
	if err != nil {
		panic(err)
	}

	err = kld.AddProvider("aws", awsProvider)
	if err != nil {
		panic(err)
	}

	// Machine handling methods
	k.HandleFunc("plan", kld.Plan)
	k.HandleFunc("apply", kld.Apply)
	k.HandleFunc("describeStack", kld.Status)
	k.HandleFunc("authenticate", kld.Authenticate)
	k.HandleFunc("bootstrap", kld.Bootstrap)

	k.HandleFunc("build", kld.Build)
	k.HandleFunc("destroy", kld.Destroy)
	k.HandleFunc("stop", kld.Stop)
	k.HandleFunc("start", kld.Start)
	k.HandleFunc("reinit", kld.Reinit)
	k.HandleFunc("restart", kld.Restart)
	k.HandleFunc("info", kld.Info)
	k.HandleFunc("event", kld.Event)
	k.HandleFunc("resize", kld.Resize)

	// Snapshot functionality
	k.HandleFunc("createSnapshot", kld.CreateSnapshot)
	k.HandleFunc("deleteSnapshot", kld.DeleteSnapshot)

	// Domain records handling methods
	k.HandleFunc("domain.set", kld.DomainSet)
	k.HandleFunc("domain.unset", kld.DomainUnset)
	k.HandleFunc("domain.add", kld.DomainAdd)
	k.HandleFunc("domain.remove", kld.DomainRemove)

	k.HandleHTTPFunc("/healthCheck", artifact.HealthCheckHandler(Name))
	k.HandleHTTPFunc("/version", artifact.VersionHandler())

	for w, s := range authorizedUsers {
		func(worker, secretKey string) {
			k.Authenticators[worker] = func(r *kite.Request) error {
				if r.Auth.Key != secretKey {
					return errors.New("wrong secret key passed, you are not authenticated")
				}
				return nil
			}
		}(w, s)
	}

	return k
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

func kontrolKeys(conf *Config) (string, string) {
	pubKey, err := ioutil.ReadFile(conf.PublicKey)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(conf.PrivateKey)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return privateKey, publicKey
}

func getKontrolURL(ownURL string) string {
	// read kontrolURL from kite.key if it doesn't exist.
	kontrolURL := kiteconfig.MustGet().KontrolURL

	if ownURL != "" {
		u, err := url.Parse(ownURL)
		if err != nil {
			log.Fatalln(err)
		}

		kontrolURL = u.String()
	}

	return kontrolURL
}
<|endoftext|>