<commit_before>// Copyright (C) 2014 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	log "github.com/Sirupsen/logrus"
	"github.com/osrg/gobgp/config"
	"github.com/osrg/gobgp/packet"
	"gopkg.in/tomb.v2"
	"net"
	"time"
)

type fsmMsgType int

const (
	_ fsmMsgType = iota
	FSM_MSG_STATE_CHANGE
	FSM_MSG_BGP_MESSAGE
)

type fsmMsg struct {
	MsgType fsmMsgType
	MsgData interface{}
}

type FSM struct {
	globalConfig    *config.GlobalType
	peerConfig      *config.NeighborType
	keepaliveTicker *time.Ticker
	state           bgp.FSMState
	incoming        chan *fsmMsg
	outgoing        chan *bgp.BGPMessage
	passiveConn     *net.TCPConn
	passiveConnCh   chan *net.TCPConn
}

func (fsm *FSM) bgpMessageStateUpdate(MessageType uint8, isIn bool) {
	state := &fsm.peerConfig.BgpNeighborCommonState
	if isIn {
		state.TotalIn++
	} else {
		state.TotalOut++
	}
	switch MessageType {
	case bgp.BGP_MSG_OPEN:
		if isIn {
			state.OpenIn++
		} else {
			state.OpenOut++
		}
	case bgp.BGP_MSG_UPDATE:
		if isIn {
			state.UpdateIn++
			state.UpdateRecvTime = time.Now()
		} else {
			state.UpdateOut++
		}
	case bgp.BGP_MSG_NOTIFICATION:
		if isIn {
			state.NotifyIn++
		} else {
			state.NotifyOut++
		}
	case bgp.BGP_MSG_KEEPALIVE:
		if isIn {
			state.KeepaliveIn++
		} else {
			state.KeepaliveOut++
		}
	case bgp.BGP_MSG_ROUTE_REFRESH:
		if isIn {
			state.RefreshIn++
		} else {
			state.RefreshOut++
		}
	}
}

func NewFSM(gConfig *config.GlobalType, pConfig *config.NeighborType, connCh chan *net.TCPConn, incoming chan *fsmMsg, outgoing chan *bgp.BGPMessage) *FSM {
	return &FSM{
		globalConfig:  gConfig,
		peerConfig:    pConfig,
		incoming:      incoming,
		outgoing:      outgoing,
		state:         bgp.BGP_FSM_IDLE,
		passiveConnCh: connCh,
	}
}

func (fsm *FSM) StateChange(nextState bgp.FSMState) {
	log.WithFields(log.Fields{
		"Topic": "Peer",
		"Key":   fsm.peerConfig.NeighborAddress,
		"old":   fsm.state.String(),
		"new":   nextState.String(),
	}).Debug("state changed")
	fsm.state = nextState
}

type FSMHandler struct {
	t       tomb.Tomb
	fsm     *FSM
	conn    *net.TCPConn
	msgCh   chan *fsmMsg
	errorCh chan bool
}

func NewFSMHandler(fsm *FSM) *FSMHandler {
	f := &FSMHandler{
		fsm:     fsm,
		errorCh: make(chan bool),
	}
	f.t.Go(f.loop)
	return f
}

func (h *FSMHandler) Wait() error {
	return h.t.Wait()
}

func (h *FSMHandler) Stop() error {
	h.t.Kill(nil)
	return h.t.Wait()
}

func (h *FSMHandler) idle() bgp.FSMState {
	fsm := h.fsm
	// TODO: support idle hold timer

	if fsm.keepaliveTicker != nil {
		fsm.keepaliveTicker.Stop()
		fsm.keepaliveTicker = nil
	}
	return bgp.BGP_FSM_ACTIVE
}

func (h *FSMHandler) active() bgp.FSMState {
	fsm := h.fsm
	select {
	case <-h.t.Dying():
		return 0
	case conn := <-fsm.passiveConnCh:
		fsm.passiveConn = conn
	}
	// we don't implement delayed open timer so move to opensent right
	// away.
	return bgp.BGP_FSM_OPENSENT
}

func buildopen(global *config.GlobalType, peerConf *config.NeighborType) *bgp.BGPMessage {
	var afi int
	if peerConf.NeighborAddress.To4() != nil {
		afi = bgp.AFI_IP
	} else {
		afi = bgp.AFI_IP6
	}
	p1 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapRouteRefresh()})
	p2 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapMultiProtocol(uint16(afi), bgp.SAFI_UNICAST)})
	p3 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapFourOctetASNumber(global.As)})
	holdTime := uint16(peerConf.Timers.HoldTime)
	as := global.As
	if as > (1<<16)-1 {
		as = bgp.AS_TRANS
	}
	return bgp.NewBGPOpenMessage(uint16(as), holdTime, global.RouterId.String(),
		[]bgp.OptionParameterInterface{p1, p2, p3})
}

func readAll(conn *net.TCPConn, length int) ([]byte, error) {
	buf := make([]byte, length)
	for cur := 0; cur < length; {
		if num, err := conn.Read(buf); err != nil {
			return nil, err
		} else {
			cur += num
		}
	}
	return buf, nil
}

func (h *FSMHandler) recvMessageWithError() error {
	headerBuf, err := readAll(h.conn, bgp.BGP_HEADER_LENGTH)
	if err != nil {
		h.errorCh <- true
		return err
	}

	hd := &bgp.BGPHeader{}
	err = hd.DecodeFromBytes(headerBuf)
	if err != nil {
		h.errorCh <- true
		return err
	}

	bodyBuf, err := readAll(h.conn, int(hd.Len)-bgp.BGP_HEADER_LENGTH)
	if err != nil {
		h.errorCh <- true
		return err
	}

	m, err := bgp.ParseBGPBody(hd, bodyBuf)
	if err != nil {
		h.errorCh <- true
		return err
	}
	e := &fsmMsg{
		MsgType: FSM_MSG_BGP_MESSAGE,
		MsgData: m,
	}
	h.msgCh <- e
	h.fsm.bgpMessageStateUpdate(m.Header.Type, true)
	return nil
}

func (h *FSMHandler) recvMessage() error {
	h.recvMessageWithError()
	return nil
}

func (h *FSMHandler) opensent() bgp.FSMState {
	fsm := h.fsm
	m := buildopen(fsm.globalConfig, fsm.peerConfig)
	b, _ := m.Serialize()
	fsm.passiveConn.Write(b)
	fsm.bgpMessageStateUpdate(m.Header.Type, false)

	h.msgCh = make(chan *fsmMsg)
	h.conn = fsm.passiveConn

	h.t.Go(h.recvMessage)

	nextState := bgp.BGP_FSM_IDLE
	select {
	case <-h.t.Dying():
		fsm.passiveConn.Close()
		return 0
	case e := <-h.msgCh:
		m := e.MsgData.(*bgp.BGPMessage)
		if m.Header.Type == bgp.BGP_MSG_OPEN {
			e := &fsmMsg{
				MsgType: FSM_MSG_BGP_MESSAGE,
				MsgData: m,
			}
			fsm.incoming <- e
			msg := bgp.NewBGPKeepAliveMessage()
			b, _ := msg.Serialize()
			fsm.passiveConn.Write(b)
			nextState = bgp.BGP_FSM_OPENCONFIRM
			fsm.bgpMessageStateUpdate(msg.Header.Type, false)
		} else {
			// send error
		}
	case <-h.errorCh:
	}
	return nextState
}

func (h *FSMHandler) openconfirm() bgp.FSMState {
	fsm := h.fsm
	sec := time.Second * time.Duration(fsm.peerConfig.Timers.KeepaliveInterval)
	fsm.keepaliveTicker = time.NewTicker(sec)

	h.msgCh = make(chan *fsmMsg)
	h.conn = fsm.passiveConn

	h.t.Go(h.recvMessage)

	for {
		select {
		case <-h.t.Dying():
			fsm.passiveConn.Close()
			return 0
		case <-fsm.keepaliveTicker.C:
			m := bgp.NewBGPKeepAliveMessage()
			b, _ := m.Serialize()
			// TODO: check error
			fsm.passiveConn.Write(b)
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		case e := <-h.msgCh:
			m := e.MsgData.(*bgp.BGPMessage)
			nextState := bgp.BGP_FSM_IDLE
			if m.Header.Type == bgp.BGP_MSG_KEEPALIVE {
				nextState = bgp.BGP_FSM_ESTABLISHED
			} else {
				// send error
			}
			return nextState
		case <-h.errorCh:
			return bgp.BGP_FSM_IDLE
		}
	}
	// panic
	return 0
}

func (h *FSMHandler) sendMessageloop() error {
	conn := h.conn
	fsm := h.fsm
	for {
		select {
		case <-h.t.Dying():
			return nil
		case m := <-fsm.outgoing:
			b, _ := m.Serialize()
			_, err := conn.Write(b)
			if err != nil {
				h.errorCh <- true
				return nil
			}
			log.WithFields(log.Fields{
				"Topic": "Peer",
				"Key":   fsm.peerConfig.NeighborAddress,
				"data":  m,
			}).Debug("sent")
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		case <-fsm.keepaliveTicker.C:
			m := bgp.NewBGPKeepAliveMessage()
			b, _ := m.Serialize()
			_, err := conn.Write(b)
			if err != nil {
				h.errorCh <- true
				return nil
			}
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		}
	}
}

func (h *FSMHandler) recvMessageloop() error {
	for {
		err := h.recvMessageWithError()
		if err != nil {
			e := err.(*bgp.MessageError)
			m := bgp.NewBGPNotificationMessage(e.TypeCode, e.SubTypeCode, e.Data)
			h.fsm.outgoing <- m
			return nil
		}
	}
}

func (h *FSMHandler) established() bgp.FSMState {
	fsm := h.fsm
	h.conn = fsm.passiveConn
	h.t.Go(h.sendMessageloop)
	h.msgCh = fsm.incoming
	h.t.Go(h.recvMessageloop)

	for {
		select {
		case <-h.errorCh:
			h.conn.Close()
			h.t.Kill(nil)
			return bgp.BGP_FSM_IDLE
		case <-h.t.Dying():
			h.conn.Close()
			return 0
		}
	}
	return 0
}

func (h *FSMHandler) loop() error {
	fsm := h.fsm
	nextState := bgp.FSMState(0)
	switch fsm.state {
	case bgp.BGP_FSM_IDLE:
		nextState = h.idle()
		//	case bgp.BGP_FSM_CONNECT:
		//		return h.connect()
	case bgp.BGP_FSM_ACTIVE:
		nextState = h.active()
	case bgp.BGP_FSM_OPENSENT:
		nextState = h.opensent()
	case bgp.BGP_FSM_OPENCONFIRM:
		nextState = h.openconfirm()
	case bgp.BGP_FSM_ESTABLISHED:
		nextState = h.established()
	}

	// zero means that tomb.Dying()
	if nextState >= bgp.BGP_FSM_IDLE {
		e := &fsmMsg{
			MsgType: FSM_MSG_STATE_CHANGE,
			MsgData: nextState,
		}
		fsm.incoming <- e
	}
	return nil
}
<commit_msg>server: fix recvMessageloop() error handling<commit_after>// Copyright (C) 2014 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	log "github.com/Sirupsen/logrus"
	"github.com/osrg/gobgp/config"
	"github.com/osrg/gobgp/packet"
	"gopkg.in/tomb.v2"
	"net"
	"time"
)

type fsmMsgType int

const (
	_ fsmMsgType = iota
	FSM_MSG_STATE_CHANGE
	FSM_MSG_BGP_MESSAGE
)

type fsmMsg struct {
	MsgType fsmMsgType
	MsgData interface{}
}

type FSM struct {
	globalConfig    *config.GlobalType
	peerConfig      *config.NeighborType
	keepaliveTicker *time.Ticker
	state           bgp.FSMState
	incoming        chan *fsmMsg
	outgoing        chan *bgp.BGPMessage
	passiveConn     *net.TCPConn
	passiveConnCh   chan *net.TCPConn
}

func (fsm *FSM) bgpMessageStateUpdate(MessageType uint8, isIn bool) {
	state := &fsm.peerConfig.BgpNeighborCommonState
	if isIn {
		state.TotalIn++
	} else {
		state.TotalOut++
	}
	switch MessageType {
	case bgp.BGP_MSG_OPEN:
		if isIn {
			state.OpenIn++
		} else {
			state.OpenOut++
		}
	case bgp.BGP_MSG_UPDATE:
		if isIn {
			state.UpdateIn++
			state.UpdateRecvTime = time.Now()
		} else {
			state.UpdateOut++
		}
	case bgp.BGP_MSG_NOTIFICATION:
		if isIn {
			state.NotifyIn++
		} else {
			state.NotifyOut++
		}
	case bgp.BGP_MSG_KEEPALIVE:
		if isIn {
			state.KeepaliveIn++
		} else {
			state.KeepaliveOut++
		}
	case bgp.BGP_MSG_ROUTE_REFRESH:
		if isIn {
			state.RefreshIn++
		} else {
			state.RefreshOut++
		}
	}
}

func NewFSM(gConfig *config.GlobalType, pConfig *config.NeighborType, connCh chan *net.TCPConn, incoming chan *fsmMsg, outgoing chan *bgp.BGPMessage) *FSM {
	return &FSM{
		globalConfig:  gConfig,
		peerConfig:    pConfig,
		incoming:      incoming,
		outgoing:      outgoing,
		state:         bgp.BGP_FSM_IDLE,
		passiveConnCh: connCh,
	}
}

func (fsm *FSM) StateChange(nextState bgp.FSMState) {
	log.WithFields(log.Fields{
		"Topic": "Peer",
		"Key":   fsm.peerConfig.NeighborAddress,
		"old":   fsm.state.String(),
		"new":   nextState.String(),
	}).Debug("state changed")
	fsm.state = nextState
}

type FSMHandler struct {
	t       tomb.Tomb
	fsm     *FSM
	conn    *net.TCPConn
	msgCh   chan *fsmMsg
	errorCh chan bool
}

func NewFSMHandler(fsm *FSM) *FSMHandler {
	f := &FSMHandler{
		fsm:     fsm,
		errorCh: make(chan bool),
	}
	f.t.Go(f.loop)
	return f
}

func (h *FSMHandler) Wait() error {
	return h.t.Wait()
}

func (h *FSMHandler) Stop() error {
	h.t.Kill(nil)
	return h.t.Wait()
}

func (h *FSMHandler) idle() bgp.FSMState {
	fsm := h.fsm
	// TODO: support idle hold timer

	if fsm.keepaliveTicker != nil {
		fsm.keepaliveTicker.Stop()
		fsm.keepaliveTicker = nil
	}
	return bgp.BGP_FSM_ACTIVE
}

func (h *FSMHandler) active() bgp.FSMState {
	fsm := h.fsm
	select {
	case <-h.t.Dying():
		return 0
	case conn := <-fsm.passiveConnCh:
		fsm.passiveConn = conn
	}
	// we don't implement delayed open timer so move to opensent right
	// away.
	return bgp.BGP_FSM_OPENSENT
}

func buildopen(global *config.GlobalType, peerConf *config.NeighborType) *bgp.BGPMessage {
	var afi int
	if peerConf.NeighborAddress.To4() != nil {
		afi = bgp.AFI_IP
	} else {
		afi = bgp.AFI_IP6
	}
	p1 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapRouteRefresh()})
	p2 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapMultiProtocol(uint16(afi), bgp.SAFI_UNICAST)})
	p3 := bgp.NewOptionParameterCapability(
		[]bgp.ParameterCapabilityInterface{bgp.NewCapFourOctetASNumber(global.As)})
	holdTime := uint16(peerConf.Timers.HoldTime)
	as := global.As
	if as > (1<<16)-1 {
		as = bgp.AS_TRANS
	}
	return bgp.NewBGPOpenMessage(uint16(as), holdTime, global.RouterId.String(),
		[]bgp.OptionParameterInterface{p1, p2, p3})
}

func readAll(conn *net.TCPConn, length int) ([]byte, error) {
	buf := make([]byte, length)
	for cur := 0; cur < length; {
		if num, err := conn.Read(buf); err != nil {
			return nil, err
		} else {
			cur += num
		}
	}
	return buf, nil
}

func (h *FSMHandler) recvMessageWithError() error {
	headerBuf, err := readAll(h.conn, bgp.BGP_HEADER_LENGTH)
	if err != nil {
		h.errorCh <- true
		return err
	}

	hd := &bgp.BGPHeader{}
	err = hd.DecodeFromBytes(headerBuf)
	if err != nil {
		h.errorCh <- true
		return err
	}

	bodyBuf, err := readAll(h.conn, int(hd.Len)-bgp.BGP_HEADER_LENGTH)
	if err != nil {
		h.errorCh <- true
		return err
	}

	m, err := bgp.ParseBGPBody(hd, bodyBuf)
	if err != nil {
		h.errorCh <- true
		return err
	}
	e := &fsmMsg{
		MsgType: FSM_MSG_BGP_MESSAGE,
		MsgData: m,
	}
	h.msgCh <- e
	h.fsm.bgpMessageStateUpdate(m.Header.Type, true)
	return nil
}

func (h *FSMHandler) recvMessage() error {
	h.recvMessageWithError()
	return nil
}

func (h *FSMHandler) opensent() bgp.FSMState {
	fsm := h.fsm
	m := buildopen(fsm.globalConfig, fsm.peerConfig)
	b, _ := m.Serialize()
	fsm.passiveConn.Write(b)
	fsm.bgpMessageStateUpdate(m.Header.Type, false)

	h.msgCh = make(chan *fsmMsg)
	h.conn = fsm.passiveConn

	h.t.Go(h.recvMessage)

	nextState := bgp.BGP_FSM_IDLE
	select {
	case <-h.t.Dying():
		fsm.passiveConn.Close()
		return 0
	case e := <-h.msgCh:
		m := e.MsgData.(*bgp.BGPMessage)
		if m.Header.Type == bgp.BGP_MSG_OPEN {
			e := &fsmMsg{
				MsgType: FSM_MSG_BGP_MESSAGE,
				MsgData: m,
			}
			fsm.incoming <- e
			msg := bgp.NewBGPKeepAliveMessage()
			b, _ := msg.Serialize()
			fsm.passiveConn.Write(b)
			nextState = bgp.BGP_FSM_OPENCONFIRM
			fsm.bgpMessageStateUpdate(msg.Header.Type, false)
		} else {
			// send error
		}
	case <-h.errorCh:
	}
	return nextState
}

func (h *FSMHandler) openconfirm() bgp.FSMState {
	fsm := h.fsm
	sec := time.Second * time.Duration(fsm.peerConfig.Timers.KeepaliveInterval)
	fsm.keepaliveTicker = time.NewTicker(sec)

	h.msgCh = make(chan *fsmMsg)
	h.conn = fsm.passiveConn

	h.t.Go(h.recvMessage)

	for {
		select {
		case <-h.t.Dying():
			fsm.passiveConn.Close()
			return 0
		case <-fsm.keepaliveTicker.C:
			m := bgp.NewBGPKeepAliveMessage()
			b, _ := m.Serialize()
			// TODO: check error
			fsm.passiveConn.Write(b)
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		case e := <-h.msgCh:
			m := e.MsgData.(*bgp.BGPMessage)
			nextState := bgp.BGP_FSM_IDLE
			if m.Header.Type == bgp.BGP_MSG_KEEPALIVE {
				nextState = bgp.BGP_FSM_ESTABLISHED
			} else {
				// send error
			}
			return nextState
		case <-h.errorCh:
			return bgp.BGP_FSM_IDLE
		}
	}
	// panic
	return 0
}

func (h *FSMHandler) sendMessageloop() error {
	conn := h.conn
	fsm := h.fsm
	for {
		select {
		case <-h.t.Dying():
			return nil
		case m := <-fsm.outgoing:
			b, _ := m.Serialize()
			_, err := conn.Write(b)
			if err != nil {
				h.errorCh <- true
				return nil
			}
			log.WithFields(log.Fields{
				"Topic": "Peer",
				"Key":   fsm.peerConfig.NeighborAddress,
				"data":  m,
			}).Debug("sent")
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		case <-fsm.keepaliveTicker.C:
			m := bgp.NewBGPKeepAliveMessage()
			b, _ := m.Serialize()
			_, err := conn.Write(b)
			if err != nil {
				h.errorCh <- true
				return nil
			}
			fsm.bgpMessageStateUpdate(m.Header.Type, false)
		}
	}
}

func (h *FSMHandler) recvMessageloop() error {
	for {
		err := h.recvMessageWithError()
		if err != nil {
			e, y := err.(*bgp.MessageError)
			if y {
				m := bgp.NewBGPNotificationMessage(e.TypeCode, e.SubTypeCode, e.Data)
				h.fsm.outgoing <- m
			}
			return nil
		}
	}
}

func (h *FSMHandler) established() bgp.FSMState {
	fsm := h.fsm
	h.conn = fsm.passiveConn
	h.t.Go(h.sendMessageloop)
	h.msgCh = fsm.incoming
	h.t.Go(h.recvMessageloop)

	for {
		select {
		case <-h.errorCh:
			h.conn.Close()
			h.t.Kill(nil)
			return bgp.BGP_FSM_IDLE
		case <-h.t.Dying():
			h.conn.Close()
			return 0
		}
	}
	return 0
}

func (h *FSMHandler) loop() error {
	fsm := h.fsm
	nextState := bgp.FSMState(0)
	switch fsm.state {
	case bgp.BGP_FSM_IDLE:
		nextState = h.idle()
		//	case bgp.BGP_FSM_CONNECT:
		//		return h.connect()
	case bgp.BGP_FSM_ACTIVE:
		nextState = h.active()
	case bgp.BGP_FSM_OPENSENT:
		nextState = h.opensent()
	case bgp.BGP_FSM_OPENCONFIRM:
		nextState = h.openconfirm()
	case bgp.BGP_FSM_ESTABLISHED:
		nextState = h.established()
	}

	// zero means that tomb.Dying()
	if nextState >= bgp.BGP_FSM_IDLE {
		e := &fsmMsg{
			MsgType: FSM_MSG_STATE_CHANGE,
			MsgData: nextState,
		}
		fsm.incoming <- e
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"os"

	"github.com/PagerDuty/godspeed"
	"github.com/vsco/dcdr/cli"
	"github.com/vsco/dcdr/cli/api"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/cli/repo"
	"github.com/vsco/dcdr/config"
)

func main() {
	cfg := config.LoadConfig()
	store, err := stores.DefaultConsulStore()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	rp := repo.New(cfg)

	cmd := ""

	if len(os.Args) > 1 {
		cmd = os.Args[1]
	}

	if cmd != "init" && cmd != "help" && cmd != "watch" && rp.Enabled() && !rp.Exists() {
		printer.SayErr("%s has not been cloned to %s. see `dcdr help init` for usage\n", cfg.Git.RepoURL, cfg.Git.RepoPath)
		os.Exit(1)
	}

	var gs *godspeed.Godspeed

	if cfg.StatsEnabled() {
		gs, err = godspeed.New(cfg.Stats.Host, cfg.Stats.Port, false)

		if err != nil {
			printer.SayErr("%v", err)
			os.Exit(1)
		}
	}

	kv := api.New(store, rp, cfg.Namespace, gs)
	ctrl := cli.NewController(cfg, kv)

	dcdr := cli.New(ctrl)
	dcdr.Run()
}
<commit_msg>Fixing default arg checks<commit_after>package main

import (
	"fmt"

	"os"

	"github.com/PagerDuty/godspeed"
	"github.com/vsco/dcdr/cli"
	"github.com/vsco/dcdr/cli/api"
	"github.com/vsco/dcdr/cli/api/stores"
	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/cli/repo"
	"github.com/vsco/dcdr/config"
)

func main() {
	cfg := config.LoadConfig()
	store, err := stores.DefaultConsulStore()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	rp := repo.New(cfg)

	cmd := ""

	if len(os.Args) > 1 {
		cmd = os.Args[1]
	}

	if cmd != "" && cmd != "init" && cmd != "help" && cmd != "watch" && rp.Enabled() && !rp.Exists() {
		printer.SayErr("%s has not been cloned to %s. see `dcdr help init` for usage\n", cfg.Git.RepoURL, cfg.Git.RepoPath)
		os.Exit(1)
	}

	var gs *godspeed.Godspeed

	if cfg.StatsEnabled() {
		gs, err = godspeed.New(cfg.Stats.Host, cfg.Stats.Port, false)

		if err != nil {
			printer.SayErr("%v", err)
			os.Exit(1)
		}
	}

	kv := api.New(store, rp, cfg.Namespace, gs)
	ctrl := cli.NewController(cfg, kv)

	dcdr := cli.New(ctrl)
	dcdr.Run()
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"reflect"

	"github.com/couchbaselabs/blance"
)

// JSON/struct definitions of what the Manager stores in the Cfg.
// NOTE: You *must* update VERSION if you change these
// definitions or the planning algorithms change.

type IndexDefs struct {
	// IndexDefs.UUID changes whenever any child IndexDef changes.
	UUID        string               `json:"uuid"`
	IndexDefs   map[string]*IndexDef `json:"indexDefs"`   // Key is IndexDef.Name.
	ImplVersion string               `json:"implVersion"` // See VERSION.
}

type IndexDef struct {
	Type         string     `json:"type"` // Ex: "bleve", "alias", "blackhole", etc.
	Name         string     `json:"name"`
	UUID         string     `json:"uuid"`
	Params       string     `json:"params"`
	SourceType   string     `json:"sourceType"`
	SourceName   string     `json:"sourceName"`
	SourceUUID   string     `json:"sourceUUID"`
	SourceParams string     `json:"sourceParams"` // Optional connection info.
	PlanParams   PlanParams `json:"planParams"`

	// NOTE: Any auth credentials to access datasource, if any, may be
	// stored as part of SourceParams.
}

type PlanParams struct {
	// MaxPartitionsPerPIndex controls the maximum number of source
	// partitions the planner can assign to or clump into a PIndex (or
	// index partition).
	MaxPartitionsPerPIndex int `json:"maxPartitionsPerPIndex"`

	// NumReplicas controls the number of replicas for a PIndex, over
	// the first copy.  The first copy is not counted as a replica.
	// For example, a NumReplicas setting of 2 means there should be a
	// primary and 2 replicas... so 3 copies in total.  A NumReplicas
	// of 0 means just the first, primary copy only.
	NumReplicas int `json:"numReplicas"`

	// HierarchyRules defines the policy the planner should follow
	// when assigning PIndexes to nodes, especially for replica
	// placement.  Through the HierarchyRules, a user can specify, for
	// example, that the first replica should be not on the same rack
	// and zone as the first copy.  Some examples:
	// Try to put the first replica on the same rack...
	// {"replica":[{"includeLevel":1,"excludeLevel":0}]}
	// Try to put the first replica on a different rack...
	// {"replica":[{"includeLevel":2,"excludeLevel":1}]}
	HierarchyRules blance.HierarchyRules `json:"hierarchyRules"`

	// NodePlanParams allows users to specify per-node input to the
	// planner, such as whether PIndexes assigned to different nodes
	// can be readable or writable.  Keyed by node UUID.  Value is
	// keyed by planPIndex.Name or indexDef.Name.  The empty string
	// ("") is used to represent any node UUID and/or any planPIndex
	// and/or any indexDef.
	NodePlanParams map[string]map[string]*NodePlanParam `json:"nodePlanParams"`

	// PlanFrozen means the planner should not change the previous
	// plan for an index, even if as nodes join or leave and even if
	// there was no previous plan.  Defaults to false (allow
	// re-planning).
	PlanFrozen bool `json:"planFrozen"`
}

type NodePlanParam struct {
	CanRead  bool `json:"canRead"`
	CanWrite bool `json:"canWrite"`
}

// ------------------------------------------------------------------------

type NodeDefs struct {
	// NodeDefs.UUID changes whenever any child NodeDef changes.
	UUID        string              `json:"uuid"`
	NodeDefs    map[string]*NodeDef `json:"nodeDefs"`    // Key is NodeDef.HostPort.
	ImplVersion string              `json:"implVersion"` // See VERSION.
}

type NodeDef struct {
	HostPort    string   `json:"hostPort"`
	UUID        string   `json:"uuid"`
	ImplVersion string   `json:"implVersion"` // See VERSION.
	Tags        []string `json:"tags"`
	Container   string   `json:"container"`
	Weight      int      `json:"weight"`
	Extras      string   `json:"extras"`
}

// ------------------------------------------------------------------------

type PlanPIndexes struct {
	// PlanPIndexes.UUID changes whenever any child PlanPIndex changes.
	UUID         string                 `json:"uuid"`
	PlanPIndexes map[string]*PlanPIndex `json:"planPIndexes"` // Key is PlanPIndex.Name.
	ImplVersion  string                 `json:"implVersion"`  // See VERSION.
	Warnings     map[string][]string    `json:"warnings"`     // Key is IndexDef.Name.
}

type PlanPIndex struct {
	Name             string `json:"name"` // Stable & unique cluster wide.
	UUID             string `json:"uuid"`
	IndexType        string `json:"indexType"`   // See IndexDef.Type.
	IndexName        string `json:"indexName"`   // See IndexDef.Name.
	IndexUUID        string `json:"indexUUID"`   // See IndefDef.UUID.
	IndexParams      string `json:"indexParams"` // See IndexDef.Params.
	SourceType       string `json:"sourceType"`
	SourceName       string `json:"sourceName"`
	SourceUUID       string `json:"sourceUUID"`
	SourceParams     string `json:"sourceParams"` // Optional connection info.
	SourcePartitions string `json:"sourcePartitions"`

	Nodes map[string]*PlanPIndexNode `json:"nodes"` // Keyed by NodeDef.UUID.
}

type PlanPIndexNode struct {
	CanRead  bool `json:"canRead"`
	CanWrite bool `json:"canWrite"`
	Priority int  `json:"priority"`
}

func PlanPIndexNodeCanRead(p *PlanPIndexNode) bool {
	return p != nil && p.CanRead
}

func PlanPIndexNodeCanWrite(p *PlanPIndexNode) bool {
	return p != nil && p.CanWrite
}

func PlanPIndexNodeOk(p *PlanPIndexNode) bool {
	return true
}

// ------------------------------------------------------------------------

const INDEX_DEFS_KEY = "indexDefs"

func NewIndexDefs(version string) *IndexDefs {
	return &IndexDefs{
		UUID:        NewUUID(),
		IndexDefs:   make(map[string]*IndexDef),
		ImplVersion: version,
	}
}

func CfgGetIndexDefs(cfg Cfg) (*IndexDefs, uint64, error) {
	v, cas, err := cfg.Get(INDEX_DEFS_KEY, 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &IndexDefs{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetIndexDefs(cfg Cfg, indexDefs *IndexDefs, cas uint64) (uint64, error) {
	buf, err := json.Marshal(indexDefs)
	if err != nil {
		return 0, err
	}
	return cfg.Set(INDEX_DEFS_KEY, buf, cas)
}

// ------------------------------------------------------------------------

func GetNodePlanParam(nodePlanParams map[string]map[string]*NodePlanParam,
	nodeUUID, indexDefName, planPIndexName string) *NodePlanParam {
	var nodePlanParam *NodePlanParam
	if nodePlanParams != nil {
		m := nodePlanParams[nodeUUID]
		if m == nil {
			m = nodePlanParams[""]
		}
		if m != nil {
			nodePlanParam = m[indexDefName]
			if nodePlanParam == nil {
				nodePlanParam = m[planPIndexName]
			}
			if nodePlanParam == nil {
				nodePlanParam = m[""]
			}
		}
	}
	return nodePlanParam
}

// ------------------------------------------------------------------------

const NODE_DEFS_KEY = "nodeDefs"
const NODE_DEFS_KNOWN = "known"
const NODE_DEFS_WANTED = "wanted"

func NewNodeDefs(version string) *NodeDefs {
	return &NodeDefs{
		UUID:        NewUUID(),
		NodeDefs:    make(map[string]*NodeDef),
		ImplVersion: version,
	}
}

func CfgNodeDefsKey(kind string) string {
	return NODE_DEFS_KEY + "-" + kind
}

func CfgGetNodeDefs(cfg Cfg, kind string) (*NodeDefs, uint64, error) {
	v, cas, err := cfg.Get(CfgNodeDefsKey(kind), 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &NodeDefs{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetNodeDefs(cfg Cfg, kind string, nodeDefs *NodeDefs,
	cas uint64) (uint64, error) {
	buf, err := json.Marshal(nodeDefs)
	if err != nil {
		return 0, err
	}
	return cfg.Set(CfgNodeDefsKey(kind), buf, cas)
}

// ------------------------------------------------------------------------

const PLAN_PINDEXES_KEY = "planPIndexes"

func NewPlanPIndexes(version string) *PlanPIndexes {
	return &PlanPIndexes{
		UUID:         NewUUID(),
		PlanPIndexes: make(map[string]*PlanPIndex),
		ImplVersion:  version,
		Warnings:     make(map[string][]string),
	}
}

func CfgGetPlanPIndexes(cfg Cfg) (*PlanPIndexes, uint64, error) {
	v, cas, err := cfg.Get(PLAN_PINDEXES_KEY, 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &PlanPIndexes{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetPlanPIndexes(cfg Cfg, planPIndexes *PlanPIndexes, cas uint64) (uint64, error) {
	buf, err := json.Marshal(planPIndexes)
	if err != nil {
		return 0, err
	}
	return cfg.Set(PLAN_PINDEXES_KEY, buf, cas)
}

// Returns true if both PlanPIndexes are the same, where we ignore any
// differences in UUID or ImplVersion.
func SamePlanPIndexes(a, b *PlanPIndexes) bool {
	if a == nil || b == nil {
		return a == nil && b == nil
	}
	if len(a.PlanPIndexes) != len(b.PlanPIndexes) {
		return false
	}
	return SubsetPlanPIndexes(a, b) && SubsetPlanPIndexes(b, a)
}

// Returns true if PlanPIndex children in a are a subset of those in
// b, using SamePlanPIndex() for sameness comparion.
func SubsetPlanPIndexes(a, b *PlanPIndexes) bool {
	for name, av := range a.PlanPIndexes {
		bv, exists := b.PlanPIndexes[name]
		if !exists {
			return false
		}
		if !SamePlanPIndex(av, bv) {
			return false
		}
	}
	return true
}

// Returns true if both PlanPIndex are the same, ignoring PlanPIndex.UUID.
func SamePlanPIndex(a, b *PlanPIndex) bool {
	// Of note, we don't compare UUID's.
	if a.Name != b.Name ||
		a.IndexName != b.IndexName ||
		a.IndexUUID != b.IndexUUID ||
		a.IndexParams != b.IndexParams ||
		a.SourceType != b.SourceType ||
		a.SourceName != b.SourceName ||
		a.SourceUUID != b.SourceUUID ||
		a.SourceParams != b.SourceParams ||
		a.SourcePartitions != b.SourcePartitions ||
		!reflect.DeepEqual(a.Nodes, b.Nodes) {
		return false
	}
	return true
}

// Returns true if both the PIndex meets the PlanPIndex, ignoring UUID.
func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexParams == planPIndex.IndexParams &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourceParams == planPIndex.SourceParams &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	return same
}
<commit_msg>comment on PlanPIndexNode.Priority interpretation<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"reflect"

	"github.com/couchbaselabs/blance"
)

// JSON/struct definitions of what the Manager stores in the Cfg.
// NOTE: You *must* update VERSION if you change these
// definitions or the planning algorithms change.

type IndexDefs struct {
	// IndexDefs.UUID changes whenever any child IndexDef changes.
	UUID        string               `json:"uuid"`
	IndexDefs   map[string]*IndexDef `json:"indexDefs"`   // Key is IndexDef.Name.
	ImplVersion string               `json:"implVersion"` // See VERSION.
}

type IndexDef struct {
	Type         string     `json:"type"` // Ex: "bleve", "alias", "blackhole", etc.
	Name         string     `json:"name"`
	UUID         string     `json:"uuid"`
	Params       string     `json:"params"`
	SourceType   string     `json:"sourceType"`
	SourceName   string     `json:"sourceName"`
	SourceUUID   string     `json:"sourceUUID"`
	SourceParams string     `json:"sourceParams"` // Optional connection info.
	PlanParams   PlanParams `json:"planParams"`

	// NOTE: Any auth credentials to access datasource, if any, may be
	// stored as part of SourceParams.
}

type PlanParams struct {
	// MaxPartitionsPerPIndex controls the maximum number of source
	// partitions the planner can assign to or clump into a PIndex (or
	// index partition).
	MaxPartitionsPerPIndex int `json:"maxPartitionsPerPIndex"`

	// NumReplicas controls the number of replicas for a PIndex, over
	// the first copy.  The first copy is not counted as a replica.
	// For example, a NumReplicas setting of 2 means there should be a
	// primary and 2 replicas... so 3 copies in total.  A NumReplicas
	// of 0 means just the first, primary copy only.
	NumReplicas int `json:"numReplicas"`

	// HierarchyRules defines the policy the planner should follow
	// when assigning PIndexes to nodes, especially for replica
	// placement.  Through the HierarchyRules, a user can specify, for
	// example, that the first replica should be not on the same rack
	// and zone as the first copy.  Some examples:
	// Try to put the first replica on the same rack...
	// {"replica":[{"includeLevel":1,"excludeLevel":0}]}
	// Try to put the first replica on a different rack...
	// {"replica":[{"includeLevel":2,"excludeLevel":1}]}
	HierarchyRules blance.HierarchyRules `json:"hierarchyRules"`

	// NodePlanParams allows users to specify per-node input to the
	// planner, such as whether PIndexes assigned to different nodes
	// can be readable or writable.  Keyed by node UUID.  Value is
	// keyed by planPIndex.Name or indexDef.Name.  The empty string
	// ("") is used to represent any node UUID and/or any planPIndex
	// and/or any indexDef.
	NodePlanParams map[string]map[string]*NodePlanParam `json:"nodePlanParams"`

	// PlanFrozen means the planner should not change the previous
	// plan for an index, even if as nodes join or leave and even if
	// there was no previous plan.  Defaults to false (allow
	// re-planning).
	PlanFrozen bool `json:"planFrozen"`
}

type NodePlanParam struct {
	CanRead  bool `json:"canRead"`
	CanWrite bool `json:"canWrite"`
}

// ------------------------------------------------------------------------

type NodeDefs struct {
	// NodeDefs.UUID changes whenever any child NodeDef changes.
	UUID        string              `json:"uuid"`
	NodeDefs    map[string]*NodeDef `json:"nodeDefs"`    // Key is NodeDef.HostPort.
	ImplVersion string              `json:"implVersion"` // See VERSION.
}

type NodeDef struct {
	HostPort    string   `json:"hostPort"`
	UUID        string   `json:"uuid"`
	ImplVersion string   `json:"implVersion"` // See VERSION.
	Tags        []string `json:"tags"`
	Container   string   `json:"container"`
	Weight      int      `json:"weight"`
	Extras      string   `json:"extras"`
}

// ------------------------------------------------------------------------

type PlanPIndexes struct {
	// PlanPIndexes.UUID changes whenever any child PlanPIndex changes.
	UUID         string                 `json:"uuid"`
	PlanPIndexes map[string]*PlanPIndex `json:"planPIndexes"` // Key is PlanPIndex.Name.
	ImplVersion  string                 `json:"implVersion"`  // See VERSION.
	Warnings     map[string][]string    `json:"warnings"`     // Key is IndexDef.Name.
}

type PlanPIndex struct {
	Name             string `json:"name"` // Stable & unique cluster wide.
	UUID             string `json:"uuid"`
	IndexType        string `json:"indexType"`   // See IndexDef.Type.
	IndexName        string `json:"indexName"`   // See IndexDef.Name.
	IndexUUID        string `json:"indexUUID"`   // See IndefDef.UUID.
	IndexParams      string `json:"indexParams"` // See IndexDef.Params.
	SourceType       string `json:"sourceType"`
	SourceName       string `json:"sourceName"`
	SourceUUID       string `json:"sourceUUID"`
	SourceParams     string `json:"sourceParams"` // Optional connection info.
	SourcePartitions string `json:"sourcePartitions"`

	Nodes map[string]*PlanPIndexNode `json:"nodes"` // Keyed by NodeDef.UUID.
}

type PlanPIndexNode struct {
	CanRead  bool `json:"canRead"`
	CanWrite bool `json:"canWrite"`
	Priority int  `json:"priority"` // Lower is higher priority, 0 is highest.
}

func PlanPIndexNodeCanRead(p *PlanPIndexNode) bool {
	return p != nil && p.CanRead
}

func PlanPIndexNodeCanWrite(p *PlanPIndexNode) bool {
	return p != nil && p.CanWrite
}

func PlanPIndexNodeOk(p *PlanPIndexNode) bool {
	return true
}

// ------------------------------------------------------------------------

const INDEX_DEFS_KEY = "indexDefs"

func NewIndexDefs(version string) *IndexDefs {
	return &IndexDefs{
		UUID:        NewUUID(),
		IndexDefs:   make(map[string]*IndexDef),
		ImplVersion: version,
	}
}

func CfgGetIndexDefs(cfg Cfg) (*IndexDefs, uint64, error) {
	v, cas, err := cfg.Get(INDEX_DEFS_KEY, 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &IndexDefs{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetIndexDefs(cfg Cfg, indexDefs *IndexDefs, cas uint64) (uint64, error) {
	buf, err := json.Marshal(indexDefs)
	if err != nil {
		return 0, err
	}
	return cfg.Set(INDEX_DEFS_KEY, buf, cas)
}

// ------------------------------------------------------------------------

func GetNodePlanParam(nodePlanParams map[string]map[string]*NodePlanParam,
	nodeUUID, indexDefName, planPIndexName string) *NodePlanParam {
	var nodePlanParam *NodePlanParam
	if nodePlanParams != nil {
		m := nodePlanParams[nodeUUID]
		if m == nil {
			m = nodePlanParams[""]
		}
		if m != nil {
			nodePlanParam = m[indexDefName]
			if nodePlanParam == nil {
				nodePlanParam = m[planPIndexName]
			}
			if nodePlanParam == nil {
				nodePlanParam = m[""]
			}
		}
	}
	return nodePlanParam
}

// ------------------------------------------------------------------------

const NODE_DEFS_KEY = "nodeDefs"
const NODE_DEFS_KNOWN = "known"
const NODE_DEFS_WANTED = "wanted"

func NewNodeDefs(version string) *NodeDefs {
	return &NodeDefs{
		UUID:        NewUUID(),
		NodeDefs:    make(map[string]*NodeDef),
		ImplVersion: version,
	}
}

func CfgNodeDefsKey(kind string) string {
	return NODE_DEFS_KEY + "-" + kind
}

func CfgGetNodeDefs(cfg Cfg, kind string) (*NodeDefs, uint64, error) {
	v, cas, err := cfg.Get(CfgNodeDefsKey(kind), 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &NodeDefs{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetNodeDefs(cfg Cfg, kind string, nodeDefs *NodeDefs,
	cas uint64) (uint64, error) {
	buf, err := json.Marshal(nodeDefs)
	if err != nil {
		return 0, err
	}
	return cfg.Set(CfgNodeDefsKey(kind), buf, cas)
}

// ------------------------------------------------------------------------

const PLAN_PINDEXES_KEY = "planPIndexes"

func NewPlanPIndexes(version string) *PlanPIndexes {
	return &PlanPIndexes{
		UUID:         NewUUID(),
		PlanPIndexes: make(map[string]*PlanPIndex),
		ImplVersion:  version,
		Warnings:     make(map[string][]string),
	}
}

func CfgGetPlanPIndexes(cfg Cfg) (*PlanPIndexes, uint64, error) {
	v, cas, err := cfg.Get(PLAN_PINDEXES_KEY, 0)
	if err != nil {
		return nil, 0, err
	}
	if v == nil {
		return nil, 0, nil
	}
	rv := &PlanPIndexes{}
	err = json.Unmarshal(v, rv)
	if err != nil {
		return nil, 0, err
	}
	return rv, cas, nil
}

func CfgSetPlanPIndexes(cfg Cfg, planPIndexes *PlanPIndexes, cas uint64) (uint64, error) {
	buf, err := json.Marshal(planPIndexes)
	if err != nil {
		return 0, err
	}
	return cfg.Set(PLAN_PINDEXES_KEY, buf, cas)
}

// Returns true if both PlanPIndexes are the same, where we ignore any
// differences in UUID or ImplVersion.
func SamePlanPIndexes(a, b *PlanPIndexes) bool {
	if a == nil || b == nil {
		return a == nil && b == nil
	}
	if len(a.PlanPIndexes) != len(b.PlanPIndexes) {
		return false
	}
	return SubsetPlanPIndexes(a, b) && SubsetPlanPIndexes(b, a)
}

// Returns true if PlanPIndex children in a are a subset of those in
// b, using SamePlanPIndex() for sameness comparion.
func SubsetPlanPIndexes(a, b *PlanPIndexes) bool {
	for name, av := range a.PlanPIndexes {
		bv, exists := b.PlanPIndexes[name]
		if !exists {
			return false
		}
		if !SamePlanPIndex(av, bv) {
			return false
		}
	}
	return true
}

// Returns true if both PlanPIndex are the same, ignoring PlanPIndex.UUID.
func SamePlanPIndex(a, b *PlanPIndex) bool {
	// Of note, we don't compare UUID's.
	if a.Name != b.Name ||
		a.IndexName != b.IndexName ||
		a.IndexUUID != b.IndexUUID ||
		a.IndexParams != b.IndexParams ||
		a.SourceType != b.SourceType ||
		a.SourceName != b.SourceName ||
		a.SourceUUID != b.SourceUUID ||
		a.SourceParams != b.SourceParams ||
		a.SourcePartitions != b.SourcePartitions ||
		!reflect.DeepEqual(a.Nodes, b.Nodes) {
		return false
	}
	return true
}

// Returns true if both the PIndex meets the PlanPIndex, ignoring UUID.
func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexParams == planPIndex.IndexParams &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourceParams == planPIndex.SourceParams &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	return same
}
<|endoftext|>


<commit_before>// Alexandria
//
// Copyright (C) 2015-2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var NoSuchScrollError = errors.New("No such scroll")

const hashes = "############################################################"

type errTemplateReader struct {
	doc string
	err error
}

// Load a template file from disk and propagate errors
func (e *errTemplateReader) readTemplate(name string) {
	if e.err != nil {
		return
	}

	tmp, err := readTemplate(name)
	e.err = err
	e.doc += tmp
}

// Create a LaTeX file from the content of the given scroll together with all
// the appropriate templates.  The resulting file stored in the temp directory.
func scrollToLatex(id Id) error {
	var e errTemplateReader

	scrollText, err := readScroll(id)
	if err != nil {
		if os.IsNotExist(err) {
			err = RemoveFromIndex(id)
			if err != nil {
				LogError(err)
			}
			return NoSuchScrollError
		}
		LogError(err)
		return err
	}
	scroll := Parse(string(id), scrollText)

	e.readTemplate("header")
	e.readTemplate(scroll.DocumentType() + "_header")
	e.doc += scroll.Content
	e.readTemplate(scroll.DocumentType() + "_footer")
	e.readTemplate("footer")

	if e.err != nil {
		return e.err
	}
	return writeTemp(id, e.doc)
}

// Compile a LaTeX file with the given id to produce a PDF file.  Both input
// and output files are in the temp directory.
func latexToPdf(id Id) error {
	msg, err := exec.Command("xelatex", "-interaction", "nonstopmode",
		"-output-directory", Config.TempDirectory,
		Config.TempDirectory+string(id)).CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
	}
	return err
}

// Convert PDF to PNG, storing the result in the cache directory.  From there,
// it can be served by the web server or displayed to the user via some other
// user interface.
func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-trim",
		"-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi),
		Config.TempDirectory+id+".pdf", Config.CacheDirectory+id+".png").Run()
}

// Generate a PNG image from a given scroll, if there is no up-to-date image.
func renderScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	err := scrollToLatex(id)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	return pdfToPng(id)
}

func RenderListOfScrolls(ids []Scroll) int {
	numScrolls := 0

	for _, foo := range ids {
		id := foo.Id
		err := renderScroll(id)
		if err == NoSuchScrollError {
			continue
		} else if err != nil {
			log.Panic("An error ocurred when processing scroll ", id, ": ", err)
		} else {
			numScrolls += 1
		}
	}

	return numScrolls
}

func RenderAllScrolls() int {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		panic(err)
	}
	var errors []error
	limitGoroutines := make(chan bool, Config.MaxProcs)
	for i := 0; i < Config.MaxProcs; i++ {
		limitGoroutines <- true
	}
	ch := make(chan int, len(files))
	for _, file := range files {
		go func(file os.FileInfo) {
			<-limitGoroutines
			if !strings.HasSuffix(file.Name(), ".tex") {
				ch <- 0
				return
			}
			id := Id(strings.TrimSuffix(file.Name(), ".tex"))
			if err := renderScroll(id); err != nil && err != NoSuchScrollError {
				log.Printf("%s\nERROR\n%s\n%v\n%s\n", hashes, hashes, err, hashes)
			}
			ch <- 1
		}(file)
	}
	counter := 0
	for i := 0; i < len(files); i++ {
		counter += <-ch
		limitGoroutines <- true
	}
	for _, err := range errors {
		log.Printf("Error: %v\n", err)
	}
	return counter
}
<commit_msg>Reorganize conditionals<commit_after>// Alexandria
//
// Copyright (C) 2015-2016  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var NoSuchScrollError = errors.New("No such scroll")

const hashes = "############################################################"

type errTemplateReader struct {
	doc string
	err error
}

// Load a template file from disk and propagate errors
func (e *errTemplateReader) readTemplate(name string) {
	if e.err != nil {
		return
	}

	tmp, err := readTemplate(name)
	e.err = err
	e.doc += tmp
}

// Create a LaTeX file from the content of the given scroll together with all
// the appropriate templates.  The resulting file stored in the temp directory.
func scrollToLatex(id Id) error {
	var e errTemplateReader

	scrollText, err := readScroll(id)
	if err != nil {
		if os.IsNotExist(err) {
			err = RemoveFromIndex(id)
			if err != nil {
				LogError(err)
			}
			return NoSuchScrollError
		}
		LogError(err)
		return err
	}
	scroll := Parse(string(id), scrollText)

	e.readTemplate("header")
	e.readTemplate(scroll.DocumentType() + "_header")
	e.doc += scroll.Content
	e.readTemplate(scroll.DocumentType() + "_footer")
	e.readTemplate("footer")

	if e.err != nil {
		return e.err
	}
	return writeTemp(id, e.doc)
}

// Compile a LaTeX file with the given id to produce a PDF file.  Both input
// and output files are in the temp directory.
func latexToPdf(id Id) error {
	msg, err := exec.Command("xelatex", "-interaction", "nonstopmode",
		"-output-directory", Config.TempDirectory,
		Config.TempDirectory+string(id)).CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
	}
	return err
}

// Convert PDF to PNG, storing the result in the cache directory.  From there,
// it can be served by the web server or displayed to the user via some other
// user interface.
func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-trim",
		"-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi),
		Config.TempDirectory+id+".pdf", Config.CacheDirectory+id+".png").Run()
}

// Generate a PNG image from a given scroll, if there is no up-to-date image.
func renderScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	err := scrollToLatex(id)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	return pdfToPng(id)
}

func RenderListOfScrolls(ids []Scroll) int {
	numScrolls := 0

	for _, foo := range ids {
		id := foo.Id
		err := renderScroll(id)
		if err != nil {
			if err == NoSuchScrollError {
				continue
			} else {
				log.Panic("An error ocurred when processing scroll ", id, ": ", err)
			}
		} else {
			numScrolls += 1
		}
	}

	return numScrolls
}

func RenderAllScrolls() int {
	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		panic(err)
	}
	var errors []error
	limitGoroutines := make(chan bool, Config.MaxProcs)
	for i := 0; i < Config.MaxProcs; i++ {
		limitGoroutines <- true
	}
	ch := make(chan int, len(files))
	for _, file := range files {
		go func(file os.FileInfo) {
			<-limitGoroutines
			if !strings.HasSuffix(file.Name(), ".tex") {
				ch <- 0
				return
			}
			id := Id(strings.TrimSuffix(file.Name(), ".tex"))
			if err := renderScroll(id); err != nil && err != NoSuchScrollError {
				log.Printf("%s\nERROR\n%s\n%v\n%s\n", hashes, hashes, err, hashes)
			}
			ch <- 1
		}(file)
	}
	counter := 0
	for i := 0; i < len(files); i++ {
		counter += <-ch
		limitGoroutines <- true
	}
	for _, err := range errors {
		log.Printf("Error: %v\n", err)
	}
	return counter
}
<|endoftext|>


<commit_before>package googlesearch
import (
	"net/http/cookiejar"
	"net/http"
	"github.com/PuerkitoBio/goquery"
	"fmt"
	"net/url"
	"time"
)

type Result struct {
	Link string
	Name string
}

type pagedSearch struct {
	current_page int
	query        string
	tld          string
	hl           string
	http_client  http.Client
	results      chan Result
}

var (
	url_search = "https://www.google.%s/search?q=%s&num=%d&start=%d&safe=off&hl=%s&filter=0"
	nb_results_per_page = 100
	searchLock = make(chan bool)
)

func Search(query string, tld string, hl string) chan Result {

	jar, err := cookiejar.New(nil)
	if err != nil {
		panic(err)
	}
	client := http.Client{Jar : jar}
	chanResult := make(chan Result, nb_results_per_page)

	pagedSearch := pagedSearch{current_page: 0, query: query, tld: tld, hl:hl, http_client: client, results: chanResult}

	go doPagedSearch(&pagedSearch)
	searchLock <- false

	return chanResult
}

func doPagedSearch(paged_search *pagedSearch) {
	for {
		<-searchLock
		go releaseSearchLock(searchLock)

		url_str := fmt.Sprintf(url_search,
			paged_search.tld,
			url.QueryEscape(paged_search.query),
			nb_results_per_page,
			paged_search.current_page * nb_results_per_page,
			paged_search.hl)

		req, _ := http.NewRequest("GET", url_str, nil)
		req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)")
		resp, err := paged_search.http_client.Do(req)
		if err != nil {
			panic(err)
		}

		doc, err := goquery.NewDocumentFromResponse(resp)
		if err != nil {
			panic(err)
		}

		resp.Body.Close()

		entries := doc.Find("h3 a")

		for _, res_dom := range entries.Nodes {
			rawUrl := res_dom.Attr[0].Val
			parsedUrl, parseUrlErr := url.Parse(rawUrl)
			if parseUrlErr != nil {
				continue
			}
			parsed_query, parse_query_err := url.ParseQuery(parsedUrl.RawQuery)
			if parse_query_err != nil {
				continue
			}
			if parsed_query["q"] == nil {
				continue
			}
			res := Result{Link:parsed_query["q"][0]}
			paged_search.results <- res
		}

		if doc.Find("#nav").Size() == 0 {
			//no navigation bar - one page result
			break
		}
		if doc.Find("#nav .b a").Size() == 1 {
			//either previous or next
			if paged_search.current_page != 0 {
				//no next button
				break;
			}
		}

		paged_search.current_page = paged_search.current_page + 1;
	}
	close(paged_search.results)
}

func releaseSearchLock(t chan bool) {
	//only one search every n seconds
	time.Sleep(time.Second * 3)
	t <- false
}
<commit_msg>godoc compliant<commit_after>/*
Google search from go
*/
package googlesearch
import (
	"net/http/cookiejar"
	"net/http"
	"github.com/PuerkitoBio/goquery"
	"fmt"
	"net/url"
	"time"
)

type Result struct {
	Link string
	Name string
}

type pagedSearch struct {
	current_page int
	query        string
	tld          string
	hl           string
	http_client  http.Client
	results      chan Result
}

var (
	url_search = "https://www.google.%s/search?q=%s&num=%d&start=%d&safe=off&hl=%s&filter=0"
	nb_results_per_page = 100
	searchLock = make(chan bool)
)

func Search(query string, tld string, hl string) chan Result {

	jar, err := cookiejar.New(nil)
	if err != nil {
		panic(err)
	}
	client := http.Client{Jar : jar}
	chanResult := make(chan Result, nb_results_per_page)

	pagedSearch := pagedSearch{current_page: 0, query: query, tld: tld, hl:hl, http_client: client, results: chanResult}

	go doPagedSearch(&pagedSearch)
	searchLock <- false

	return chanResult
}

func doPagedSearch(paged_search *pagedSearch) {
	for {
		<-searchLock
		go releaseSearchLock(searchLock)

		url_str := fmt.Sprintf(url_search,
			paged_search.tld,
			url.QueryEscape(paged_search.query),
			nb_results_per_page,
			paged_search.current_page * nb_results_per_page,
			paged_search.hl)

		req, _ := http.NewRequest("GET", url_str, nil)
		req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)")
		resp, err := paged_search.http_client.Do(req)
		if err != nil {
			panic(err)
		}

		doc, err := goquery.NewDocumentFromResponse(resp)
		if err != nil {
			panic(err)
		}

		resp.Body.Close()

		entries := doc.Find("h3 a")

		for _, res_dom := range entries.Nodes {
			rawUrl := res_dom.Attr[0].Val
			parsedUrl, parseUrlErr := url.Parse(rawUrl)
			if parseUrlErr != nil {
				continue
			}
			parsed_query, parse_query_err := url.ParseQuery(parsedUrl.RawQuery)
			if parse_query_err != nil {
				continue
			}
			if parsed_query["q"] == nil {
				continue
			}
			res := Result{Link:parsed_query["q"][0]}
			paged_search.results <- res
		}

		if doc.Find("#nav").Size() == 0 {
			//no navigation bar - one page result
			break
		}
		if doc.Find("#nav .b a").Size() == 1 {
			//either previous or next
			if paged_search.current_page != 0 {
				//no next button
				break;
			}
		}

		paged_search.current_page = paged_search.current_page + 1;
	}
	close(paged_search.results)
}

func releaseSearchLock(t chan bool) {
	//only one search every n seconds
	time.Sleep(time.Second * 3)
	t <- false
}
<|endoftext|>


<commit_before>package hydrator

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	etcd "github.com/coreos/etcd/client"
	"github.com/mcuadros/go-defaults"
)

var (
	DefaultSeparator    = "."
	Debug               = false
	DebugEnvironmentVar = "ETCD_HYDRATOR_DEBUG"
)

func init() {
	if len(os.Getenv(DebugEnvironmentVar)) != 0 {
		Debug = true
	}
}

type Hydrator struct {
	client    etcd.Client
	kAPI      etcd.KeysAPI
	filler    *defaults.Filler
	Folder    string
	Separator string
}

func NewHydrator(client etcd.Client) *Hydrator {
	h := &Hydrator{
		client:    client,
		kAPI:      etcd.NewKeysAPI(client),
		Separator: DefaultSeparator,
	}

	funcs := make(map[reflect.Kind]defaults.FillerFunc, 0)

	funcs[reflect.Bool] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseBool(raw)
		field.Value.SetBool(value)
	}

	funcs[reflect.String] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		field.Value.SetString(raw)
	}

	funcs[reflect.Int] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseInt(raw, 10, 64)
		field.Value.SetInt(value)
	}

	funcs[reflect.Int8] = funcs[reflect.Int]
	funcs[reflect.Int16] = funcs[reflect.Int]
	funcs[reflect.Int32] = funcs[reflect.Int]
	funcs[reflect.Int64] = funcs[reflect.Int]

	funcs[reflect.Float32] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseFloat(raw, 64)
		field.Value.SetFloat(value)
	}

	funcs[reflect.Float64] = funcs[reflect.Float32]

	funcs[reflect.Uint] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseUint(raw, 10, 64)
		field.Value.SetUint(value)
	}

	funcs[reflect.Uint8] = funcs[reflect.Uint]
	funcs[reflect.Uint16] = funcs[reflect.Uint]
	funcs[reflect.Uint32] = funcs[reflect.Uint]
	funcs[reflect.Uint64] = funcs[reflect.Uint]

	funcs[reflect.Slice] = func(field *defaults.FieldData) {
		if field.Value.Type().Elem().Kind() == reflect.Uint8 {
			if field.Value.Bytes() != nil {
				return
			}

			raw := h.getKey(field)
			if raw == "" {
				return
			}

			field.Value.SetBytes([]byte(raw))
		}
	}

	funcs[reflect.Struct] = func(field *defaults.FieldData) {
		fields := h.filler.GetFieldsFromValue(field.Value, field)
		h.filler.SetDefaultValues(fields)
	}

	h.filler = &defaults.Filler{FuncByKind: funcs, Tag: "etcd"}

	return h
}

func (h *Hydrator) Hydrate(variable interface{}) {
	if Debug {
		fmt.Printf("Hydrating var: %q\n", reflect.TypeOf(variable).String())
	}

	h.filler.Fill(variable)
}

func (h *Hydrator) getKey(field *defaults.FieldData) string {
	key := h.buildKey(field)

	response, err := h.kAPI.Get(context.Background(), key, nil)
	if err != nil {
		if eerr, ok := err.(etcd.Error); ok && eerr.Code == etcd.ErrorCodeKeyNotFound {
			if Debug {
				fmt.Printf("Missing key %q\n", key)
			}

			return ""
		}

		panic(err)
	}

	value := response.Node.Value

	if Debug {
		fmt.Printf("Recovered key %q with value %q\n", key, value)
	}

	return value
}

func (h *Hydrator) buildKey(field *defaults.FieldData) string {
	if h.Folder == "" {
		return h.buildFilename(field)
	}

	return fmt.Sprintf("%s/%s", h.Folder, h.buildFilename(field))
}

func (h *Hydrator) buildFilename(field *defaults.FieldData) string {
	key := field.TagValue
	if key == "" {
		key = strings.ToLower(field.Field.Name)
	}

	if field.Parent == nil {
		return key
	}

	return h.buildFilename(field.Parent) + h.Separator + key
}
<commit_msg>removing panic at not found<commit_after>package hydrator

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	etcd "github.com/coreos/etcd/client"
	"github.com/mcuadros/go-defaults"
)

var (
	DefaultSeparator    = "."
	Debug               = false
	DebugEnvironmentVar = "ETCD_HYDRATOR_DEBUG"
)

func init() {
	if len(os.Getenv(DebugEnvironmentVar)) != 0 {
		Debug = true
	}
}

type Hydrator struct {
	client    etcd.Client
	kAPI      etcd.KeysAPI
	filler    *defaults.Filler
	Folder    string
	Separator string
}

func NewHydrator(client etcd.Client) *Hydrator {
	h := &Hydrator{
		client:    client,
		kAPI:      etcd.NewKeysAPI(client),
		Separator: DefaultSeparator,
	}

	funcs := make(map[reflect.Kind]defaults.FillerFunc, 0)

	funcs[reflect.Bool] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseBool(raw)
		field.Value.SetBool(value)
	}

	funcs[reflect.String] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		field.Value.SetString(raw)
	}

	funcs[reflect.Int] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseInt(raw, 10, 64)
		field.Value.SetInt(value)
	}

	funcs[reflect.Int8] = funcs[reflect.Int]
	funcs[reflect.Int16] = funcs[reflect.Int]
	funcs[reflect.Int32] = funcs[reflect.Int]
	funcs[reflect.Int64] = funcs[reflect.Int]

	funcs[reflect.Float32] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseFloat(raw, 64)
		field.Value.SetFloat(value)
	}

	funcs[reflect.Float64] = funcs[reflect.Float32]

	funcs[reflect.Uint] = func(field *defaults.FieldData) {
		raw := h.getKey(field)
		if raw == "" {
			return
		}

		value, _ := strconv.ParseUint(raw, 10, 64)
		field.Value.SetUint(value)
	}

	funcs[reflect.Uint8] = funcs[reflect.Uint]
	funcs[reflect.Uint16] = funcs[reflect.Uint]
	funcs[reflect.Uint32] = funcs[reflect.Uint]
	funcs[reflect.Uint64] = funcs[reflect.Uint]

	funcs[reflect.Slice] = func(field *defaults.FieldData) {
		if field.Value.Type().Elem().Kind() == reflect.Uint8 {
			if field.Value.Bytes() != nil {
				return
			}

			raw := h.getKey(field)
			if raw == "" {
				return
			}

			field.Value.SetBytes([]byte(raw))
		}
	}

	funcs[reflect.Struct] = func(field *defaults.FieldData) {
		fields := h.filler.GetFieldsFromValue(field.Value, field)
		h.filler.SetDefaultValues(fields)
	}

	h.filler = &defaults.Filler{FuncByKind: funcs, Tag: "etcd"}

	return h
}

func (h *Hydrator) Hydrate(variable interface{}) {
	if Debug {
		fmt.Printf("Hydrating var: %q\n", reflect.TypeOf(variable).String())
	}

	h.filler.Fill(variable)
}

func (h *Hydrator) getKey(field *defaults.FieldData) string {
	key := h.buildKey(field)

	response, err := h.kAPI.Get(context.Background(), key, nil)
	if err != nil {
		if eerr, ok := err.(etcd.Error); ok && eerr.Code == etcd.ErrorCodeKeyNotFound {
			if Debug {
				fmt.Printf("Missing key %q\n", key)
			}

			return ""
		}
	}

	value := response.Node.Value

	if Debug {
		fmt.Printf("Recovered key %q with value %q\n", key, value)
	}

	return value
}

func (h *Hydrator) buildKey(field *defaults.FieldData) string {
	if h.Folder == "" {
		return h.buildFilename(field)
	}

	return fmt.Sprintf("%s/%s", h.Folder, h.buildFilename(field))
}

func (h *Hydrator) buildFilename(field *defaults.FieldData) string {
	key := field.TagValue
	if key == "" {
		key = strings.ToLower(field.Field.Name)
	}

	if field.Parent == nil {
		return key
	}

	return h.buildFilename(field.Parent) + h.Separator + key
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/coreos/go-etcd/etcd"
)

func EtcdServicePath(serviceName string) string {
	return fmt.Sprintf("/sarpa/%s", serviceName)
}

// TODO: For client use.
func EtcdInsertHost(client *etcd.Client, serviceName, host string) bool {
	// Create a directory in /sarpa/:service_name
	// Create a random child in the directory with key=host value=ip. Set ttl=30seconds
	// On quit kill the child if possible
	return false
}
<commit_msg>Update the sarpa client<commit_after>package main

import (
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"log"
)

func EtcdServicePath(serviceName string) string {
	return fmt.Sprintf("/sarpa/%s", serviceName)
}

// TODO: For client use.
func SarpaUpdater(client *etcd.Client, serviceName, host string, ttl int) bool {
	servicePath := EtcdServicePath(serviceName)

	if _, err := client.Get(servicePath, false, false); err != nil {
		// Directory doesn't exist.

		_, err := client.CreateDir(servicePath, 300)
		if err != nil {
			log.Println(err)
		}
	}

	// Create a random child in the directory with value=host. Set ttl=30seconds
	_, err := client.AddChild(servicePath, host, 30)
	if err != nil {
		log.Println(err)
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/web"
	"github.com/coreos/go-raft"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	//"strconv"
	"strings"
	"time"
)

//------------------------------------------------------------------------------
//
// Initialization
//
//------------------------------------------------------------------------------

var verbose bool

var cluster string

var address string
var clientPort int
var serverPort int
var webPort int

var serverCertFile string
var serverKeyFile string
var serverCAFile string

var clientCertFile string
var clientKeyFile string
var clientCAFile string

var dirPath string

var ignore bool

var maxSize int

func init() {
	flag.BoolVar(&verbose, "v", false, "verbose logging")

	flag.StringVar(&cluster, "C", "", "the ip address and port of a existing cluster")

	flag.StringVar(&address, "a", "", "the ip address of the local machine")
	flag.IntVar(&clientPort, "c", 4001, "the port to communicate with clients")
	flag.IntVar(&serverPort, "s", 7001, "the port to communicate with servers")
	flag.IntVar(&webPort, "w", -1, "the port of web interface")

	flag.StringVar(&serverCAFile, "serverCAFile", "", "the path of the CAFile")
	flag.StringVar(&serverCertFile, "serverCert", "", "the cert file of the server")
	flag.StringVar(&serverKeyFile, "serverKey", "", "the key file of the server")

	flag.StringVar(&clientCAFile, "clientCAFile", "", "the path of the client CAFile")
	flag.StringVar(&clientCertFile, "clientCert", "", "the cert file of the client")
	flag.StringVar(&clientKeyFile, "clientKey", "", "the key file of the client")

	flag.StringVar(&dirPath, "d", "./", "the directory to store log and snapshot")

	flag.BoolVar(&ignore, "i", false, "ignore the old configuration, create a new node")

	flag.IntVar(&maxSize, "m", 1024, "the max size of result buffer")
}

// CONSTANTS
const (
	HTTP = iota
	HTTPS
	HTTPSANDVERIFY
)

const (
	SERVER = iota
	CLIENT
)

const (
	ELECTIONTIMTOUT  = 200 * time.Millisecond
	HEARTBEATTIMEOUT = 50 * time.Millisecond
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

type Info struct {
	Address    string `json:"address"`
	ServerPort int    `json:"serverPort"`
	ClientPort int    `json:"clientPort"`
	WebPort    int    `json:"webPort"`
}

//------------------------------------------------------------------------------
//
// Variables
//
//------------------------------------------------------------------------------

var server *raft.Server
var serverTransHandler transHandler
var logger *log.Logger

var storeMsg chan string

//------------------------------------------------------------------------------
//
// Functions
//
//------------------------------------------------------------------------------

//--------------------------------------
// Main
//--------------------------------------

func main() {
	var err error
	logger = log.New(os.Stdout, "", log.LstdFlags)
	flag.Parse()

	// Setup commands.
	raft.RegisterCommand(&JoinCommand{})
	raft.RegisterCommand(&SetCommand{})
	raft.RegisterCommand(&GetCommand{})
	raft.RegisterCommand(&DeleteCommand{})

	if err := os.MkdirAll(dirPath, 0744); err != nil {
		fatal("Unable to create path: %v", err)
	}

	// Read server info from file or grab it from user.
	var info *Info = getInfo(dirPath)

	name := fmt.Sprintf("%s:%d", info.Address, info.ServerPort)

	fmt.Printf("ServerName: %s\n\n", name)

	// secrity type
	st := securityType(SERVER)

	if st == -1 {
		panic("ERROR type")
	}

	serverTransHandler = createTranHandler(st)

	// Setup new raft server.
	s := store.CreateStore(maxSize)

	// create raft server
	server, err = raft.NewServer(name, dirPath, serverTransHandler, s, nil)

	if err != nil {
		fatal("%v", err)
	}

	err = server.LoadSnapshot()

	if err == nil {
		debug("%s finished load snapshot", server.Name())
	} else {
		fmt.Println(err)
		debug("%s bad snapshot", server.Name())
	}
	server.Initialize()
	debug("%s finished init", server.Name())
	server.SetElectionTimeout(ELECTIONTIMTOUT)
	server.SetHeartbeatTimeout(HEARTBEATTIMEOUT)
	debug("%s finished set timeout", server.Name())

	if server.IsLogEmpty() {

		// start as a leader in a new cluster
		if cluster == "" {
			server.StartLeader()

			time.Sleep(time.Millisecond * 20)

			// join self as a peer
			for {
				command := &JoinCommand{}
				command.Name = server.Name()
				_, err := server.Do(command)
				if err == nil {
					break
				}
			}
			debug("%s start as a leader", server.Name())

			// start as a fellower in a existing cluster
		} else {
			server.StartFollower()

			err := Join(server, cluster)
			if err != nil {
				panic(err)
			}
			fmt.Println("success join")
		}

		// rejoin the previous cluster
	} else {
		server.StartFollower()
		debug("%s start as a follower", server.Name())
	}

	// open the snapshot
	//go server.Snapshot()

	if webPort != -1 {
		// start web
		s.SetMessager(&storeMsg)
		go webHelper()
		go web.Start(server, webPort)
	}

	go startServTransport(info.ServerPort, st)
	startClientTransport(info.ClientPort, securityType(CLIENT))

}

func usage() {
	fatal("usage: raftd [PATH]")
}

func createTranHandler(st int) transHandler {
	t := transHandler{}

	switch st {
	case HTTP:
		t := transHandler{}
		t.client = nil
		return t

	case HTTPS:
		fallthrough
	case HTTPSANDVERIFY:
		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)

		if err != nil {
			panic(err)
		}

		tr := &http.Transport{
			TLSClientConfig: &tls.Config{
				Certificates:       []tls.Certificate{tlsCert},
				InsecureSkipVerify: true,
			},
			DisableCompression: true,
		}

		t.client = &http.Client{Transport: tr}
		return t
	}

	// for complier
	return transHandler{}
}

func startServTransport(port int, st int) {

	// internal commands
	http.HandleFunc("/join", JoinHttpHandler)
	http.HandleFunc("/vote", VoteHttpHandler)
	http.HandleFunc("/log", GetLogHttpHandler)
	http.HandleFunc("/log/append", AppendEntriesHttpHandler)
	http.HandleFunc("/snapshot", SnapshotHttpHandler)
	http.HandleFunc("/client", clientHttpHandler)

	switch st {

	case HTTP:
		debug("raft server [%s] listen on http", server.Name())
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))

	case HTTPS:
		http.ListenAndServeTLS(fmt.Sprintf(":%d", port), serverCertFile, serverKeyFile, nil)

	case HTTPSANDVERIFY:
		pemByte, _ := ioutil.ReadFile(serverCAFile)

		block, pemByte := pem.Decode(pemByte)

		cert, err := x509.ParseCertificate(block.Bytes)

		if err != nil {
			fmt.Println(err)
		}

		certPool := x509.NewCertPool()

		certPool.AddCert(cert)

		server := &http.Server{
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  certPool,
			},
			Addr: fmt.Sprintf(":%d", port),
		}
		err = server.ListenAndServeTLS(serverCertFile, serverKeyFile)

		if err != nil {
			log.Fatal(err)
		}
	}

}

func startClientTransport(port int, st int) {
	// external commands
	http.HandleFunc("/v1/keys/", Multiplexer)
	http.HandleFunc("/v1/watch/", WatchHttpHandler)
	http.HandleFunc("/v1/list/", ListHttpHandler)
	http.HandleFunc("/v1/testAndSet/", TestAndSetHttpHandler)
	http.HandleFunc("/master", MasterHttpHandler)

	switch st {

	case HTTP:
		debug("etcd [%s] listen on http", server.Name())
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))

	case HTTPS:
		http.ListenAndServeTLS(fmt.Sprintf(":%d", port), clientCertFile, clientKeyFile, nil)

	case HTTPSANDVERIFY:
		pemByte, _ := ioutil.ReadFile(clientCAFile)

		block, pemByte := pem.Decode(pemByte)

		cert, err := x509.ParseCertificate(block.Bytes)

		if err != nil {
			fmt.Println(err)
		}

		certPool := x509.NewCertPool()

		certPool.AddCert(cert)

		server := &http.Server{
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  certPool,
			},
			Addr: fmt.Sprintf(":%d", port),
		}
		err = server.ListenAndServeTLS(clientCertFile, clientKeyFile)

		if err != nil {
			log.Fatal(err)
		}
	}
}

//--------------------------------------
// Config
//--------------------------------------

func securityType(source int) int {

	var keyFile, certFile, CAFile string

	switch source {
	case SERVER:
		keyFile = serverKeyFile
		certFile = serverCertFile
		CAFile = serverCAFile

	case CLIENT:
		keyFile = clientKeyFile
		certFile = clientCertFile
		CAFile = clientCAFile
	}

	if keyFile == "" && certFile == "" && CAFile == "" {

		return HTTP

	}

	if keyFile != "" && certFile != "" {

		if CAFile != "" {
			return HTTPSANDVERIFY
		}

		return HTTPS
	}

	return -1
}

func getInfo(path string) *Info {
	info := &Info{}

	// Read in the server info if available.
	infoPath := fmt.Sprintf("%s/info", path)

	// delete the old configuration if exist
	if ignore {
		logPath := fmt.Sprintf("%s/log", path)
		snapshotPath := fmt.Sprintf("%s/snapshotPath", path)
		os.Remove(infoPath)
		os.Remove(logPath)
		os.RemoveAll(snapshotPath)

	}

	if file, err := os.Open(infoPath); err == nil {
		if content, err := ioutil.ReadAll(file); err != nil {
			fatal("Unable to read info: %v", err)
		} else {
			if err = json.Unmarshal(content, &info); err != nil {
				fatal("Unable to parse info: %v", err)
			}
		}
		file.Close()

		// Otherwise ask user for info and write it to file.
	} else {

		if address == "" {
			fatal("Please give the address of the local machine")
		}

		info.Address = address
		info.Address = strings.TrimSpace(info.Address)
		fmt.Println("address ", info.Address)

		info.ServerPort = serverPort
		info.ClientPort = clientPort
		info.WebPort = webPort

		// Write to file.
		content, _ := json.Marshal(info)
		content = []byte(string(content) + "\n")
		if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
			fatal("Unable to write info to file: %v", err)
		}
	}

	return info
}

//--------------------------------------
// Handlers
//--------------------------------------

// Send join requests to the leader.
func Join(s *raft.Server, serverName string) error {
	var b bytes.Buffer

	command := &JoinCommand{}
	command.Name = s.Name()

	json.NewEncoder(&b).Encode(command)

	// t must be ok
	t, _ := server.Transporter().(transHandler)
	debug("Send Join Request to %s", serverName)
	resp, err := Post(&t, fmt.Sprintf("%s/join", serverName), &b)

	for {
		if resp != nil {
			defer resp.Body.Close()
			if resp.StatusCode == http.StatusOK {
				return nil
			}
			if resp.StatusCode == http.StatusServiceUnavailable {
				address, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					warn("Cannot Read Leader info: %v", err)
				}
				debug("Leader is %s", address)
				debug("Send Join Request to %s", address)
				json.NewEncoder(&b).Encode(command)
				resp, err = Post(&t, fmt.Sprintf("%s/join", address), &b)
			}
		}
	}
	return fmt.Errorf("Unable to join: %v", err)
}
<commit_msg>register new commands<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/web"
	"github.com/coreos/go-raft"
	//"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	//"strconv"
	"strings"
	"time"
)

//------------------------------------------------------------------------------
//
// Initialization
//
//------------------------------------------------------------------------------

var verbose bool

var cluster string

var address string
var clientPort int
var serverPort int
var webPort int

var serverCertFile string
var serverKeyFile string
var serverCAFile string

var clientCertFile string
var clientKeyFile string
var clientCAFile string

var dirPath string

var ignore bool

var maxSize int

func init() {
	flag.BoolVar(&verbose, "v", false, "verbose logging")

	flag.StringVar(&cluster, "C", "", "the ip address and port of a existing cluster")

	flag.StringVar(&address, "a", "", "the ip address of the local machine")
	flag.IntVar(&clientPort, "c", 4001, "the port to communicate with clients")
	flag.IntVar(&serverPort, "s", 7001, "the port to communicate with servers")
	flag.IntVar(&webPort, "w", -1, "the port of web interface")

	flag.StringVar(&serverCAFile, "serverCAFile", "", "the path of the CAFile")
	flag.StringVar(&serverCertFile, "serverCert", "", "the cert file of the server")
	flag.StringVar(&serverKeyFile, "serverKey", "", "the key file of the server")

	flag.StringVar(&clientCAFile, "clientCAFile", "", "the path of the client CAFile")
	flag.StringVar(&clientCertFile, "clientCert", "", "the cert file of the client")
	flag.StringVar(&clientKeyFile, "clientKey", "", "the key file of the client")

	flag.StringVar(&dirPath, "d", "./", "the directory to store log and snapshot")

	flag.BoolVar(&ignore, "i", false, "ignore the old configuration, create a new node")

	flag.IntVar(&maxSize, "m", 1024, "the max size of result buffer")
}

// CONSTANTS
const (
	HTTP = iota
	HTTPS
	HTTPSANDVERIFY
)

const (
	SERVER = iota
	CLIENT
)

const (
	ELECTIONTIMTOUT  = 200 * time.Millisecond
	HEARTBEATTIMEOUT = 50 * time.Millisecond
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

type Info struct {
	Address    string `json:"address"`
	ServerPort int    `json:"serverPort"`
	ClientPort int    `json:"clientPort"`
	WebPort    int    `json:"webPort"`
}

//------------------------------------------------------------------------------
//
// Variables
//
//------------------------------------------------------------------------------

var server *raft.Server
var serverTransHandler transHandler
var logger *log.Logger

var storeMsg chan string

//------------------------------------------------------------------------------
//
// Functions
//
//------------------------------------------------------------------------------

//--------------------------------------
// Main
//--------------------------------------

func main() {
	var err error
	logger = log.New(os.Stdout, "", log.LstdFlags)
	flag.Parse()

	// Setup commands.
	raft.RegisterCommand(&JoinCommand{})
	raft.RegisterCommand(&SetCommand{})
	raft.RegisterCommand(&GetCommand{})
	raft.RegisterCommand(&DeleteCommand{})
	raft.RegisterCommand(&WatchCommand{})
	raft.RegisterCommand(&ListCommand{})
	raft.RegisterCommand(&TestAndSetCommand{})

	if err := os.MkdirAll(dirPath, 0744); err != nil {
		fatal("Unable to create path: %v", err)
	}

	// Read server info from file or grab it from user.
	var info *Info = getInfo(dirPath)

	name := fmt.Sprintf("%s:%d", info.Address, info.ServerPort)

	fmt.Printf("ServerName: %s\n\n", name)

	// secrity type
	st := securityType(SERVER)

	if st == -1 {
		panic("ERROR type")
	}

	serverTransHandler = createTranHandler(st)

	// Setup new raft server.
	s := store.CreateStore(maxSize)

	// create raft server
	server, err = raft.NewServer(name, dirPath, serverTransHandler, s, nil)

	if err != nil {
		fatal("%v", err)
	}

	err = server.LoadSnapshot()

	if err == nil {
		debug("%s finished load snapshot", server.Name())
	} else {
		fmt.Println(err)
		debug("%s bad snapshot", server.Name())
	}
	server.Initialize()
	debug("%s finished init", server.Name())
	server.SetElectionTimeout(ELECTIONTIMTOUT)
	server.SetHeartbeatTimeout(HEARTBEATTIMEOUT)
	debug("%s finished set timeout", server.Name())

	if server.IsLogEmpty() {

		// start as a leader in a new cluster
		if cluster == "" {
			server.StartLeader()

			time.Sleep(time.Millisecond * 20)

			// join self as a peer
			for {
				command := &JoinCommand{}
				command.Name = server.Name()
				_, err := server.Do(command)
				if err == nil {
					break
				}
			}
			debug("%s start as a leader", server.Name())

			// start as a fellower in a existing cluster
		} else {
			server.StartFollower()

			err := Join(server, cluster)
			if err != nil {
				panic(err)
			}
			fmt.Println("success join")
		}

		// rejoin the previous cluster
	} else {
		server.StartFollower()
		debug("%s start as a follower", server.Name())
	}

	// open the snapshot
	//go server.Snapshot()

	if webPort != -1 {
		// start web
		s.SetMessager(&storeMsg)
		go webHelper()
		go web.Start(server, webPort)
	}

	go startServTransport(info.ServerPort, st)
	startClientTransport(info.ClientPort, securityType(CLIENT))

}

func usage() {
	fatal("usage: raftd [PATH]")
}

func createTranHandler(st int) transHandler {
	t := transHandler{}

	switch st {
	case HTTP:
		t := transHandler{}
		t.client = nil
		return t

	case HTTPS:
		fallthrough
	case HTTPSANDVERIFY:
		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)

		if err != nil {
			panic(err)
		}

		tr := &http.Transport{
			TLSClientConfig: &tls.Config{
				Certificates:       []tls.Certificate{tlsCert},
				InsecureSkipVerify: true,
			},
			DisableCompression: true,
		}

		t.client = &http.Client{Transport: tr}
		return t
	}

	// for complier
	return transHandler{}
}

func startServTransport(port int, st int) {

	// internal commands
	http.HandleFunc("/join", JoinHttpHandler)
	http.HandleFunc("/vote", VoteHttpHandler)
	http.HandleFunc("/log", GetLogHttpHandler)
	http.HandleFunc("/log/append", AppendEntriesHttpHandler)
	http.HandleFunc("/snapshot", SnapshotHttpHandler)
	http.HandleFunc("/client", clientHttpHandler)

	switch st {

	case HTTP:
		debug("raft server [%s] listen on http", server.Name())
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))

	case HTTPS:
		http.ListenAndServeTLS(fmt.Sprintf(":%d", port), serverCertFile, serverKeyFile, nil)

	case HTTPSANDVERIFY:
		pemByte, _ := ioutil.ReadFile(serverCAFile)

		block, pemByte := pem.Decode(pemByte)

		cert, err := x509.ParseCertificate(block.Bytes)

		if err != nil {
			fmt.Println(err)
		}

		certPool := x509.NewCertPool()

		certPool.AddCert(cert)

		server := &http.Server{
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  certPool,
			},
			Addr: fmt.Sprintf(":%d", port),
		}
		err = server.ListenAndServeTLS(serverCertFile, serverKeyFile)

		if err != nil {
			log.Fatal(err)
		}
	}

}

func startClientTransport(port int, st int) {
	// external commands
	http.HandleFunc("/v1/keys/", Multiplexer)
	http.HandleFunc("/v1/watch/", WatchHttpHandler)
	http.HandleFunc("/v1/list/", ListHttpHandler)
	http.HandleFunc("/v1/testAndSet/", TestAndSetHttpHandler)
	http.HandleFunc("/master", MasterHttpHandler)

	switch st {

	case HTTP:
		debug("etcd [%s] listen on http", server.Name())
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))

	case HTTPS:
		http.ListenAndServeTLS(fmt.Sprintf(":%d", port), clientCertFile, clientKeyFile, nil)

	case HTTPSANDVERIFY:
		pemByte, _ := ioutil.ReadFile(clientCAFile)

		block, pemByte := pem.Decode(pemByte)

		cert, err := x509.ParseCertificate(block.Bytes)

		if err != nil {
			fmt.Println(err)
		}

		certPool := x509.NewCertPool()

		certPool.AddCert(cert)

		server := &http.Server{
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  certPool,
			},
			Addr: fmt.Sprintf(":%d", port),
		}
		err = server.ListenAndServeTLS(clientCertFile, clientKeyFile)

		if err != nil {
			log.Fatal(err)
		}
	}
}

//--------------------------------------
// Config
//--------------------------------------

func securityType(source int) int {

	var keyFile, certFile, CAFile string

	switch source {
	case SERVER:
		keyFile = serverKeyFile
		certFile = serverCertFile
		CAFile = serverCAFile

	case CLIENT:
		keyFile = clientKeyFile
		certFile = clientCertFile
		CAFile = clientCAFile
	}

	if keyFile == "" && certFile == "" && CAFile == "" {

		return HTTP

	}

	if keyFile != "" && certFile != "" {

		if CAFile != "" {
			return HTTPSANDVERIFY
		}

		return HTTPS
	}

	return -1
}

func getInfo(path string) *Info {
	info := &Info{}

	// Read in the server info if available.
	infoPath := fmt.Sprintf("%s/info", path)

	// delete the old configuration if exist
	if ignore {
		logPath := fmt.Sprintf("%s/log", path)
		snapshotPath := fmt.Sprintf("%s/snapshotPath", path)
		os.Remove(infoPath)
		os.Remove(logPath)
		os.RemoveAll(snapshotPath)

	}

	if file, err := os.Open(infoPath); err == nil {
		if content, err := ioutil.ReadAll(file); err != nil {
			fatal("Unable to read info: %v", err)
		} else {
			if err = json.Unmarshal(content, &info); err != nil {
				fatal("Unable to parse info: %v", err)
			}
		}
		file.Close()

		// Otherwise ask user for info and write it to file.
	} else {

		if address == "" {
			fatal("Please give the address of the local machine")
		}

		info.Address = address
		info.Address = strings.TrimSpace(info.Address)
		fmt.Println("address ", info.Address)

		info.ServerPort = serverPort
		info.ClientPort = clientPort
		info.WebPort = webPort

		// Write to file.
		content, _ := json.Marshal(info)
		content = []byte(string(content) + "\n")
		if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
			fatal("Unable to write info to file: %v", err)
		}
	}

	return info
}

//--------------------------------------
// Handlers
//--------------------------------------

// Send join requests to the leader.
func Join(s *raft.Server, serverName string) error {
	var b bytes.Buffer

	command := &JoinCommand{}
	command.Name = s.Name()

	json.NewEncoder(&b).Encode(command)

	// t must be ok
	t, _ := server.Transporter().(transHandler)
	debug("Send Join Request to %s", serverName)
	resp, err := Post(&t, fmt.Sprintf("%s/join", serverName), &b)

	for {
		if resp != nil {
			defer resp.Body.Close()
			if resp.StatusCode == http.StatusOK {
				return nil
			}
			if resp.StatusCode == http.StatusServiceUnavailable {
				address, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					warn("Cannot Read Leader info: %v", err)
				}
				debug("Leader is %s", address)
				debug("Send Join Request to %s", address)
				json.NewEncoder(&b).Encode(command)
				resp, err = Post(&t, fmt.Sprintf("%s/join", address), &b)
			}
		}
	}
	return fmt.Errorf("Unable to join: %v", err)
}
<|endoftext|>


<commit_before>package main

import (
	"time"
)

// ewma hold the state for an Exponential Weighted Moving Average. For details see:
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
// https://github.com/dgryski/trifles/blob/master/ewmaest/ewmaest.go
// https://stackoverflow.com/a/936720
type ewma struct {
	total         int
	completed     int
	lastCompleted int

	last time.Time

	perItem time.Duration

	α float64
}

// newEWMA returns a new EWMA for total items.
func newEWMA(start time.Time, totalItems int) *ewma {
	return &ewma{
		last:  start,
		total: totalItems,
		α:     0.10,
	}
}

// Report tells the ewma how many items have been processed.
func (e *ewma) Report(totalCompletedItems int) {
	// return early if no new information is being reported
	if totalCompletedItems == 0 || e.completed == totalCompletedItems {
		return
	}

	e.completed = totalCompletedItems

	lastBlockTime := time.Since(e.last)
	e.last = time.Now()

	lastItemEstimate := lastBlockTime / time.Duration(e.completed-e.lastCompleted)
	e.lastCompleted = e.completed

	// use the first measurement directly, without applying α
	if e.perItem == 0 {
		e.perItem = lastItemEstimate
		return
	}

	e.perItem = time.Duration(e.α*float64(lastItemEstimate)) + time.Duration((1-e.α)*float64(e.perItem))
}

// ETA returns the estimated remaining time.
func (e *ewma) ETA() time.Duration {
	remaining := e.total - e.completed
	d := time.Duration(remaining) * e.perItem
	return d
}
<commit_msg>ETA: Take proportional time into account<commit_after>package main

import (
	"time"
)

// ewma hold the state for an Exponential Weighted Moving Average. For details see:
// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
// https://github.com/dgryski/trifles/blob/master/ewmaest/ewmaest.go
// https://stackoverflow.com/a/936720
type ewma struct {
	total         int
	completed     int
	lastCompleted int

	start time.Time
	last  time.Time

	perItem time.Duration

	α, β float64
}

// newEWMA returns a new EWMA for total items.
func newEWMA(start time.Time, totalItems int) *ewma {
	return &ewma{
		start: start,
		last:  start,
		total: totalItems,
		α:     0.10,
		β:     0.5,
	}
}

// Report tells the ewma how many items have been processed.
func (e *ewma) Report(totalCompletedItems int) {
	// return early if no new information is being reported
	if totalCompletedItems == 0 || e.completed == totalCompletedItems {
		return
	}

	e.completed = totalCompletedItems

	lastBlockTime := time.Since(e.last)
	e.last = time.Now()

	lastItemEstimate := lastBlockTime / time.Duration(e.completed-e.lastCompleted)
	e.lastCompleted = e.completed

	// use the first measurement directly, without applying α
	if e.perItem == 0 {
		e.perItem = lastItemEstimate
		return
	}

	e.perItem = time.Duration(e.α*float64(lastItemEstimate)) + time.Duration((1-e.α)*float64(e.perItem))
}

// ETA returns the estimated remaining time.
func (e *ewma) ETA() time.Duration {
	remaining := e.total - e.completed

	perItem := time.Duration(e.β * float64(e.last.Sub(e.start)) / float64(e.completed))
	perItem += time.Duration((1 - e.β) * float64(e.perItem))

	d := time.Duration(remaining) * perItem
	return d
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/AlexanderThaller/logger"
	"github.com/SlyMarbo/rss"
	"github.com/juju/errgo"
	"github.com/vmihailenco/msgpack"
)

type Feed struct {
	Url     string
	Filters []string
	Folder  string
}

func (feed Feed) Launch(conf *Config) {
	l := logger.New(name, "Feed", "Launch", feed.Url)
	l.Info("Starting")

	l.Debug("Will try to get feed")
	data, err := feed.Get(conf)
	if err != nil {
		l.Error("Problem when getting feed: ", errgo.Details(err))
		return
	}
	l.Debug("Got feed")
	l.Trace("Feed data: ", data)

	feed.Watch(data, conf)
}

func (feed *Feed) Watch(data *rss.Feed, conf *Config) {
	l := logger.New(name, "Feed", "Watch", feed.Url)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for {
		d := time.Duration(r.Intn(60000)) * time.Millisecond
		l.Debug("Sleep for ", d)
		time.Sleep(d)

		l.Debug("Try to update feed")
		updated, err := feed.Update(data)
		if err != nil {
			l.Warning("Can not update feed: ", errgo.Details(err))
		}

		if updated {
			l.Debug("Updated feed will now try to save")
			err = feed.Save(data, conf.DataFolder)
			if err != nil {
				l.Error("Problem while saving: ", errgo.Details(err))
				return
			}
		} else {
			l.Debug("Not updated")
		}

		d = 5 * time.Minute
		l.Debug("Sleep for ", d)
		time.Sleep(d)
	}
}

func (feed *Feed) Filter(items []*rss.Item) []*Item {
	var out []*Item
	return out
}

func (feed *Feed) Check(newitems []*rss.Item, items map[string]struct{}) []*rss.Item {
	var out []*rss.Item

	for _, d := range newitems {
		if _, exists := items[d.ID]; exists {
			out = append(out, d)
		}
	}

	return out
}

func (feed *Feed) Get(conf *Config) (*rss.Feed, error) {
	l := logger.New(name, "Feed", "Get", feed.Url)

	if conf.SaveFeeds {
		l.Debug("Will try to restore feed")

		data, err := feed.Restore(conf.DataFolder)
		if err == nil {
			l.Debug("Restored feed. Will return feed")
			return data, nil
		}

		l.Debug("Can not restore feed")
		if !os.IsNotExist(err) {
			l.Debug("Error is not a not exists error we will return this")
			return nil, err
		}

		l.Trace("Error while restoring: ", err)
	}

	l.Debug("Will try to fetch feed")
	data, err := rss.Fetch(feed.Url)
	if err != nil {
		return nil, err
	}
	l.Debug("Fetched feed")

	if conf.SaveFeeds {
		err = feed.Save(data, conf.DataFolder)
		if err != nil {
			return data, err
		}
	}

	return data, err
}

func (feed *Feed) Restore(datafolder string) (*rss.Feed, error) {
	l := logger.New(name, "Feed", "Restore", feed.Url)

	filename := feed.Filename(datafolder) + ".msgpack"
	l.Trace("Filename: ", filename)

	l.Debug("Check if file exists")
	_, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}
	l.Debug("File does exist")

	l.Debug("Read from file")
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	l.Debug("Finished reading file")

	var data rss.Feed
	l.Debug("Unmarshal bytes from file")
	err = msgpack.Unmarshal(bytes, &data)
	if err != nil {
		return nil, err
	}
	l.Debug("Finished unmarshaling")
	l.Debug("Finished restoring")
	l.Trace("Data: ", data)
	return &data, nil
}

func (feed *Feed) Update(data *rss.Feed) (bool, error) {
	l := logger.New(name, "Feed", "Update", feed.Url)

	l.Trace("Refresh: ", data.Refresh)
	l.Trace("After: ", data.Refresh.After(time.Now()))
	if data.Refresh.After(time.Now()) {
		l.Debug("Its not time to update yet")
		return false, nil
	}
	l.Debug("Will update feed")

	err := data.Update()
	if err != nil {
		return false, err
	}
	l.Debug("Updated feed")
	l.Trace("New refresh: ", data.Refresh)

	return true, nil
}

func (feed *Feed) Save(data *rss.Feed, datafolder string) error {
	l := logger.New(name, "Feed", "Save", feed.Url)

	l.Debug("Will try to save feed")
	filename := feed.Filename(datafolder) + ".msgpack"

	err := os.MkdirAll(datafolder, 0755)
	if err != nil {
		return err
	}

	bytes, err := msgpack.Marshal(data)
	err = ioutil.WriteFile(filename, bytes, 0644)
	if err != nil {
		return err
	}

	l.Debug("Saved feed")
	return nil
}

func (feed *Feed) Filename(datafolder string) string {
	saveurl := strings.Replace(feed.Url, "/", "_", -1)
	filename := filepath.Join(datafolder, saveurl)

	return filename
}
<commit_msg>will now sleep from 10s to 60s when starting to watch feed.<commit_after>package main

import (
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/AlexanderThaller/logger"
	"github.com/SlyMarbo/rss"
	"github.com/juju/errgo"
	"github.com/vmihailenco/msgpack"
)

type Feed struct {
	Url     string
	Filters []string
	Folder  string
}

func (feed Feed) Launch(conf *Config) {
	l := logger.New(name, "Feed", "Launch", feed.Url)
	l.Info("Starting")

	l.Debug("Will try to get feed")
	data, err := feed.Get(conf)
	if err != nil {
		l.Error("Problem when getting feed: ", errgo.Details(err))
		return
	}
	l.Debug("Got feed")
	l.Trace("Feed data: ", data)

	feed.Watch(data, conf)
}

func (feed *Feed) Watch(data *rss.Feed, conf *Config) {
	l := logger.New(name, "Feed", "Watch", feed.Url)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for {
		d := time.Duration(r.Intn(50000)+10000) * time.Millisecond
		l.Debug("Sleep for ", d)
		time.Sleep(d)

		l.Debug("Try to update feed")
		updated, err := feed.Update(data)
		if err != nil {
			l.Warning("Can not update feed: ", errgo.Details(err))
		}

		if updated {
			l.Debug("Updated feed will now try to save")
			err = feed.Save(data, conf.DataFolder)
			if err != nil {
				l.Error("Problem while saving: ", errgo.Details(err))
				return
			}
		} else {
			l.Debug("Not updated")
		}

		d = 5 * time.Minute
		l.Debug("Sleep for ", d)
		time.Sleep(d)
	}
}

func (feed *Feed) Filter(items []*rss.Item) []*Item {
	var out []*Item
	return out
}

func (feed *Feed) Check(newitems []*rss.Item, items map[string]struct{}) []*rss.Item {
	var out []*rss.Item

	for _, d := range newitems {
		if _, exists := items[d.ID]; exists {
			out = append(out, d)
		}
	}

	return out
}

func (feed *Feed) Get(conf *Config) (*rss.Feed, error) {
	l := logger.New(name, "Feed", "Get", feed.Url)

	if conf.SaveFeeds {
		l.Debug("Will try to restore feed")

		data, err := feed.Restore(conf.DataFolder)
		if err == nil {
			l.Debug("Restored feed. Will return feed")
			return data, nil
		}

		l.Debug("Can not restore feed")
		if !os.IsNotExist(err) {
			l.Debug("Error is not a not exists error we will return this")
			return nil, err
		}

		l.Trace("Error while restoring: ", err)
	}

	l.Debug("Will try to fetch feed")
	data, err := rss.Fetch(feed.Url)
	if err != nil {
		return nil, err
	}
	l.Debug("Fetched feed")

	if conf.SaveFeeds {
		err = feed.Save(data, conf.DataFolder)
		if err != nil {
			return data, err
		}
	}

	return data, err
}

func (feed *Feed) Restore(datafolder string) (*rss.Feed, error) {
	l := logger.New(name, "Feed", "Restore", feed.Url)

	filename := feed.Filename(datafolder) + ".msgpack"
	l.Trace("Filename: ", filename)

	l.Debug("Check if file exists")
	_, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}
	l.Debug("File does exist")

	l.Debug("Read from file")
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	l.Debug("Finished reading file")

	var data rss.Feed
	l.Debug("Unmarshal bytes from file")
	err = msgpack.Unmarshal(bytes, &data)
	if err != nil {
		return nil, err
	}
	l.Debug("Finished unmarshaling")
	l.Debug("Finished restoring")
	l.Trace("Data: ", data)
	return &data, nil
}

func (feed *Feed) Update(data *rss.Feed) (bool, error) {
	l := logger.New(name, "Feed", "Update", feed.Url)

	l.Trace("Refresh: ", data.Refresh)
	l.Trace("After: ", data.Refresh.After(time.Now()))
	if data.Refresh.After(time.Now()) {
		l.Debug("Its not time to update yet")
		return false, nil
	}
	l.Debug("Will update feed")

	err := data.Update()
	if err != nil {
		return false, err
	}
	l.Debug("Updated feed")
	l.Trace("New refresh: ", data.Refresh)

	return true, nil
}

func (feed *Feed) Save(data *rss.Feed, datafolder string) error {
	l := logger.New(name, "Feed", "Save", feed.Url)

	l.Debug("Will try to save feed")
	filename := feed.Filename(datafolder) + ".msgpack"

	err := os.MkdirAll(datafolder, 0755)
	if err != nil {
		return err
	}

	bytes, err := msgpack.Marshal(data)
	err = ioutil.WriteFile(filename, bytes, 0644)
	if err != nil {
		return err
	}

	l.Debug("Saved feed")
	return nil
}

func (feed *Feed) Filename(datafolder string) string {
	saveurl := strings.Replace(feed.Url, "/", "_", -1)
	filename := filepath.Join(datafolder, saveurl)

	return filename
}
<|endoftext|>