<commit_before>package core

/*
#cgo CFLAGS: -I./src/include
#include "lwip/tcp.h"
*/
import "C"
import (
	"errors"
	"fmt"
	"net"
	"sync"
	"unsafe"
)

type tcpConn struct {
	sync.Mutex

	pcb         *C.struct_tcp_pcb
	handler     ConnectionHandler
	network     string
	remoteAddr  net.Addr
	localAddr   net.Addr
	connKeyArg  unsafe.Pointer
	connKey     uint32
	closing     bool
	localClosed bool
	aborting    bool
	canWrite    *sync.Cond // Condition variable to implement TCP backpressure.
}

func NewTCPConnection(pcb *C.struct_tcp_pcb, handler ConnectionHandler, connKey uint32, connKeyArg unsafe.Pointer) (Connection, error) {
	// Pass the key as arg for subsequent tcp callbacks.
	C.tcp_arg(pcb, unsafe.Pointer(connKeyArg))

	// Register callbacks.
	SetTCPRecvCallback(pcb)
	SetTCPSentCallback(pcb)
	SetTCPErrCallback(pcb)
	SetTCPPollCallback(pcb, C.u8_t(TCP_POLL_INTERVAL))

	conn := &tcpConn{
		pcb:         pcb,
		handler:     handler,
		network:     "tcp",
		localAddr:   ParseTCPAddr(IPAddrNTOA(pcb.remote_ip), uint16(pcb.remote_port)),
		remoteAddr:  ParseTCPAddr(IPAddrNTOA(pcb.local_ip), uint16(pcb.local_port)),
		connKeyArg:  connKeyArg,
		connKey:     connKey,
		closing:     false,
		localClosed: false,
		aborting:    false,
		canWrite:    sync.NewCond(&sync.Mutex{}),
	}

	// Unlocks lwip thread during connecting remote host, gives other goroutines
	// chances to interact with the lwip thread. Assuming lwip thread has already
	// been locked.
	lwipMutex.Unlock()
	err := handler.Connect(conn, conn.RemoteAddr())
	lwipMutex.Lock()
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (conn *tcpConn) RemoteAddr() net.Addr {
	return conn.remoteAddr
}

func (conn *tcpConn) LocalAddr() net.Addr {
	return conn.localAddr
}

func (conn *tcpConn) Receive(data []byte) error {
	if conn.isClosing() {
		return errors.New(fmt.Sprintf("connection %v->%v was closed by remote", conn.LocalAddr(), conn.RemoteAddr()))
	}
	if conn.isAborting() {
		return errors.New(fmt.Sprintf("connection %v->%v is aborting", conn.LocalAddr(), conn.RemoteAddr()))
	}
	// Unlocks lwip thread during sending data to remote, gives other goroutines
	// chances to interact with the lwip thread. Assuming lwip thread has already
	// been locked.
	lwipMutex.Unlock()
	err := conn.handler.DidReceive(conn, data)
	lwipMutex.Lock()
	if err != nil {
		return errors.New(fmt.Sprintf("write proxy failed: %v", err))
	}
	C.tcp_recved(conn.pcb, C.u16_t(len(data)))
	return nil
}

// tcpWrite enqueues data to snd_buf, and treats ERR_MEM returned by tcp_write not an error,
// but instead tells the caller that data is not successfully enqueued, and should try
// again another time. By calling this function, the lwIP thread is assumed to be already
// locked by the caller.
func (conn *tcpConn) tcpWrite(data []byte) (int, error) {
	err := C.tcp_write(conn.pcb, unsafe.Pointer(&data[0]), C.u16_t(len(data)), C.TCP_WRITE_FLAG_COPY)
	if err == C.ERR_OK {
		C.tcp_output(conn.pcb)
		return len(data), nil
	} else if err == C.ERR_MEM {
		return 0, nil
	}
	return 0, fmt.Errorf("lwip tcp_write failed with error code: %v", int(err))
}

func (conn *tcpConn) Write(data []byte) (int, error) {
	totalWritten := 0

	conn.canWrite.L.Lock()
	defer conn.canWrite.L.Unlock()

	for len(data) > 0 {
		if conn.isLocalClosed() {
			return totalWritten, fmt.Errorf("connection %v->%v was closed by local", conn.LocalAddr(), conn.RemoteAddr())
		}
		if conn.isAborting() {
			return totalWritten, fmt.Errorf("connection %v->%v is aborting", conn.LocalAddr(), conn.RemoteAddr())
		}

		lwipMutex.Lock()
		toWrite := len(data)
		if toWrite > int(conn.pcb.snd_buf) {
			// Write at most the size of the LWIP buffer.
			toWrite = int(conn.pcb.snd_buf)
		}
		if toWrite > 0 {
			written, err := conn.tcpWrite(data[0:toWrite])
			totalWritten += written
			if err != nil {
				lwipMutex.Unlock()
				return totalWritten, err
			}
			data = data[written:len(data)]
		}
		lwipMutex.Unlock()
		if len(data) == 0 {
			break // Don't block if all the data has been written.
		}
		conn.canWrite.Wait()
	}

	return totalWritten, nil
}

func (conn *tcpConn) Sent(len uint16) error {
	conn.handler.DidSend(conn, len)
	// Some packets are acknowledged by local client, check if any pending data to send.
	return conn.CheckState()
}

func (conn *tcpConn) isClosing() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.closing
}

func (conn *tcpConn) isAborting() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.aborting
}

func (conn *tcpConn) isLocalClosed() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.localClosed
}

func (conn *tcpConn) CheckState() error {
	if !conn.isLocalClosed() {
		// Signal the writer to try writting.
		conn.canWrite.Broadcast()
		// Return and wait for the Sent() callback to be called, and then check again.
		return NewLWIPError(LWIP_ERR_OK)
	}

	if conn.isClosing() || conn.isLocalClosed() {
		conn.closeInternal()
	}

	if conn.isAborting() {
		conn.abortInternal()
		return NewLWIPError(LWIP_ERR_ABRT)
	}

	return NewLWIPError(LWIP_ERR_OK)
}

func (conn *tcpConn) Close() error {
	conn.Lock()
	defer conn.Unlock()

	// Close maybe called outside of lwIP thread, we should not call tcp_close() in this
	// function, instead just make a flag to indicate we are closing the connection.
	conn.closing = true
	return nil
}

func (conn *tcpConn) setLocalClosed() error {
	conn.Lock()
	defer conn.Unlock()

	conn.localClosed = true
	conn.canWrite.Broadcast()
	return nil
}

func (conn *tcpConn) closeInternal() error {
	C.tcp_arg(conn.pcb, nil)
	C.tcp_recv(conn.pcb, nil)
	C.tcp_sent(conn.pcb, nil)
	C.tcp_err(conn.pcb, nil)
	C.tcp_poll(conn.pcb, nil, 0)

	conn.Release()

	// TODO: may return ERR_MEM if no memory to allocate segments use for closing the conn,
	// should check and try again in Sent() for Poll() callbacks.
	err := C.tcp_close(conn.pcb)
	if err == C.ERR_OK {
		return nil
	} else {
		return errors.New(fmt.Sprintf("close TCP connection failed, lwip error code %d", int(err)))
	}
}

func (conn *tcpConn) abortInternal() {
	conn.Release()
	C.tcp_abort(conn.pcb)
}

func (conn *tcpConn) Abort() {
	conn.Lock()
	defer conn.Unlock()

	conn.aborting = true
	conn.canWrite.Broadcast()
}

// The corresponding pcb is already freed when this callback is called
func (conn *tcpConn) Err(err error) {
	conn.Release()
	conn.handler.DidClose(conn)
}

func (conn *tcpConn) LocalDidClose() error {
	conn.handler.LocalDidClose(conn)
	conn.setLocalClosed()    // flag closing
	return conn.CheckState() // check pending data
}

func (conn *tcpConn) Release() {
	if _, found := tcpConns.Load(conn.connKey); found {
		FreeConnKeyArg(conn.connKeyArg)
		tcpConns.Delete(conn.connKey)
	}
}

func (conn *tcpConn) Poll() error {
	return conn.CheckState()
}
<commit_msg>properly shuts down a closing/aborting conn<commit_after>package core

/*
#cgo CFLAGS: -I./src/include
#include "lwip/tcp.h"
*/
import "C"
import (
	"errors"
	"fmt"
	"net"
	"sync"
	"unsafe"
)

type tcpConn struct {
	sync.Mutex

	pcb         *C.struct_tcp_pcb
	handler     ConnectionHandler
	network     string
	remoteAddr  net.Addr
	localAddr   net.Addr
	connKeyArg  unsafe.Pointer
	connKey     uint32
	closing     bool
	localClosed bool
	aborting    bool
	canWrite    *sync.Cond // Condition variable to implement TCP backpressure.
}

func NewTCPConnection(pcb *C.struct_tcp_pcb, handler ConnectionHandler, connKey uint32, connKeyArg unsafe.Pointer) (Connection, error) {
	// Pass the key as arg for subsequent tcp callbacks.
	C.tcp_arg(pcb, unsafe.Pointer(connKeyArg))

	// Register callbacks.
	SetTCPRecvCallback(pcb)
	SetTCPSentCallback(pcb)
	SetTCPErrCallback(pcb)
	SetTCPPollCallback(pcb, C.u8_t(TCP_POLL_INTERVAL))

	conn := &tcpConn{
		pcb:         pcb,
		handler:     handler,
		network:     "tcp",
		localAddr:   ParseTCPAddr(IPAddrNTOA(pcb.remote_ip), uint16(pcb.remote_port)),
		remoteAddr:  ParseTCPAddr(IPAddrNTOA(pcb.local_ip), uint16(pcb.local_port)),
		connKeyArg:  connKeyArg,
		connKey:     connKey,
		closing:     false,
		localClosed: false,
		aborting:    false,
		canWrite:    sync.NewCond(&sync.Mutex{}),
	}

	// Unlocks lwip thread during connecting remote host, gives other goroutines
	// chances to interact with the lwip thread. Assuming lwip thread has already
	// been locked.
	lwipMutex.Unlock()
	err := handler.Connect(conn, conn.RemoteAddr())
	lwipMutex.Lock()
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (conn *tcpConn) RemoteAddr() net.Addr {
	return conn.remoteAddr
}

func (conn *tcpConn) LocalAddr() net.Addr {
	return conn.localAddr
}

func (conn *tcpConn) Receive(data []byte) error {
	if conn.isClosing() {
		return errors.New(fmt.Sprintf("connection %v->%v was closed by remote", conn.LocalAddr(), conn.RemoteAddr()))
	}
	if conn.isAborting() {
		return errors.New(fmt.Sprintf("connection %v->%v is aborting", conn.LocalAddr(), conn.RemoteAddr()))
	}
	// Unlocks lwip thread during sending data to remote, gives other goroutines
	// chances to interact with the lwip thread. Assuming lwip thread has already
	// been locked.
	lwipMutex.Unlock()
	err := conn.handler.DidReceive(conn, data)
	lwipMutex.Lock()
	if err != nil {
		return errors.New(fmt.Sprintf("write proxy failed: %v", err))
	}
	C.tcp_recved(conn.pcb, C.u16_t(len(data)))
	return nil
}

// tcpWrite enqueues data to snd_buf, and treats ERR_MEM returned by tcp_write not an error,
// but instead tells the caller that data is not successfully enqueued, and should try
// again another time. By calling this function, the lwIP thread is assumed to be already
// locked by the caller.
func (conn *tcpConn) tcpWrite(data []byte) (int, error) {
	err := C.tcp_write(conn.pcb, unsafe.Pointer(&data[0]), C.u16_t(len(data)), C.TCP_WRITE_FLAG_COPY)
	if err == C.ERR_OK {
		C.tcp_output(conn.pcb)
		return len(data), nil
	} else if err == C.ERR_MEM {
		return 0, nil
	}
	return 0, fmt.Errorf("lwip tcp_write failed with error code: %v", int(err))
}

func (conn *tcpConn) Write(data []byte) (int, error) {
	totalWritten := 0

	conn.canWrite.L.Lock()
	defer conn.canWrite.L.Unlock()

	for len(data) > 0 {
		if conn.isLocalClosed() {
			return totalWritten, fmt.Errorf("connection %v->%v was closed by local", conn.LocalAddr(), conn.RemoteAddr())
		}
		if conn.isAborting() {
			return totalWritten, fmt.Errorf("connection %v->%v is aborting", conn.LocalAddr(), conn.RemoteAddr())
		}

		lwipMutex.Lock()
		toWrite := len(data)
		if toWrite > int(conn.pcb.snd_buf) {
			// Write at most the size of the LWIP buffer.
			toWrite = int(conn.pcb.snd_buf)
		}
		if toWrite > 0 {
			written, err := conn.tcpWrite(data[0:toWrite])
			totalWritten += written
			if err != nil {
				lwipMutex.Unlock()
				return totalWritten, err
			}
			data = data[written:len(data)]
		}
		lwipMutex.Unlock()
		if len(data) == 0 {
			break // Don't block if all the data has been written.
		}
		conn.canWrite.Wait()
	}

	return totalWritten, nil
}

func (conn *tcpConn) Sent(len uint16) error {
	conn.handler.DidSend(conn, len)
	// Some packets are acknowledged by local client, check if any pending data to send.
	return conn.CheckState()
}

func (conn *tcpConn) isClosing() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.closing
}

func (conn *tcpConn) isAborting() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.aborting
}

func (conn *tcpConn) isLocalClosed() bool {
	conn.Lock()
	defer conn.Unlock()
	return conn.localClosed
}

func (conn *tcpConn) CheckState() error {
	if conn.isAborting() {
		conn.abortInternal()
		return NewLWIPError(LWIP_ERR_ABRT)
	}

	if conn.isClosing() || conn.isLocalClosed() {
		conn.closeInternal()
		return NewLWIPError(LWIP_ERR_OK)
	}

	// Signal the writer to try writting.
	conn.canWrite.Broadcast()

	return NewLWIPError(LWIP_ERR_OK)
}

func (conn *tcpConn) Close() error {
	conn.Lock()
	defer conn.Unlock()

	// Close maybe called outside of lwIP thread, we should not call tcp_close() in this
	// function, instead just make a flag to indicate we are closing the connection.
	conn.closing = true
	return nil
}

func (conn *tcpConn) setLocalClosed() error {
	conn.Lock()
	defer conn.Unlock()

	conn.localClosed = true
	conn.canWrite.Broadcast()
	return nil
}

func (conn *tcpConn) closeInternal() error {
	C.tcp_arg(conn.pcb, nil)
	C.tcp_recv(conn.pcb, nil)
	C.tcp_sent(conn.pcb, nil)
	C.tcp_err(conn.pcb, nil)
	C.tcp_poll(conn.pcb, nil, 0)

	conn.Release()

	// TODO: may return ERR_MEM if no memory to allocate segments use for closing the conn,
	// should check and try again in Sent() for Poll() callbacks.
	err := C.tcp_close(conn.pcb)
	if err == C.ERR_OK {
		return nil
	} else {
		return errors.New(fmt.Sprintf("close TCP connection failed, lwip error code %d", int(err)))
	}
}

func (conn *tcpConn) abortInternal() {
	conn.Release()
	C.tcp_abort(conn.pcb)
}

func (conn *tcpConn) Abort() {
	conn.Lock()
	defer conn.Unlock()

	conn.aborting = true
	conn.canWrite.Broadcast()
}

// The corresponding pcb is already freed when this callback is called
func (conn *tcpConn) Err(err error) {
	conn.Release()
	conn.handler.DidClose(conn)
}

func (conn *tcpConn) LocalDidClose() error {
	conn.handler.LocalDidClose(conn)
	conn.setLocalClosed()    // flag closing
	return conn.CheckState() // check pending data
}

func (conn *tcpConn) Release() {
	if _, found := tcpConns.Load(conn.connKey); found {
		FreeConnKeyArg(conn.connKeyArg)
		tcpConns.Delete(conn.connKey)
	}
}

func (conn *tcpConn) Poll() error {
	return conn.CheckState()
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "os/exec"
  "strings"
  "strconv"
  "encoding/json"
  "io/ioutil"
  "errors"
  "github.com/jawher/mow.cli"
  log "github.com/Sirupsen/logrus"
)

var version string
var commit string

type configuration struct {
  Periscope periscope `json:"periscope"`
  Video video `json:"video"`
}

type periscope struct {
  Key string `json:"key"`
}

type video struct {
  Width int `json:"width"`
  Height int `json:"height"`
  Sharpness int `json:"sharpness"`
  Quality int `json:"quality"`
  Bitrate int `json:"bitrate"`
  VFlip bool `json:"vflip"`
  HFlip bool `json:"hflip"`
}

func joinProps(props map[string]string, kvSeparator string, fieldSeparator string) string {
  parts := []string{}
  for key, value := range props {
    parts = append(parts, key + kvSeparator + value)
  }
  return strings.Join(parts, fieldSeparator)
}

func showCommand(cmd *exec.Cmd) {
  log.WithFields(log.Fields{
    "cmd": cmd.Path,
    "args": strings.Join(cmd.Args[1:], " "),
  }).Debug("Running command")
}

func mergeString(l string, r string) string {
  if r == "" {
    return l
  } else {
    return r
  }
}

func mergeInt(l int, r int) int {
  if r == 0 {
    return l
  } else {
    return r
  }
}

func mergeBool(l bool, r bool, def bool) bool {
  if r == def {
    return l
  } else {
    return r
  }
}

func mergePeriscope(l *periscope, r *periscope) *periscope {
  return &periscope{
    Key: mergeString(l.Key, r.Key),
  }
}

func mergeVideo(l *video, r *video) *video {
  return &video{
    Width: mergeInt(l.Width, r.Width),
    Height: mergeInt(l.Height, r.Height),
    Sharpness: mergeInt(l.Sharpness, r.Sharpness),
    Quality: mergeInt(l.Quality, r.Quality),
    Bitrate: mergeInt(l.Bitrate, r.Bitrate),
    VFlip: mergeBool(l.VFlip, r.VFlip, false),
    HFlip: mergeBool(l.HFlip, r.HFlip, false),
  }
}

func mergeConfig(l *configuration, r *configuration) *configuration {
  return &configuration{
    Periscope: *mergePeriscope(&l.Periscope, &r.Periscope),
    Video: *mergeVideo(&l.Video, &r.Video),
  }
}

func runStream(config *configuration) error {
  if config.Periscope.Key == "" {
    return errors.New("Periscope stream key (-k, --key) is required.")
  }

  streamUrl := "rtmp://va.pscp.tv:80/x/" + config.Periscope.Key

  width := strconv.Itoa(config.Video.Width)
  height := strconv.Itoa(config.Video.Height)
  sharpness := strconv.Itoa(config.Video.Sharpness)
  quality := strconv.Itoa(config.Video.Quality)
  bitrate := strconv.Itoa(config.Video.Bitrate)

  videoProps := map[string]string{
    "width": width,
    "height": height,
    "pixelformat": "4",
  }

  v4l2 := exec.Command("v4l2-ctl", "--set-fmt-video=" + joinProps(videoProps, "=", ","))
  v4l2.Stdout = os.Stdout
  v4l2.Stderr = os.Stderr

  showCommand(v4l2)
  if err := v4l2.Run(); err != nil {
    return err
  }

  controlProps := map[string]string{
    "sharpness": sharpness,
    "compression_quality": quality,
    "video_bitrate_mode": "1",
    "video_bitrate": bitrate,
  }

  v4l2 = exec.Command("v4l2-ctl", "--set-ctrl=" + joinProps(controlProps, "=", ","))
  v4l2.Stdout = os.Stdout
  v4l2.Stderr = os.Stderr

  showCommand(v4l2)
  if err := v4l2.Run(); err != nil {
    return err
  }

  raspividArgs := []string{
    "-o", "-",                        // Write video to stdout.
    "-t", "0",                        // Continuous capturing (no timeout).
    "-w", width,                      // Output video width.
    "-h", height,                     // Output video height.
    "-fps", "30",                     // Capture video at 30 frames per second.
    "-b", bitrate,                    // Capture video bitrate.
  }

  if config.Video.VFlip {
    raspividArgs = append(raspividArgs, "-vf")
  }

  if config.Video.HFlip {
    raspividArgs = append(raspividArgs, "-hf")
  }

  raspivid := exec.Command("raspivid", raspividArgs...)

  ffmpegArgs := []string{
    "-re",                            // Read from input at its native framerate. Best for real-time/streaming output.
    "-f", "lavfi", "-i", "anullsrc",  // No input audio.
    "-i", "-",                        // Use stdin for video (from raspivid).
    "-acodec", "aac",                 // Use AAC codec for audio (Periscope requirement).
    "-b:a", "0",                      // Zero audio bitrate since we have no input audio.
    "-map", "0:a",                    // Use stream 0 for audio (anullsrc).
    "-map", "1:v",                    // Use stream 1 for video (stdin).
    "-f", "h264",                     // Use H.264 codec for video (Periscope requirement).
    "-vcodec", "copy",                // Copy video data directly from input.
    "-g", "60",                       // Keyframe interval: one keyframe every 60 frames (2 seconds for 30 fps video; Periscope requirement).
    "-f", "flv",                      // Package output in a Flash Video container (Periscope requirement).
    streamUrl,                        // RTMP streaming destination.
  }

  ffmpeg := exec.Command("ffmpeg", ffmpegArgs...)

  ffmpegStdin, err := ffmpeg.StdinPipe()
  if err != nil {
    return err
  }

  raspivid.Stderr = os.Stderr
  raspivid.Stdout = ffmpegStdin
  ffmpeg.Stdout = os.Stdout
  ffmpeg.Stderr = os.Stderr

  defer func () {
    raspivid.Wait()
    ffmpegStdin.Close()
    ffmpeg.Wait()
  }()

  showCommand(ffmpeg)
  if err := ffmpeg.Start(); err != nil {
    return err
  }

  showCommand(raspivid)
  if err := raspivid.Start(); err != nil {
    return err
  }

  return nil
}

func main() {
  app := cli.App("piriscope", "Piriscope - https://github.com/schmich/piriscope")
  key := app.StringOpt("k key", "", "Periscope stream key")
  conf := app.StringOpt("c conf", "", "Configuration file")
  verbose := app.BoolOpt("v verbose", false, "Verbose output")

  app.Version("version", "piriscope " + version + " " + commit)

  app.Action = func () {
    if *verbose {
      log.SetLevel(log.DebugLevel)
    }

    var fileConfig configuration
    if *conf != "" {
      content, err := ioutil.ReadFile(*conf)
      if err != nil {
        log.Fatal(err)
      }

      log.WithFields(log.Fields{ "file": *conf }).Info("Using configuration file")

      err = json.Unmarshal(content, &fileConfig)
      if err != nil {
        log.Fatal("Error in configuration file: ", err)
      }
    }

    defaultConfig := &configuration{
      Periscope: periscope{
        Key: "",
      },
      Video: video{
        Width: 960,
        Height: 540,
        Sharpness: 30,
        Quality: 80,
        Bitrate: 800000,
        VFlip: false,
        HFlip: false,
      },
    }

    cliConfig := &configuration{
      Periscope: periscope{
        Key: *key,
      },
    }

    config := mergeConfig(defaultConfig, mergeConfig(&fileConfig, cliConfig))
    if err := runStream(config); err != nil {
      log.Fatal(err)
    }
  }

  app.Run(os.Args)
}
<commit_msg>Drop raspivid dependency in favor of just vl42-ctl.<commit_after>package main

import (
  "os"
  "os/exec"
  "strings"
  "strconv"
  "encoding/json"
  "io/ioutil"
  "errors"
  "github.com/jawher/mow.cli"
  log "github.com/Sirupsen/logrus"
)

var version string
var commit string

type configuration struct {
  Periscope periscope `json:"periscope"`
  Video video `json:"video"`
}

type periscope struct {
  Key string `json:"key"`
}

type video struct {
  Width int `json:"width"`
  Height int `json:"height"`
  Sharpness int `json:"sharpness"`
  Quality int `json:"quality"`
  Bitrate int `json:"bitrate"`
  VFlip bool `json:"vflip"`
  HFlip bool `json:"hflip"`
}

func joinProps(props map[string]string, kvSeparator string, fieldSeparator string) string {
  parts := []string{}
  for key, value := range props {
    parts = append(parts, key + kvSeparator + value)
  }
  return strings.Join(parts, fieldSeparator)
}

func showCommand(cmd *exec.Cmd) {
  log.WithFields(log.Fields{
    "cmd": cmd.Path,
    "args": strings.Join(cmd.Args[1:], " "),
  }).Debug("Running command")
}

func mergeString(l string, r string) string {
  if r == "" {
    return l
  } else {
    return r
  }
}

func mergeInt(l int, r int) int {
  if r == 0 {
    return l
  } else {
    return r
  }
}

func mergeBool(l bool, r bool, def bool) bool {
  if r == def {
    return l
  } else {
    return r
  }
}

func mergePeriscope(l *periscope, r *periscope) *periscope {
  return &periscope{
    Key: mergeString(l.Key, r.Key),
  }
}

func mergeVideo(l *video, r *video) *video {
  return &video{
    Width: mergeInt(l.Width, r.Width),
    Height: mergeInt(l.Height, r.Height),
    Sharpness: mergeInt(l.Sharpness, r.Sharpness),
    Quality: mergeInt(l.Quality, r.Quality),
    Bitrate: mergeInt(l.Bitrate, r.Bitrate),
    VFlip: mergeBool(l.VFlip, r.VFlip, false),
    HFlip: mergeBool(l.HFlip, r.HFlip, false),
  }
}

func mergeConfig(l *configuration, r *configuration) *configuration {
  return &configuration{
    Periscope: *mergePeriscope(&l.Periscope, &r.Periscope),
    Video: *mergeVideo(&l.Video, &r.Video),
  }
}

func runStream(config *configuration) error {
  if config.Periscope.Key == "" {
    return errors.New("Periscope stream key (-k, --key) is required.")
  }

  streamUrl := "rtmp://va.pscp.tv:80/x/" + config.Periscope.Key

  width := strconv.Itoa(config.Video.Width)
  height := strconv.Itoa(config.Video.Height)
  sharpness := strconv.Itoa(config.Video.Sharpness)
  quality := strconv.Itoa(config.Video.Quality)
  bitrate := strconv.Itoa(config.Video.Bitrate)

  videoProps := map[string]string{
    "width": width,
    "height": height,
    "pixelformat": "4",
  }

  v4l2 := exec.Command("v4l2-ctl", "--set-fmt-video=" + joinProps(videoProps, "=", ","))
  v4l2.Stdout = os.Stdout
  v4l2.Stderr = os.Stderr

  showCommand(v4l2)
  if err := v4l2.Run(); err != nil {
    return err
  }

  controlProps := map[string]string{
    "sharpness": sharpness,
    "compression_quality": quality,
    "video_bitrate_mode": "1",
    "video_bitrate": bitrate,
    "vertical_flip": "0",
    "horizontal_flip": "0",
  }

  if config.Video.VFlip {
    controlProps["vertical_flip"] = "1"
  }

  if config.Video.HFlip {
    controlProps["horizontal_flip"] = "1"
  }

  v4l2 = exec.Command("v4l2-ctl", "--set-ctrl=" + joinProps(controlProps, "=", ","))
  v4l2.Stdout = os.Stdout
  v4l2.Stderr = os.Stderr

  showCommand(v4l2)
  if err := v4l2.Run(); err != nil {
    return err
  }

  ffmpegArgs := []string{
    "-re",                              // Read from input at its native framerate. Best for real-time/streaming output.
    "-f", "lavfi", "-i", "anullsrc",    // No input audio.
    "-f", "h264", "-i", "/dev/video0",  // Use H.264-formatted video from /dev/video0.
    "-acodec", "aac",                   // Use AAC codec for audio (Periscope requirement).
    "-b:a", "0",                        // Zero audio bitrate since we have no input audio.
    "-map", "0:a",                      // Use stream 0 for audio (anullsrc).
    "-map", "1:v",                      // Use stream 1 for video (stdin).
    "-f", "h264",                       // Use H.264 codec for video (Periscope requirement).
    "-vcodec", "copy",                  // Copy video data directly from input.
    "-g", "60",                         // Keyframe interval: one keyframe every 60 frames (2 seconds for 30 fps video; Periscope requirement).
    "-f", "flv",                        // Package output in a Flash Video container (Periscope requirement).
    streamUrl,                          // RTMP streaming destination.
  }

  ffmpeg := exec.Command("ffmpeg", ffmpegArgs...)
  ffmpeg.Stdout = os.Stdout
  ffmpeg.Stderr = os.Stderr

  defer func () {
    ffmpeg.Wait()
  }()

  showCommand(ffmpeg)
  if err := ffmpeg.Start(); err != nil {
    return err
  }

  return nil
}

func main() {
  app := cli.App("piriscope", "Piriscope - https://github.com/schmich/piriscope")
  key := app.StringOpt("k key", "", "Periscope stream key")
  conf := app.StringOpt("c conf", "", "Configuration file")
  verbose := app.BoolOpt("v verbose", false, "Verbose output")

  app.Version("version", "piriscope " + version + " " + commit)

  app.Action = func () {
    if *verbose {
      log.SetLevel(log.DebugLevel)
    }

    var fileConfig configuration
    if *conf != "" {
      content, err := ioutil.ReadFile(*conf)
      if err != nil {
        log.Fatal(err)
      }

      log.WithFields(log.Fields{ "file": *conf }).Info("Using configuration file")

      err = json.Unmarshal(content, &fileConfig)
      if err != nil {
        log.Fatal("Error in configuration file: ", err)
      }
    }

    defaultConfig := &configuration{
      Periscope: periscope{
        Key: "",
      },
      Video: video{
        Width: 960,
        Height: 540,
        Sharpness: 30,
        Quality: 80,
        Bitrate: 800000,
        VFlip: false,
        HFlip: false,
      },
    }

    cliConfig := &configuration{
      Periscope: periscope{
        Key: *key,
      },
    }

    config := mergeConfig(defaultConfig, mergeConfig(&fileConfig, cliConfig))
    if err := runStream(config); err != nil {
      log.Fatal(err)
    }
  }

  app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package licensecheck

import (
	"bytes"
	"testing"
)

// TestSelfCoverage makes sure that the license texts match themselves.
// That is a mininum requirement.
func TestSelfCoverage(t *testing.T) {
	for _, l := range builtin.licenses {
		cov, ok := Cover(l.doc.text, Options{})
		if !ok {
			t.Errorf("no coverage for %s", l.name)
			continue
		}
		if cov.Percent != 100.0 {
			t.Errorf("%s got %.2f%% coverage", l.name, cov.Percent)
		}
	}
}

// TestApache_2_0_User tests that the "user" form of Apache 2.0 reports the license correctly.
func TestApache_2_0_User(t *testing.T) {
	text := findLicense("Apache-2.0-User").doc.text
	cov, ok := Cover(text, Options{})
	if !ok {
		t.Error("no coverage")
	}
	// Coverage should be 100%
	if cov.Percent != 100 {
		t.Errorf("coverage is %.2f%%, should be 100%%", cov.Percent)
	}
	if len(cov.Match) == 0 {
		t.Fatalf("expected a match; got none")
	}
	for _, m := range cov.Match {
		if m.Name != "Apache-2.0" || m.Percent != 100 {
			t.Errorf("expected license name %q at 100%%; got %q %.2f%%", "Apache-2.0", m.Name, m.Percent)
		}
	}
}

// TestMultiCoverage makes sure that concatenated license texts match themselves in sequence.
func TestMultiCoverage(t *testing.T) {
	mit := findLicense("MIT")
	apache := findLicense("Apache-2.0")
	bsd2 := findLicense("BSD-2-Clause")
	text := bytes.Join([][]byte{mit.doc.text, apache.doc.text, bsd2.doc.text},
		[]byte("\nHere is some intervening text\n"))
	cov, ok := Cover(text, Options{})
	if !ok {
		t.Error("no coverage")
	}
	// Coverage should be >=98% (it's actually about 99%) but not 100% - almost the entire file is a license.
	if cov.Percent == 100 {
		t.Errorf("coverage is %.2g%%, should be less than 100%%", cov.Percent)
	}
	if cov.Percent < 98.0 {
		t.Errorf("coverage is %.2g%%, should be closer to 100%%", cov.Percent)
	}
	// But all three should match by 100% individually.
	for _, match := range cov.Match {
		if match.Percent != 100.0 {
			t.Errorf("%s got only %.2g%% coverage", match.Name, cov.Percent)
		}
	}
	// Matches should be in the right order and not overlap.
	if len(cov.Match) != 3 {
		t.Fatalf("got %d matches; expect 3", len(cov.Match))
	}
	checkMatch(t, cov.Match[0], "MIT", -1)
	checkMatch(t, cov.Match[1], "Apache-2.0", cov.Match[0].End)
	checkMatch(t, cov.Match[2], "BSD-2-Clause", cov.Match[1].End)
}

func findLicense(name string) license {
	for _, l := range builtin.licenses {
		if l.name == name {
			return l
		}
	}
	panic("no license named " + name)
}

func checkMatch(t *testing.T, m Match, name string, prevEnd int) {
	t.Helper()
	if m.Name != name {
		t.Errorf("got %q; expected %q", m.Name, name)
		return
	}
	lic := findLicense(name)
	length := len(lic.doc.text)
	// There is some fudge factor in the match lengths because of terminal spaces, so be forgiving.
	min, max := length-5, length
	if name == "BSD-2-Clause" {
		min = length - 40
	}
	if n := m.End - m.Start; n < min || max < n {
		t.Errorf("match for %s is %d bytes long; expected %d", name, m.End-m.Start, length)
	}
	if m.End <= m.Start {
		t.Errorf("match for %s starts at %d after it ends at %d", name, m.Start, m.End)
	}
	if m.Start <= prevEnd {
		t.Errorf("match for %s starts at %d before previous ends at %d", name, m.Start, prevEnd)
	}
}

func TestWordOffset(t *testing.T) {
	mit := findLicense("MIT") // A reasonably short one.
	doc := mit.doc
	for i, byteOff := range doc.byteOff {
		wordOff := mit.doc.wordOffset(int(byteOff))
		if wordOff != i {
			t.Fatalf("%d: got word %d; expected %d", i, wordOff, i)
		}
	}
	wordOff := doc.wordOffset(len(doc.text))
	if wordOff != len(doc.words) {
		t.Fatalf("%d: got word %d; expected %d", len(doc.words), wordOff, len(doc.words))
	}
}
<commit_msg>licensecheck: remove hack in TestMultiCoverage<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package licensecheck

import (
	"bytes"
	"testing"
)

// TestSelfCoverage makes sure that the license texts match themselves.
// That is a mininum requirement.
func TestSelfCoverage(t *testing.T) {
	for _, l := range builtin.licenses {
		cov, ok := Cover(l.doc.text, Options{})
		if !ok {
			t.Errorf("no coverage for %s", l.name)
			continue
		}
		if cov.Percent != 100.0 {
			t.Errorf("%s got %.2f%% coverage", l.name, cov.Percent)
		}
	}
}

// TestApache_2_0_User tests that the "user" form of Apache 2.0 reports the license correctly.
func TestApache_2_0_User(t *testing.T) {
	text := findLicense("Apache-2.0-User").doc.text
	cov, ok := Cover(text, Options{})
	if !ok {
		t.Error("no coverage")
	}
	// Coverage should be 100%
	if cov.Percent != 100 {
		t.Errorf("coverage is %.2f%%, should be 100%%", cov.Percent)
	}
	if len(cov.Match) == 0 {
		t.Fatalf("expected a match; got none")
	}
	for _, m := range cov.Match {
		if m.Name != "Apache-2.0" || m.Percent != 100 {
			t.Errorf("expected license name %q at 100%%; got %q %.2f%%", "Apache-2.0", m.Name, m.Percent)
		}
	}
}

// TestMultiCoverage makes sure that concatenated license texts match themselves in sequence.
func TestMultiCoverage(t *testing.T) {
	mit := findLicense("MIT")
	apache := findLicense("Apache-2.0")
	bsd2 := findLicense("BSD-2-Clause")
	text := bytes.Join([][]byte{mit.doc.text, apache.doc.text, bsd2.doc.text},
		[]byte("\nHere is some intervening text\n"))
	cov, ok := Cover(text, Options{})
	if !ok {
		t.Error("no coverage")
	}
	// Coverage should be >=98% (it's actually about 99%) but not 100% - almost the entire file is a license.
	if cov.Percent == 100 {
		t.Errorf("coverage is %.2g%%, should be less than 100%%", cov.Percent)
	}
	if cov.Percent < 98.0 {
		t.Errorf("coverage is %.2g%%, should be closer to 100%%", cov.Percent)
	}
	// But all three should match by 100% individually.
	for _, match := range cov.Match {
		if match.Percent != 100.0 {
			t.Errorf("%s got only %.2g%% coverage", match.Name, cov.Percent)
		}
	}
	// Matches should be in the right order and not overlap.
	if len(cov.Match) != 3 {
		t.Fatalf("got %d matches; expect 3", len(cov.Match))
	}
	checkMatch(t, cov.Match[0], "MIT", -1)
	checkMatch(t, cov.Match[1], "Apache-2.0", cov.Match[0].End)
	checkMatch(t, cov.Match[2], "BSD-2-Clause", cov.Match[1].End)
}

func findLicense(name string) license {
	for _, l := range builtin.licenses {
		if l.name == name {
			return l
		}
	}
	panic("no license named " + name)
}

func checkMatch(t *testing.T, m Match, name string, prevEnd int) {
	t.Helper()
	if m.Name != name {
		t.Errorf("got %q; expected %q", m.Name, name)
		return
	}
	lic := findLicense(name)
	length := len(lic.doc.text)
	// There is some fudge factor in the match lengths because of terminal spaces, so be forgiving.
	min, max := length-5, length
	if n := m.End - m.Start; n < min || max < n {
		t.Errorf("match for %s is %d bytes long; expected %d", name, m.End-m.Start, length)
	}
	if m.End <= m.Start {
		t.Errorf("match for %s starts at %d after it ends at %d", name, m.Start, m.End)
	}
	if m.Start <= prevEnd {
		t.Errorf("match for %s starts at %d before previous ends at %d", name, m.Start, prevEnd)
	}
}

func TestWordOffset(t *testing.T) {
	mit := findLicense("MIT") // A reasonably short one.
	doc := mit.doc
	for i, byteOff := range doc.byteOff {
		wordOff := mit.doc.wordOffset(int(byteOff))
		if wordOff != i {
			t.Fatalf("%d: got word %d; expected %d", i, wordOff, i)
		}
	}
	wordOff := doc.wordOffset(len(doc.text))
	if wordOff != len(doc.words) {
		t.Fatalf("%d: got word %d; expected %d", len(doc.words), wordOff, len(doc.words))
	}
}
<|endoftext|>


<commit_before><commit_msg>Modified test routines to use new directory structure.<commit_after><|endoftext|>


<commit_before>package main

import (
	"context"
	"log"
	"os/user"

	"github.com/pkg/errors"

	pb "github.com/ContextLogic/eventmaster/proto"
)

func inject(ctx context.Context, c pb.EventMasterClient) error {
	u, err := user.Current()
	if err != nil {
		return errors.Wrap(err, "getting user")
	}
	dcs, err := c.GetDCs(ctx, &pb.EmptyRequest{})
	if err != nil {
		return errors.Wrap(err, "getting dcs")
	}
	topics, err := c.GetTopics(ctx, &pb.EmptyRequest{})
	if err != nil {
		return errors.Wrap(err, "getting topics")
	}

	for _, dc := range dcs.Results {
		for _, topic := range topics.Results {
			e := &pb.Event{
				DC:        dc.DCName,
				TopicName: topic.TopicName,
				Host:      "inject.emctl.i.wish.com",
				TagSet:    []string{"a", "b"},
				User:      u.Username,
			}
			resp, err := c.AddEvent(ctx, e)
			if err != nil {
				return errors.Wrap(err, "add event")
			}
			log.Printf("%v %v %v", resp.Id, dc.DCName, topic.TopicName)
		}
	}
	return nil
}
<commit_msg>Sort dc/topic before injection<commit_after>package main

import (
	"context"
	"log"
	"os/user"
	"sort"
	"time"

	"github.com/pkg/errors"

	pb "github.com/ContextLogic/eventmaster/proto"
)

func inject(ctx context.Context, c pb.EventMasterClient) error {
	u, err := user.Current()
	if err != nil {
		return errors.Wrap(err, "getting user")
	}
	dcs, err := c.GetDCs(ctx, &pb.EmptyRequest{})
	if err != nil {
		return errors.Wrap(err, "getting dcs")
	}
	topics, err := c.GetTopics(ctx, &pb.EmptyRequest{})
	if err != nil {
		return errors.Wrap(err, "getting topics")
	}

	ds, ts := []string{}, []string{}
	for _, dc := range dcs.Results {
		ds = append(ds, dc.DCName)
	}
	for _, tc := range topics.Results {
		ts = append(ts, tc.TopicName)
	}

	sort.Strings(ds)
	sort.Strings(ts)

	for _, dc := range ds {
		for _, topic := range ts {
			e := &pb.Event{
				DC:        dc,
				TopicName: topic,
				Host:      "inject.emctl.i.wish.com",
				TagSet:    []string{"a", "b"},
				User:      u.Username,
			}
			resp, err := c.AddEvent(ctx, e)
			if err != nil {
				return errors.Wrap(err, "add event")
			}
			log.Printf("%v %v %v", resp.Id, dc, topic)
			time.Sleep(2 * time.Second)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/drbig/ricons"
)

type format struct {
	mime string
	fmt  ricons.Format
}

var (
	fAddr  string
	fGens  bool
	fBound int
	gens   map[string]string
	fmts   map[string]format
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.StringVar(&fAddr, "a", ":3232", "server bind address")
	flag.BoolVar(&fGens, "l", false, "show generators and exit")
	flag.IntVar(&fBound, "b", 256, "image bound (e.g. max 256x256)")

	gens = make(map[string]string, len(ricons.Registry))
	for k, v := range ricons.Registry {
		gens[k] = v.String()
	}

	fmts = map[string]format{
		"png":  format{"image/png", ricons.PNG},
		"gif":  format{"image/gif", ricons.GIF},
		"jpeg": format{"image/jpeg", ricons.JPEG},
	}
}

func main() {
	flag.Parse()
	if fGens {
		for _, v := range gens {
			fmt.Println(v)
		}
		os.Exit(0)
	}

	http.HandleFunc("/", handleIcon)
	log.Println("HTTP server started at", fAddr)
	log.Fatalln(http.ListenAndServe(fAddr, nil))
}

func handleIcon(w http.ResponseWriter, req *http.Request) {
	log.Println(req.RequestURI)

	w.Header().Set("Content-Type", "application/json")
	parts := strings.Split(req.URL.Path, "/")
	if len(parts) != 6 {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "wrong request URI"}`))
		return
	}
	g, ok := ricons.Registry[parts[2]]
	if !ok {
		w.WriteHeader(http.StatusNotImplemented)
		w.Write([]byte(`{"success": false, "msg": "generator not found"}`))
		return
	}
	f, ok := fmts[parts[3]]
	if !ok {
		w.WriteHeader(http.StatusNotImplemented)
		w.Write([]byte(`{"success": false, "msg": "unknown image format"}`))
		return
	}
	wi, err := strconv.Atoi(parts[4])
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "error parsing image width"}`))
		return

	}
	if wi < 1 || wi > fBound {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "image width out of range"}`))
		return
	}
	h, err := strconv.Atoi(parts[5])
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "error parsing image height"}`))
		return
	}
	if h < 1 || h > fBound {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "image height out of range"}`))
		return
	}
	i, err := g.NewIcon(wi, h)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`{"success": false, "msg": "error generating icon, sorry"}`))
		return
	}
	w.Header().Set("Content-Type", f.mime)
	w.WriteHeader(http.StatusOK)
	i.Encode(f.fmt, w)
	return
}
<commit_msg>cmd/riconsd: Add info URL<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/drbig/ricons"
)

type format struct {
	mime string
	fmt  ricons.Format
}

type Info struct {
	Generators map[string]string `json:"generators"`
	Formats    []string          `json:"formats"`
}

var (
	fAddr  string
	fGens  bool
	fBound int
	gens   map[string]string
	fmts   map[string]format
	info   []byte
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.StringVar(&fAddr, "a", ":3232", "server bind address")
	flag.BoolVar(&fGens, "l", false, "show generators and exit")
	flag.IntVar(&fBound, "b", 256, "image bound (e.g. max 256x256)")

	gens = make(map[string]string, len(ricons.Registry))
	for k, v := range ricons.Registry {
		gens[k] = v.String()
	}

	fmts = map[string]format{
		"png":  format{"image/png", ricons.PNG},
		"gif":  format{"image/gif", ricons.GIF},
		"jpeg": format{"image/jpeg", ricons.JPEG},
	}

	is := &Info{gens, make([]string, len(fmts))}
	i := 0
	for k, _ := range fmts {
		is.Formats[i] = k
		i++
	}
	in, err := json.Marshal(is)
	if err != nil {
		panic(fmt.Sprintf("can't marshal info: %s", err))
	}
	info = in
}

func main() {
	flag.Parse()
	if fGens {
		for _, v := range gens {
			fmt.Println(v)
		}
		os.Exit(0)
	}

	http.HandleFunc("/info.json", handleInfo)
	http.HandleFunc("/", handleIcon)
	log.Println("HTTP server started at", fAddr)
	log.Fatalln(http.ListenAndServe(fAddr, nil))
}

func handleInfo(w http.ResponseWriter, req *http.Request) {
	log.Println(req.RequestURI)

	w.Header().Set("Content-Type", "application/json")
	w.Write(info)
	return
}

func handleIcon(w http.ResponseWriter, req *http.Request) {
	log.Println(req.RequestURI)

	w.Header().Set("Content-Type", "application/json")
	parts := strings.Split(req.URL.Path, "/")
	if len(parts) != 6 {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "wrong request URI"}`))
		return
	}
	g, ok := ricons.Registry[parts[2]]
	if !ok {
		w.WriteHeader(http.StatusNotImplemented)
		w.Write([]byte(`{"success": false, "msg": "generator not found"}`))
		return
	}
	f, ok := fmts[parts[3]]
	if !ok {
		w.WriteHeader(http.StatusNotImplemented)
		w.Write([]byte(`{"success": false, "msg": "unknown image format"}`))
		return
	}
	wi, err := strconv.Atoi(parts[4])
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "error parsing image width"}`))
		return

	}
	if wi < 1 || wi > fBound {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "image width out of range"}`))
		return
	}
	h, err := strconv.Atoi(parts[5])
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "error parsing image height"}`))
		return
	}
	if h < 1 || h > fBound {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`{"success": false, "msg": "image height out of range"}`))
		return
	}
	i, err := g.NewIcon(wi, h)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`{"success": false, "msg": "error generating icon, sorry"}`))
		return
	}
	w.Header().Set("Content-Type", f.mime)
	w.WriteHeader(http.StatusOK)
	i.Encode(f.fmt, w)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	GardenClient "github.com/cloudfoundry-incubator/garden/client"
	GardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/turbine/api"
	"github.com/concourse/turbine/builder"
	"github.com/concourse/turbine/builder/inputs"
	"github.com/concourse/turbine/builder/outputs"
	"github.com/concourse/turbine/config"
	"github.com/concourse/turbine/resource"
	"github.com/concourse/turbine/scheduler"
	"github.com/concourse/turbine/snapshotter"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:4637",
	"listening address",
)

var peerAddr = flag.String(
	"peerAddr",
	"127.0.0.1:4637",
	"external address of the api server, used for callbacks",
)

var debugListenAddr = flag.String(
	"debugListenAddr",
	":4636",
	"port for the pprof debugger to listen on",
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	"tcp",
	"garden API connection network (unix or tcp)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	"127.0.0.1:7777",
	"garden API connection address",
)

var resourceTypes = flag.String(
	"resourceTypes",
	`{
		"git": "docker:///concourse/git-resource",
		"archive": "docker:///concourse/archive-resource",
		"docker-image": "docker:///concourse/docker-image-resource",
		"time": "docker:///concourse/time-resource",
		"s3": "docker:///concourse/s3-resource",
		"tracker": "docker:///concourse/tracker-resource",
		"semver": "docker:///concourse/semver-resource"
	}`,
	"map of resource type to its docker image",
)

var snapshotPath = flag.String(
	"snapshotPath",
	"/tmp/builds-snapshot.json",
	"path to file to store/load snapshots from",
)

func main() {
	flag.Parse()

	gardenClient := GardenClient.New(GardenConnection.New(
		*gardenNetwork,
		*gardenAddr,
	))

	logger := lager.NewLogger("turbine")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	resourceTypesMap := map[string]string{}
	err := json.Unmarshal([]byte(*resourceTypes), &resourceTypesMap)
	if err != nil {
		logger.Fatal("failed-to-parse-resource-types", err)
	}

	var resourceTypesConfig config.ResourceTypes
	for typ, image := range resourceTypesMap {
		resourceTypesConfig = append(resourceTypesConfig, config.ResourceType{
			Name:  typ,
			Image: image,
		})
	}

	resourceTracker := resource.NewTracker(resourceTypesConfig, gardenClient)

	builder := builder.NewBuilder(
		gardenClient,
		inputs.NewParallelFetcher(resourceTracker),
		outputs.NewParallelPerformer(resourceTracker),
	)

	scheduler := scheduler.NewScheduler(logger.Session("scheduler"), builder, scheduler.NewClock())

	drain := make(chan struct{})

	handler, err := api.New(logger.Session("api"), scheduler, resourceTracker, "http://"+*peerAddr, drain)
	if err != nil {
		logger.Fatal("failed-to-initialize-handler", err)
	}

	var pingErr error
	for i := 0; i < 10; i++ {
		pingErr = gardenClient.Ping()
		if pingErr == nil {
			break
		}

		time.Sleep(10 * time.Second)
	}

	if pingErr != nil {
		logger.Fatal("failed-to-ping-garden", err)
	}

	group := grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"api", http_server.New(*listenAddr, handler)},
		{"debug", http_server.New(*debugListenAddr, http.DefaultServeMux)},
		{"snapshotter", snapshotter.NewSnapshotter(logger.Session("snapshotter"), *snapshotPath, scheduler)},
		{"drainer", &drainer{drain}},
	})

	running := ifrit.Envoke(sigmon.New(group))

	logger.Info("listening", lager.Data{
		"api": *listenAddr,
	})

	err = <-running.Wait()
	if err == nil {
		logger.Info("exited")
	} else {
		logger.Error("failed", err)
		os.Exit(1)
	}
}

type drainer struct {
	drain chan<- struct{}
}

func (drainer *drainer) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)
	<-signals
	close(drainer.drain)
	return nil
}
<commit_msg>start the snapshotter before other Runners<commit_after>package main

import (
	"encoding/json"
	"flag"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	GardenClient "github.com/cloudfoundry-incubator/garden/client"
	GardenConnection "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"github.com/concourse/turbine/api"
	"github.com/concourse/turbine/builder"
	"github.com/concourse/turbine/builder/inputs"
	"github.com/concourse/turbine/builder/outputs"
	"github.com/concourse/turbine/config"
	"github.com/concourse/turbine/resource"
	"github.com/concourse/turbine/scheduler"
	"github.com/concourse/turbine/snapshotter"
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:4637",
	"listening address",
)

var peerAddr = flag.String(
	"peerAddr",
	"127.0.0.1:4637",
	"external address of the api server, used for callbacks",
)

var debugListenAddr = flag.String(
	"debugListenAddr",
	":4636",
	"port for the pprof debugger to listen on",
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	"tcp",
	"garden API connection network (unix or tcp)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	"127.0.0.1:7777",
	"garden API connection address",
)

var resourceTypes = flag.String(
	"resourceTypes",
	`{
		"git": "docker:///concourse/git-resource",
		"archive": "docker:///concourse/archive-resource",
		"docker-image": "docker:///concourse/docker-image-resource",
		"time": "docker:///concourse/time-resource",
		"s3": "docker:///concourse/s3-resource",
		"tracker": "docker:///concourse/tracker-resource",
		"semver": "docker:///concourse/semver-resource"
	}`,
	"map of resource type to its docker image",
)

var snapshotPath = flag.String(
	"snapshotPath",
	"/tmp/builds-snapshot.json",
	"path to file to store/load snapshots from",
)

func main() {
	flag.Parse()

	gardenClient := GardenClient.New(GardenConnection.New(
		*gardenNetwork,
		*gardenAddr,
	))

	logger := lager.NewLogger("turbine")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	resourceTypesMap := map[string]string{}
	err := json.Unmarshal([]byte(*resourceTypes), &resourceTypesMap)
	if err != nil {
		logger.Fatal("failed-to-parse-resource-types", err)
	}

	var resourceTypesConfig config.ResourceTypes
	for typ, image := range resourceTypesMap {
		resourceTypesConfig = append(resourceTypesConfig, config.ResourceType{
			Name:  typ,
			Image: image,
		})
	}

	resourceTracker := resource.NewTracker(resourceTypesConfig, gardenClient)

	builder := builder.NewBuilder(
		gardenClient,
		inputs.NewParallelFetcher(resourceTracker),
		outputs.NewParallelPerformer(resourceTracker),
	)

	scheduler := scheduler.NewScheduler(logger.Session("scheduler"), builder, scheduler.NewClock())

	drain := make(chan struct{})

	handler, err := api.New(logger.Session("api"), scheduler, resourceTracker, "http://"+*peerAddr, drain)
	if err != nil {
		logger.Fatal("failed-to-initialize-handler", err)
	}

	var pingErr error
	for i := 0; i < 10; i++ {
		pingErr = gardenClient.Ping()
		if pingErr == nil {
			break
		}

		time.Sleep(10 * time.Second)
	}

	if pingErr != nil {
		logger.Fatal("failed-to-ping-garden", err)
	}

	parallel := grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"api", http_server.New(*listenAddr, handler)},
		{"debug", http_server.New(*debugListenAddr, http.DefaultServeMux)},
		{"drainer", &drainer{drain}},
	})

	// Snapshotter must start before the rest to avoid race conditions when
	// re-attaching to builds.
	script := grouper.NewOrdered(os.Interrupt, []grouper.Member{
		{"snapshotter", snapshotter.NewSnapshotter(logger.Session("snapshotter"), *snapshotPath, scheduler)},
		{"parallel", parallel},
	})

	running := ifrit.Envoke(sigmon.New(script))

	logger.Info("listening", lager.Data{
		"api": *listenAddr,
	})

	err = <-running.Wait()
	if err == nil {
		logger.Info("exited")
	} else {
		logger.Error("failed", err)
		os.Exit(1)
	}
}

type drainer struct {
	drain chan<- struct{}
}

func (drainer *drainer) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)
	<-signals
	close(drainer.drain)
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func displayWhatWillChange(rules []Rule) {
	updates := WillUpdateRulesAndTargets(rules)
	deletes := WillDeleteRulesAndTargets(rules)
	if len(updates) == 0 && len(deletes) == 0 {
		fmt.Println("No Changes")
	}
	if len(updates) > 0 {
		fmt.Println("Updates")
		for _, r := range updates {
			ShowWillUpdateFieldInRule(r)
			for _, t := range r.Targets {
				if t.NeedUpdate && !t.NeedDelete {
					ShowWillUpdateFieldInTarget(t)
				}
			}
		}
	}
	if len(deletes) > 0 {
		fmt.Println("Deletes")
		for _, r := range deletes {
			ShowWillDeleteRule(r)
			for _, t := range r.Targets {
				if t.NeedDelete {
					ShowWillDeleteTarget(t)
				}
			}
		}
	}
}

// return will be updated rules and targets
func WillUpdateRulesAndTargets(rules []Rule) []Rule {
	u := make([]Rule, 0)
	for _, rule := range rules {
		if rule.NeedUpdate && !rule.NeedDelete {
			u = append(u, rule)
		} else {
			for _, target := range rule.Targets {
				if target.NeedUpdate && !target.NeedDelete {
					u = append(u, rule)
					break
				}
			}
		}
	}
	return u
}

// return will be deleted rules and targets
func WillDeleteRulesAndTargets(rules []Rule) []Rule {
	d := make([]Rule, 0)
	for _, rule := range rules {
		if rule.NeedDelete {
			d = append(d, rule)
		} else {
			for _, target := range rule.Targets {
				if target.NeedDelete {
					d = append(d, rule)
					break
				}
			}
		}
	}
	return d
}

func ShowWillUpdateFieldInRule(rule Rule) {
	fmt.Printf("Rule: %s\n", rule.Name)
	if !CompareString(&rule.Name, rule.ActualRule.Name) {
		fmt.Printf("Name: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.Name), rule.Name)
	}
	if !CompareString(&rule.Description, rule.ActualRule.Description) {
		fmt.Printf("Description: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.Description), rule.Description)
	}
	if !CompareString(&rule.EventPattern, rule.ActualRule.EventPattern) {
		fmt.Printf("EventPattern: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.EventPattern), rule.EventPattern)
	}
	if !CompareString(&rule.RoleArn, rule.ActualRule.RoleArn) {
		fmt.Printf("RoleArn: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.RoleArn), rule.RoleArn)
	}
	if !CompareString(&rule.ScheduleExpression, rule.ActualRule.ScheduleExpression) {
		fmt.Printf("ScheduleExpression: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.ScheduleExpression), rule.ScheduleExpression)
	}
	if !CompareString(&rule.State, rule.ActualRule.State) {
		fmt.Printf("State: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.State), rule.State)
	}
}

func ShowWillUpdateFieldInTarget(target Target) {
	fmt.Printf("Target: %s\n", target.Arn)
	if !CompareString(&target.Arn, target.ActualTarget.Arn) {
		fmt.Printf("Arn: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Arn), target.Arn)
	}
	if !CompareString(&target.Id, target.ActualTarget.Id) {
		fmt.Printf("Id: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Id), target.Id)
	}
	if !CompareString(&target.Input, target.ActualTarget.Input) {
		fmt.Printf("Input: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Input), target.Input)
	}
	if !CompareString(&target.InputPath, target.ActualTarget.InputPath) {
		fmt.Printf("InputPath: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.InputPath), target.InputPath)
	}
}

func ShowWillDeleteRule(rule Rule) {
	if rule.NeedDelete {
		fmt.Printf("Rule: %s this will delete\n", *rule.ActualRule.Name)
	} else {
		fmt.Printf("Rule: %s\n", rule.Name)
	}
}

func ShowWillDeleteTarget(target Target) {
	fmt.Printf("Target: %s this will delete\n", *target.ActualTarget.Id)
}
<commit_msg>Add comments to functions in presenter.go<commit_after>package main

import "fmt"

func displayWhatWillChange(rules []Rule) {
	updates := WillUpdateRulesAndTargets(rules)
	deletes := WillDeleteRulesAndTargets(rules)
	if len(updates) == 0 && len(deletes) == 0 {
		fmt.Println("No Changes")
	}
	if len(updates) > 0 {
		fmt.Println("Updates")
		for _, r := range updates {
			ShowWillUpdateFieldInRule(r)
			for _, t := range r.Targets {
				if t.NeedUpdate && !t.NeedDelete {
					ShowWillUpdateFieldInTarget(t)
				}
			}
		}
	}
	if len(deletes) > 0 {
		fmt.Println("Deletes")
		for _, r := range deletes {
			ShowWillDeleteRule(r)
			for _, t := range r.Targets {
				if t.NeedDelete {
					ShowWillDeleteTarget(t)
				}
			}
		}
	}
}

// WillUpdateRulesAndTargets return will be updated rules and targets
func WillUpdateRulesAndTargets(rules []Rule) []Rule {
	u := make([]Rule, 0)
	for _, rule := range rules {
		if rule.NeedUpdate && !rule.NeedDelete {
			u = append(u, rule)
		} else {
			for _, target := range rule.Targets {
				if target.NeedUpdate && !target.NeedDelete {
					u = append(u, rule)
					break
				}
			}
		}
	}
	return u
}

// WillDeleteRulesAndTargets return will be deleted rules and targets
func WillDeleteRulesAndTargets(rules []Rule) []Rule {
	d := make([]Rule, 0)
	for _, rule := range rules {
		if rule.NeedDelete {
			d = append(d, rule)
		} else {
			for _, target := range rule.Targets {
				if target.NeedDelete {
					d = append(d, rule)
					break
				}
			}
		}
	}
	return d
}

// ShowWillUpdateFieldInRule print what will rule changes to stdout
func ShowWillUpdateFieldInRule(rule Rule) {
	fmt.Printf("Rule: %s\n", rule.Name)
	if !CompareString(&rule.Name, rule.ActualRule.Name) {
		fmt.Printf("Name: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.Name), rule.Name)
	}
	if !CompareString(&rule.Description, rule.ActualRule.Description) {
		fmt.Printf("Description: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.Description), rule.Description)
	}
	if !CompareString(&rule.EventPattern, rule.ActualRule.EventPattern) {
		fmt.Printf("EventPattern: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.EventPattern), rule.EventPattern)
	}
	if !CompareString(&rule.RoleArn, rule.ActualRule.RoleArn) {
		fmt.Printf("RoleArn: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.RoleArn), rule.RoleArn)
	}
	if !CompareString(&rule.ScheduleExpression, rule.ActualRule.ScheduleExpression) {
		fmt.Printf("ScheduleExpression: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.ScheduleExpression), rule.ScheduleExpression)
	}
	if !CompareString(&rule.State, rule.ActualRule.State) {
		fmt.Printf("State: %s \t->\t %s\n", NilSafeStr(rule.ActualRule.State), rule.State)
	}
}

// ShowWillUpdateFieldInTarget print what will target changes to stdout
func ShowWillUpdateFieldInTarget(target Target) {
	fmt.Printf("Target: %s\n", target.Arn)
	if !CompareString(&target.Arn, target.ActualTarget.Arn) {
		fmt.Printf("Arn: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Arn), target.Arn)
	}
	if !CompareString(&target.Id, target.ActualTarget.Id) {
		fmt.Printf("Id: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Id), target.Id)
	}
	if !CompareString(&target.Input, target.ActualTarget.Input) {
		fmt.Printf("Input: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.Input), target.Input)
	}
	if !CompareString(&target.InputPath, target.ActualTarget.InputPath) {
		fmt.Printf("InputPath: %s \t->\t %s\n", NilSafeStr(target.ActualTarget.InputPath), target.InputPath)
	}
}

// ShowWillDeleteRule print the rule will delete to stdout
func ShowWillDeleteRule(rule Rule) {
	if rule.NeedDelete {
		fmt.Printf("Rule: %s this will delete\n", *rule.ActualRule.Name)
	} else {
		fmt.Printf("Rule: %s\n", rule.Name)
	}
}

// ShowWillDeleteTarget print the target will delete to stdout
func ShowWillDeleteTarget(target Target) {
	fmt.Printf("Target: %s this will delete\n", *target.ActualTarget.Id)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/gorilla/mux"
	"github.com/weaveworks/mesh"

	"github.com/weaveworks/go-checkpoint"
	"github.com/weaveworks/weave/ipam"
	"github.com/weaveworks/weave/nameserver"
	"github.com/weaveworks/weave/net/address"
	weave "github.com/weaveworks/weave/router"
)

var rootTemplate = template.New("root").Funcs(map[string]interface{}{
	"countDNSEntries": func(entries []nameserver.EntryStatus) int {
		count := 0
		for _, entry := range entries {
			if entry.Tombstone == 0 {
				count++
			}
		}
		return count
	},
	"printList": func(list []string) string {
		if len(list) == 0 {
			return "none"
		}
		return strings.Join(list, ", ")
	},
	"printIPAMRanges": func(router weave.NetworkRouterStatus, status ipam.Status) string {
		var buffer bytes.Buffer

		type stats struct {
			ips       uint32
			nickname  string
			reachable bool
		}

		peerStats := make(map[string]*stats)

		for _, entry := range status.Entries {
			s, found := peerStats[entry.Peer]
			if !found {
				s = &stats{nickname: entry.Nickname, reachable: entry.IsKnownPeer}
				peerStats[entry.Peer] = s
			}
			s.ips += entry.Size
		}

		printOwned := func(name string, nickName string, reachable bool, ips uint32) {
			reachableStr := ""
			if !reachable {
				reachableStr = "- unreachable!"
			}
			percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs)

			displayName := name + "(" + nickName + ")"
			fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n",
				displayName, ips, percentageRanges, reachableStr)
		}

		// print the local info first
		if ourStats := peerStats[router.Name]; ourStats != nil {
			printOwned(router.Name, ourStats.nickname, true, ourStats.ips)
		}

		// and then the rest
		for peer, stats := range peerStats {
			if peer != router.Name {
				printOwned(peer, stats.nickname, stats.reachable, stats.ips)
			}
		}

		return buffer.String()
	},
	"allIPAMOwnersUnreachable": func(status ipam.Status) bool {
		for _, entry := range status.Entries {
			if entry.Size > 0 && entry.IsKnownPeer {
				return false
			}
		}
		return true
	},
	"printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string {
		counts := make(map[string]int)
		for _, conn := range conns {
			counts[conn.State]++
		}
		return printCounts(counts, []string{"established", "pending", "retrying", "failed", "connecting"})
	},
	"printPeerConnectionCounts": func(peers []mesh.PeerStatus) string {
		counts := make(map[string]int)
		for _, peer := range peers {
			for _, conn := range peer.Connections {
				if conn.Established {
					counts["established"]++
				} else {
					counts["pending"]++
				}
			}
		}
		return printCounts(counts, []string{"established", "pending"})
	},
	"printState": func(enabled bool) string {
		if enabled {
			return "enabled"
		}
		return "disabled"
	},
	"trimSuffix": strings.TrimSuffix,
})

// Print counts in a specified order
func printCounts(counts map[string]int, keys []string) string {
	var stringCounts []string
	for _, key := range keys {
		if count, ok := counts[key]; ok {
			stringCounts = append(stringCounts, fmt.Sprintf("%d %s", count, key))
		}
	}
	return strings.Join(stringCounts, ", ")
}

// Strip escaped newlines from template
func escape(template string) string {
	return strings.Replace(template, "\\\n", "", -1)
}

// Define a named template panicking on error
func defTemplate(name string, text string) *template.Template {
	return template.Must(rootTemplate.New(name).Parse(escape(text)))
}

var statusTemplate = defTemplate("status", `\
        Version: {{.Version}} ({{.VersionCheck}})

        Service: router
       Protocol: {{.Router.Protocol}} \
{{if eq .Router.ProtocolMinVersion .Router.ProtocolMaxVersion}}\
{{.Router.ProtocolMaxVersion}}\
{{else}}\
{{.Router.ProtocolMinVersion}}..{{.Router.ProtocolMaxVersion}}\
{{end}}
           Name: {{.Router.Name}}({{.Router.NickName}})
     Encryption: {{printState .Router.Encryption}}
  PeerDiscovery: {{printState .Router.PeerDiscovery}}
        Targets: {{len .Router.Targets}}
    Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
          Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
 TrustedSubnets: {{printList .Router.TrustedSubnets}}
{{if .IPAM}}\

        Service: ipam
{{if .IPAM.Entries}}\
{{if allIPAMOwnersUnreachable .IPAM}}\
         Status: all IP ranges owned by unreachable peers - use 'rmpeer' if they are dead
{{else if len .IPAM.PendingAllocates}}\
         Status: waiting for IP range grant from peers
{{else}}\
         Status: ready
{{end}}\
{{else if .IPAM.Paxos}}\
{{if .IPAM.Paxos.Elector}}\
         Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
{{else}}\
         Status: priming
{{end}}\
{{else}}\
         Status: idle
{{end}}\
          Range: {{.IPAM.Range}}
  DefaultSubnet: {{.IPAM.DefaultSubnet}}
{{end}}\
{{if .DNS}}\

        Service: dns
         Domain: {{.DNS.Domain}}
       Upstream: {{printList .DNS.Upstream}}
            TTL: {{.DNS.TTL}}
        Entries: {{countDNSEntries .DNS.Entries}}
{{end}}\
`)

var targetsTemplate = defTemplate("targetsTemplate", `\
{{range .Router.Targets}}{{.}}
{{end}}\
`)

var connectionsTemplate = defTemplate("connectionsTemplate", `\
{{range .Router.Connections}}\
{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}}
{{end}}\
`)

var peersTemplate = defTemplate("peers", `\
{{range .Router.Peers}}\
{{.Name}}({{.NickName}})
{{range .Connections}}\
   {{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} \
{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-37v" $nameNickName}} \
{{if .Established}}established{{else}}pending{{end}}
{{end}}\
{{end}}\
`)

var dnsEntriesTemplate = defTemplate("dnsEntries", `\
{{$domain := printf ".%v" .DNS.Domain}}\
{{range .DNS.Entries}}\
{{if eq .Tombstone 0}}\
{{$hostname := trimSuffix .Hostname $domain}}\
{{printf "%-12v" $hostname}} {{printf "%-15v" .Address}} {{printf "%12.12v" .ContainerID}} {{.Origin}}
{{end}}\
{{end}}\
`)

var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}}`)

type VersionCheck struct {
	Enabled     bool
	NewVersion  string
	NextCheckAt time.Time
}

func versionCheck() *VersionCheck {
	v := &VersionCheck{}
	if checkpoint.IsCheckDisabled() {
		return v
	}

	v.Enabled = true
	v.NewVersion = newVersion.Load().(string)
	v.NextCheckAt = checker.NextCheckAt()

	return v
}

func (v *VersionCheck) String() string {
	switch {
	case !v.Enabled:
		return "version check update disabled"
	case v.NewVersion != "":
		return fmt.Sprintf("version %s available - please upgrade!",
			v.NewVersion)
	default:
		return fmt.Sprintf("up to date; next check at %s",
			v.NextCheckAt.Format("2006/01/02 15:04:05"))
	}
}

type WeaveStatus struct {
	Version      string
	VersionCheck *VersionCheck              `json:"VersionCheck,omitempty"`
	Router       *weave.NetworkRouterStatus `json:"Router,omitempty"`
	IPAM         *ipam.Status               `json:"IPAM,omitempty"`
	DNS          *nameserver.Status         `json:"DNS,omitempty"`
}

func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRouter, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
	status := func() WeaveStatus {
		return WeaveStatus{
			version,
			versionCheck(),
			weave.NewNetworkRouterStatus(router),
			ipam.NewStatus(allocator, defaultSubnet),
			nameserver.NewStatus(ns, dnsserver)}
	}
	muxRouter.Methods("GET").Path("/report").Headers("Accept", "application/json").HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			json, err := json.MarshalIndent(status(), "", "    ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				Log.Error("Error during report marshalling: ", err)
				return
			}
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		})

	muxRouter.Methods("GET").Path("/report").Queries("format", "{format}").HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			funcs := template.FuncMap{
				"json": func(v interface{}) string {
					a, _ := json.Marshal(v)
					return string(a)
				},
			}
			formatTemplate, err := template.New("format").Funcs(funcs).Parse(mux.Vars(r)["format"])
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			if err := formatTemplate.Execute(w, status()); err != nil {
				http.Error(w, "error during template execution", http.StatusInternalServerError)
				Log.Error(err)
			}
		})

	defHandler := func(path string, template *template.Template) {
		muxRouter.Methods("GET").Path(path).HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				if err := template.Execute(w, status()); err != nil {
					http.Error(w, "error during template execution", http.StatusInternalServerError)
					Log.Error(err)
				}
			})
	}

	defHandler("/status", statusTemplate)
	defHandler("/status/targets", targetsTemplate)
	defHandler("/status/connections", connectionsTemplate)
	defHandler("/status/peers", peersTemplate)
	defHandler("/status/dns", dnsEntriesTemplate)
	defHandler("/status/ipam", ipamTemplate)
}
<commit_msg>Refactor: move 'unreachable' logic out of printing function<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/gorilla/mux"
	"github.com/weaveworks/mesh"

	"github.com/weaveworks/go-checkpoint"
	"github.com/weaveworks/weave/ipam"
	"github.com/weaveworks/weave/nameserver"
	"github.com/weaveworks/weave/net/address"
	weave "github.com/weaveworks/weave/router"
)

var rootTemplate = template.New("root").Funcs(map[string]interface{}{
	"countDNSEntries": func(entries []nameserver.EntryStatus) int {
		count := 0
		for _, entry := range entries {
			if entry.Tombstone == 0 {
				count++
			}
		}
		return count
	},
	"printList": func(list []string) string {
		if len(list) == 0 {
			return "none"
		}
		return strings.Join(list, ", ")
	},
	"printIPAMRanges": func(router weave.NetworkRouterStatus, status ipam.Status) string {
		var buffer bytes.Buffer

		type stats struct {
			ips       uint32
			nickname  string
			reachable bool
		}

		peerStats := make(map[string]*stats)

		for _, entry := range status.Entries {
			s, found := peerStats[entry.Peer]
			if !found {
				s = &stats{nickname: entry.Nickname, reachable: entry.IsKnownPeer}
				peerStats[entry.Peer] = s
			}
			s.ips += entry.Size
		}

		printOwned := func(name string, nickName string, info string, ips uint32) {
			percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs)

			displayName := name + "(" + nickName + ")"
			fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n",
				displayName, ips, percentageRanges, info)
		}

		// print the local info first
		if ourStats := peerStats[router.Name]; ourStats != nil {
			printOwned(router.Name, ourStats.nickname, "", ourStats.ips)
		}

		// and then the rest
		for peer, stats := range peerStats {
			if peer != router.Name {
				reachableStr := ""
				if !stats.reachable {
					reachableStr = "- unreachable!"
				}
				printOwned(peer, stats.nickname, reachableStr, stats.ips)
			}
		}

		return buffer.String()
	},
	"allIPAMOwnersUnreachable": func(status ipam.Status) bool {
		for _, entry := range status.Entries {
			if entry.Size > 0 && entry.IsKnownPeer {
				return false
			}
		}
		return true
	},
	"printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string {
		counts := make(map[string]int)
		for _, conn := range conns {
			counts[conn.State]++
		}
		return printCounts(counts, []string{"established", "pending", "retrying", "failed", "connecting"})
	},
	"printPeerConnectionCounts": func(peers []mesh.PeerStatus) string {
		counts := make(map[string]int)
		for _, peer := range peers {
			for _, conn := range peer.Connections {
				if conn.Established {
					counts["established"]++
				} else {
					counts["pending"]++
				}
			}
		}
		return printCounts(counts, []string{"established", "pending"})
	},
	"printState": func(enabled bool) string {
		if enabled {
			return "enabled"
		}
		return "disabled"
	},
	"trimSuffix": strings.TrimSuffix,
})

// Print counts in a specified order
func printCounts(counts map[string]int, keys []string) string {
	var stringCounts []string
	for _, key := range keys {
		if count, ok := counts[key]; ok {
			stringCounts = append(stringCounts, fmt.Sprintf("%d %s", count, key))
		}
	}
	return strings.Join(stringCounts, ", ")
}

// Strip escaped newlines from template
func escape(template string) string {
	return strings.Replace(template, "\\\n", "", -1)
}

// Define a named template panicking on error
func defTemplate(name string, text string) *template.Template {
	return template.Must(rootTemplate.New(name).Parse(escape(text)))
}

var statusTemplate = defTemplate("status", `\
        Version: {{.Version}} ({{.VersionCheck}})

        Service: router
       Protocol: {{.Router.Protocol}} \
{{if eq .Router.ProtocolMinVersion .Router.ProtocolMaxVersion}}\
{{.Router.ProtocolMaxVersion}}\
{{else}}\
{{.Router.ProtocolMinVersion}}..{{.Router.ProtocolMaxVersion}}\
{{end}}
           Name: {{.Router.Name}}({{.Router.NickName}})
     Encryption: {{printState .Router.Encryption}}
  PeerDiscovery: {{printState .Router.PeerDiscovery}}
        Targets: {{len .Router.Targets}}
    Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
          Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
 TrustedSubnets: {{printList .Router.TrustedSubnets}}
{{if .IPAM}}\

        Service: ipam
{{if .IPAM.Entries}}\
{{if allIPAMOwnersUnreachable .IPAM}}\
         Status: all IP ranges owned by unreachable peers - use 'rmpeer' if they are dead
{{else if len .IPAM.PendingAllocates}}\
         Status: waiting for IP range grant from peers
{{else}}\
         Status: ready
{{end}}\
{{else if .IPAM.Paxos}}\
{{if .IPAM.Paxos.Elector}}\
         Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
{{else}}\
         Status: priming
{{end}}\
{{else}}\
         Status: idle
{{end}}\
          Range: {{.IPAM.Range}}
  DefaultSubnet: {{.IPAM.DefaultSubnet}}
{{end}}\
{{if .DNS}}\

        Service: dns
         Domain: {{.DNS.Domain}}
       Upstream: {{printList .DNS.Upstream}}
            TTL: {{.DNS.TTL}}
        Entries: {{countDNSEntries .DNS.Entries}}
{{end}}\
`)

var targetsTemplate = defTemplate("targetsTemplate", `\
{{range .Router.Targets}}{{.}}
{{end}}\
`)

var connectionsTemplate = defTemplate("connectionsTemplate", `\
{{range .Router.Connections}}\
{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}}
{{end}}\
`)

var peersTemplate = defTemplate("peers", `\
{{range .Router.Peers}}\
{{.Name}}({{.NickName}})
{{range .Connections}}\
   {{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} \
{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-37v" $nameNickName}} \
{{if .Established}}established{{else}}pending{{end}}
{{end}}\
{{end}}\
`)

var dnsEntriesTemplate = defTemplate("dnsEntries", `\
{{$domain := printf ".%v" .DNS.Domain}}\
{{range .DNS.Entries}}\
{{if eq .Tombstone 0}}\
{{$hostname := trimSuffix .Hostname $domain}}\
{{printf "%-12v" $hostname}} {{printf "%-15v" .Address}} {{printf "%12.12v" .ContainerID}} {{.Origin}}
{{end}}\
{{end}}\
`)

var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}}`)

type VersionCheck struct {
	Enabled     bool
	NewVersion  string
	NextCheckAt time.Time
}

func versionCheck() *VersionCheck {
	v := &VersionCheck{}
	if checkpoint.IsCheckDisabled() {
		return v
	}

	v.Enabled = true
	v.NewVersion = newVersion.Load().(string)
	v.NextCheckAt = checker.NextCheckAt()

	return v
}

func (v *VersionCheck) String() string {
	switch {
	case !v.Enabled:
		return "version check update disabled"
	case v.NewVersion != "":
		return fmt.Sprintf("version %s available - please upgrade!",
			v.NewVersion)
	default:
		return fmt.Sprintf("up to date; next check at %s",
			v.NextCheckAt.Format("2006/01/02 15:04:05"))
	}
}

type WeaveStatus struct {
	Version      string
	VersionCheck *VersionCheck              `json:"VersionCheck,omitempty"`
	Router       *weave.NetworkRouterStatus `json:"Router,omitempty"`
	IPAM         *ipam.Status               `json:"IPAM,omitempty"`
	DNS          *nameserver.Status         `json:"DNS,omitempty"`
}

func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRouter, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
	status := func() WeaveStatus {
		return WeaveStatus{
			version,
			versionCheck(),
			weave.NewNetworkRouterStatus(router),
			ipam.NewStatus(allocator, defaultSubnet),
			nameserver.NewStatus(ns, dnsserver)}
	}
	muxRouter.Methods("GET").Path("/report").Headers("Accept", "application/json").HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			json, err := json.MarshalIndent(status(), "", "    ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				Log.Error("Error during report marshalling: ", err)
				return
			}
			w.Header().Set("Content-Type", "application/json")
			w.Write(json)
		})

	muxRouter.Methods("GET").Path("/report").Queries("format", "{format}").HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			funcs := template.FuncMap{
				"json": func(v interface{}) string {
					a, _ := json.Marshal(v)
					return string(a)
				},
			}
			formatTemplate, err := template.New("format").Funcs(funcs).Parse(mux.Vars(r)["format"])
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			if err := formatTemplate.Execute(w, status()); err != nil {
				http.Error(w, "error during template execution", http.StatusInternalServerError)
				Log.Error(err)
			}
		})

	defHandler := func(path string, template *template.Template) {
		muxRouter.Methods("GET").Path(path).HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				if err := template.Execute(w, status()); err != nil {
					http.Error(w, "error during template execution", http.StatusInternalServerError)
					Log.Error(err)
				}
			})
	}

	defHandler("/status", statusTemplate)
	defHandler("/status/targets", targetsTemplate)
	defHandler("/status/connections", connectionsTemplate)
	defHandler("/status/peers", peersTemplate)
	defHandler("/status/dns", dnsEntriesTemplate)
	defHandler("/status/ipam", ipamTemplate)
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"errors"
	"sync"

	pb "github.com/coreos/etcd/raft/raftpb"
)

// ErrCompacted is returned by Storage.Entries/Compact when a requested
// index is unavailable because it predates the last snapshot.
var ErrCompacted = errors.New("requested index is unavailable due to compaction")

// ErrSnapOutOfDate is returned by Storage.CreateSnapshot when a requested
// index is older than the existing snapshot.
var ErrSnapOutOfDate = errors.New("requested index is older than the existing snapshot")

// ErrUnavailable is returned by Storage interface when the requested log entries
// are unavailable.
var ErrUnavailable = errors.New("requested entry at index is unavailable")

// ErrSnapshotTemporarilyUnavailable is returned by the Storage interface when the required
// snapshot is temporarily unavailable.
var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unavailable")

// Storage is an interface that may be implemented by the application
// to retrieve log entries from storage.
//
// If any Storage method returns an error, the raft instance will
// become inoperable and refuse to participate in elections; the
// application is responsible for cleanup and recovery in this case.
type Storage interface {
	// InitialState returns the saved HardState and ConfState information.
	InitialState() (pb.HardState, pb.ConfState, error)
	// Entries returns a slice of log entries in the range [lo,hi).
	// MaxSize limits the total size of the log entries returned, but
	// Entries returns at least one entry if any.
	Entries(lo, hi, maxSize uint64) ([]pb.Entry, error)
	// Term returns the term of entry i, which must be in the range
	// [FirstIndex()-1, LastIndex()]. The term of the entry before
	// FirstIndex is retained for matching purposes even though the
	// rest of that entry may not be available.
	Term(i uint64) (uint64, error)
	// LastIndex returns the index of the last entry in the log.
	LastIndex() (uint64, error)
	// FirstIndex returns the index of the first log entry that is
	// possibly available via Entries (older entries have been incorporated
	// into the latest Snapshot; if storage only contains the dummy entry the
	// first log entry is not available).
	FirstIndex() (uint64, error)
	// Snapshot returns the most recent snapshot.
	// If snapshot is temporarily unavailable, it should return ErrSnapshotTemporarilyUnavailable,
	// so raft state machine could know that Storage needs some time to prepare
	// snapshot and call Snapshot later.
	Snapshot() (pb.Snapshot, error)
}

// MemoryStorage implements the Storage interface backed by an
// in-memory array.
type MemoryStorage struct {
	// Protects access to all fields. Most methods of MemoryStorage are
	// run on the raft goroutine, but Append() is run on an application
	// goroutine.
	sync.Mutex

	hardState pb.HardState
	snapshot  pb.Snapshot
	// ents[i] has raft log position i+snapshot.Metadata.Index
	ents []pb.Entry
}

// NewMemoryStorage creates an empty MemoryStorage.
func NewMemoryStorage() *MemoryStorage {
	return &MemoryStorage{
		// When starting from scratch populate the list with a dummy entry at term zero.
		ents: make([]pb.Entry, 1),
	}
}

// InitialState implements the Storage interface.
func (ms *MemoryStorage) InitialState() (pb.HardState, pb.ConfState, error) {
	return ms.hardState, ms.snapshot.Metadata.ConfState, nil
}

// SetHardState saves the current HardState.
func (ms *MemoryStorage) SetHardState(st pb.HardState) error {
	ms.hardState = st
	return nil
}

// Entries implements the Storage interface.
func (ms *MemoryStorage) Entries(lo, hi, maxSize uint64) ([]pb.Entry, error) {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if lo <= offset {
		return nil, ErrCompacted
	}
	if hi > ms.lastIndex()+1 {
		raftLogger.Panicf("entries' hi(%d) is out of bound lastindex(%d)", hi, ms.lastIndex())
	}
	// only contains dummy entries.
	if len(ms.ents) == 1 {
		return nil, ErrUnavailable
	}

	ents := ms.ents[lo-offset : hi-offset]
	return limitSize(ents, maxSize), nil
}

// Term implements the Storage interface.
func (ms *MemoryStorage) Term(i uint64) (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if i < offset {
		return 0, ErrCompacted
	}
	return ms.ents[i-offset].Term, nil
}

// LastIndex implements the Storage interface.
func (ms *MemoryStorage) LastIndex() (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.lastIndex(), nil
}

func (ms *MemoryStorage) lastIndex() uint64 {
	return ms.ents[0].Index + uint64(len(ms.ents)) - 1
}

// FirstIndex implements the Storage interface.
func (ms *MemoryStorage) FirstIndex() (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.firstIndex(), nil
}

func (ms *MemoryStorage) firstIndex() uint64 {
	return ms.ents[0].Index + 1
}

// Snapshot implements the Storage interface.
func (ms *MemoryStorage) Snapshot() (pb.Snapshot, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.snapshot, nil
}

// ApplySnapshot overwrites the contents of this Storage object with
// those of the given snapshot.
func (ms *MemoryStorage) ApplySnapshot(snap pb.Snapshot) error {
	ms.Lock()
	defer ms.Unlock()

	// TODO: return ErrSnapOutOfDate?
	ms.snapshot = snap
	ms.ents = []pb.Entry{{Term: snap.Metadata.Term, Index: snap.Metadata.Index}}
	return nil
}

// CreateSnapshot makes a snapshot which can be retrieved with Snapshot() and
// can be used to reconstruct the state at that point.
// If any configuration changes have been made since the last compaction,
// the result of the last ApplyConfChange must be passed in.
func (ms *MemoryStorage) CreateSnapshot(i uint64, cs *pb.ConfState, data []byte) (pb.Snapshot, error) {
	ms.Lock()
	defer ms.Unlock()
	if i <= ms.snapshot.Metadata.Index {
		return pb.Snapshot{}, ErrSnapOutOfDate
	}

	offset := ms.ents[0].Index
	if i > ms.lastIndex() {
		raftLogger.Panicf("snapshot %d is out of bound lastindex(%d)", i, ms.lastIndex())
	}

	ms.snapshot.Metadata.Index = i
	ms.snapshot.Metadata.Term = ms.ents[i-offset].Term
	if cs != nil {
		ms.snapshot.Metadata.ConfState = *cs
	}
	ms.snapshot.Data = data
	return ms.snapshot, nil
}

// Compact discards all log entries prior to compactIndex.
// It is the application's responsibility to not attempt to compact an index
// greater than raftLog.applied.
func (ms *MemoryStorage) Compact(compactIndex uint64) error {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if compactIndex <= offset {
		return ErrCompacted
	}
	if compactIndex > ms.lastIndex() {
		raftLogger.Panicf("compact %d is out of bound lastindex(%d)", compactIndex, ms.lastIndex())
	}

	i := compactIndex - offset
	ents := make([]pb.Entry, 1, 1+uint64(len(ms.ents))-i)
	ents[0].Index = ms.ents[i].Index
	ents[0].Term = ms.ents[i].Term
	ents = append(ents, ms.ents[i+1:]...)
	ms.ents = ents
	return nil
}

// Append the new entries to storage.
// TODO (xiangli): ensure the entries are continuous and
// entries[0].Index > ms.entries[0].Index
func (ms *MemoryStorage) Append(entries []pb.Entry) error {
	ms.Lock()
	defer ms.Unlock()
	if len(entries) == 0 {
		return nil
	}
	first := ms.ents[0].Index + 1
	last := entries[0].Index + uint64(len(entries)) - 1

	// shortcut if there is no new entry.
	if last < first {
		return nil
	}
	// truncate compacted entries
	if first > entries[0].Index {
		entries = entries[first-entries[0].Index:]
	}

	offset := entries[0].Index - ms.ents[0].Index
	switch {
	case uint64(len(ms.ents)) > offset:
		ms.ents = append([]pb.Entry{}, ms.ents[:offset]...)
		ms.ents = append(ms.ents, entries...)
	case uint64(len(ms.ents)) == offset:
		ms.ents = append(ms.ents, entries...)
	default:
		raftLogger.Panicf("missing log entry [last: %d, append at: %d]",
			ms.lastIndex(), entries[0].Index)
	}
	return nil
}
<commit_msg>raft: len(entries) before Lock, use firstIndex<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"errors"
	"sync"

	pb "github.com/coreos/etcd/raft/raftpb"
)

// ErrCompacted is returned by Storage.Entries/Compact when a requested
// index is unavailable because it predates the last snapshot.
var ErrCompacted = errors.New("requested index is unavailable due to compaction")

// ErrSnapOutOfDate is returned by Storage.CreateSnapshot when a requested
// index is older than the existing snapshot.
var ErrSnapOutOfDate = errors.New("requested index is older than the existing snapshot")

// ErrUnavailable is returned by Storage interface when the requested log entries
// are unavailable.
var ErrUnavailable = errors.New("requested entry at index is unavailable")

// ErrSnapshotTemporarilyUnavailable is returned by the Storage interface when the required
// snapshot is temporarily unavailable.
var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unavailable")

// Storage is an interface that may be implemented by the application
// to retrieve log entries from storage.
//
// If any Storage method returns an error, the raft instance will
// become inoperable and refuse to participate in elections; the
// application is responsible for cleanup and recovery in this case.
type Storage interface {
	// InitialState returns the saved HardState and ConfState information.
	InitialState() (pb.HardState, pb.ConfState, error)
	// Entries returns a slice of log entries in the range [lo,hi).
	// MaxSize limits the total size of the log entries returned, but
	// Entries returns at least one entry if any.
	Entries(lo, hi, maxSize uint64) ([]pb.Entry, error)
	// Term returns the term of entry i, which must be in the range
	// [FirstIndex()-1, LastIndex()]. The term of the entry before
	// FirstIndex is retained for matching purposes even though the
	// rest of that entry may not be available.
	Term(i uint64) (uint64, error)
	// LastIndex returns the index of the last entry in the log.
	LastIndex() (uint64, error)
	// FirstIndex returns the index of the first log entry that is
	// possibly available via Entries (older entries have been incorporated
	// into the latest Snapshot; if storage only contains the dummy entry the
	// first log entry is not available).
	FirstIndex() (uint64, error)
	// Snapshot returns the most recent snapshot.
	// If snapshot is temporarily unavailable, it should return ErrSnapshotTemporarilyUnavailable,
	// so raft state machine could know that Storage needs some time to prepare
	// snapshot and call Snapshot later.
	Snapshot() (pb.Snapshot, error)
}

// MemoryStorage implements the Storage interface backed by an
// in-memory array.
type MemoryStorage struct {
	// Protects access to all fields. Most methods of MemoryStorage are
	// run on the raft goroutine, but Append() is run on an application
	// goroutine.
	sync.Mutex

	hardState pb.HardState
	snapshot  pb.Snapshot
	// ents[i] has raft log position i+snapshot.Metadata.Index
	ents []pb.Entry
}

// NewMemoryStorage creates an empty MemoryStorage.
func NewMemoryStorage() *MemoryStorage {
	return &MemoryStorage{
		// When starting from scratch populate the list with a dummy entry at term zero.
		ents: make([]pb.Entry, 1),
	}
}

// InitialState implements the Storage interface.
func (ms *MemoryStorage) InitialState() (pb.HardState, pb.ConfState, error) {
	return ms.hardState, ms.snapshot.Metadata.ConfState, nil
}

// SetHardState saves the current HardState.
func (ms *MemoryStorage) SetHardState(st pb.HardState) error {
	ms.hardState = st
	return nil
}

// Entries implements the Storage interface.
func (ms *MemoryStorage) Entries(lo, hi, maxSize uint64) ([]pb.Entry, error) {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if lo <= offset {
		return nil, ErrCompacted
	}
	if hi > ms.lastIndex()+1 {
		raftLogger.Panicf("entries' hi(%d) is out of bound lastindex(%d)", hi, ms.lastIndex())
	}
	// only contains dummy entries.
	if len(ms.ents) == 1 {
		return nil, ErrUnavailable
	}

	ents := ms.ents[lo-offset : hi-offset]
	return limitSize(ents, maxSize), nil
}

// Term implements the Storage interface.
func (ms *MemoryStorage) Term(i uint64) (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if i < offset {
		return 0, ErrCompacted
	}
	return ms.ents[i-offset].Term, nil
}

// LastIndex implements the Storage interface.
func (ms *MemoryStorage) LastIndex() (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.lastIndex(), nil
}

func (ms *MemoryStorage) lastIndex() uint64 {
	return ms.ents[0].Index + uint64(len(ms.ents)) - 1
}

// FirstIndex implements the Storage interface.
func (ms *MemoryStorage) FirstIndex() (uint64, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.firstIndex(), nil
}

func (ms *MemoryStorage) firstIndex() uint64 {
	return ms.ents[0].Index + 1
}

// Snapshot implements the Storage interface.
func (ms *MemoryStorage) Snapshot() (pb.Snapshot, error) {
	ms.Lock()
	defer ms.Unlock()
	return ms.snapshot, nil
}

// ApplySnapshot overwrites the contents of this Storage object with
// those of the given snapshot.
func (ms *MemoryStorage) ApplySnapshot(snap pb.Snapshot) error {
	ms.Lock()
	defer ms.Unlock()

	// TODO: return ErrSnapOutOfDate?
	ms.snapshot = snap
	ms.ents = []pb.Entry{{Term: snap.Metadata.Term, Index: snap.Metadata.Index}}
	return nil
}

// CreateSnapshot makes a snapshot which can be retrieved with Snapshot() and
// can be used to reconstruct the state at that point.
// If any configuration changes have been made since the last compaction,
// the result of the last ApplyConfChange must be passed in.
func (ms *MemoryStorage) CreateSnapshot(i uint64, cs *pb.ConfState, data []byte) (pb.Snapshot, error) {
	ms.Lock()
	defer ms.Unlock()
	if i <= ms.snapshot.Metadata.Index {
		return pb.Snapshot{}, ErrSnapOutOfDate
	}

	offset := ms.ents[0].Index
	if i > ms.lastIndex() {
		raftLogger.Panicf("snapshot %d is out of bound lastindex(%d)", i, ms.lastIndex())
	}

	ms.snapshot.Metadata.Index = i
	ms.snapshot.Metadata.Term = ms.ents[i-offset].Term
	if cs != nil {
		ms.snapshot.Metadata.ConfState = *cs
	}
	ms.snapshot.Data = data
	return ms.snapshot, nil
}

// Compact discards all log entries prior to compactIndex.
// It is the application's responsibility to not attempt to compact an index
// greater than raftLog.applied.
func (ms *MemoryStorage) Compact(compactIndex uint64) error {
	ms.Lock()
	defer ms.Unlock()
	offset := ms.ents[0].Index
	if compactIndex <= offset {
		return ErrCompacted
	}
	if compactIndex > ms.lastIndex() {
		raftLogger.Panicf("compact %d is out of bound lastindex(%d)", compactIndex, ms.lastIndex())
	}

	i := compactIndex - offset
	ents := make([]pb.Entry, 1, 1+uint64(len(ms.ents))-i)
	ents[0].Index = ms.ents[i].Index
	ents[0].Term = ms.ents[i].Term
	ents = append(ents, ms.ents[i+1:]...)
	ms.ents = ents
	return nil
}

// Append the new entries to storage.
// TODO (xiangli): ensure the entries are continuous and
// entries[0].Index > ms.entries[0].Index
func (ms *MemoryStorage) Append(entries []pb.Entry) error {
	if len(entries) == 0 {
		return nil
	}

	ms.Lock()
	defer ms.Unlock()

	first := ms.firstIndex()
	last := entries[0].Index + uint64(len(entries)) - 1

	// shortcut if there is no new entry.
	if last < first {
		return nil
	}
	// truncate compacted entries
	if first > entries[0].Index {
		entries = entries[first-entries[0].Index:]
	}

	offset := entries[0].Index - ms.ents[0].Index
	switch {
	case uint64(len(ms.ents)) > offset:
		ms.ents = append([]pb.Entry{}, ms.ents[:offset]...)
		ms.ents = append(ms.ents, entries...)
	case uint64(len(ms.ents)) == offset:
		ms.ents = append(ms.ents, entries...)
	default:
		raftLogger.Panicf("missing log entry [last: %d, append at: %d]",
			ms.lastIndex(), entries[0].Index)
	}
	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdCompare = &Command{
	Run:   compare,
	Usage: "compare [-u] [USER] [<START>...]<END>",
	Short: "Open a compare page on GitHub",
	Long: `Open a GitHub compare view page in the system's default web browser.
<START> to <END> are branch names, tag names, or commit SHA1s specifying
the range of history to compare. If a range with two dots ("a..b") is given,
it will be transformed into one with three dots. If <START> is omitted,
GitHub will compare against the base branch (the default is "master").
If <END> is omitted, GitHub compare view is opened for the current branch.
With "-u", outputs the URL rather than opening the browser.
`,
}

var (
	flagCompareURLOnly bool
)

func init() {
	cmdCompare.Flag.BoolVarP(&flagCompareURLOnly, "url-only", "u", false, "URL only")

	CmdRunner.Use(cmdCompare)
}

/*
  $ gh compare refactor
  > open https://github.com/CURRENT_REPO/compare/refactor

  $ gh compare 1.0..1.1
  > open https://github.com/CURRENT_REPO/compare/1.0...1.1

  $ gh compare -u other-user patch
  > open https://github.com/other-user/REPO/compare/patch
*/
func compare(command *Command, args *Args) {
	localRepo, err := github.LocalRepo()
	utils.Check(err)

	var (
		branch  *github.Branch
		project *github.Project
		r       string
	)

	branch, project, err = localRepo.RemoteBranchAndProject("")
	utils.Check(err)

	if args.IsParamsEmpty() {
		master := localRepo.MasterBranch()
		if master.ShortName() == branch.ShortName() {
			err = fmt.Errorf(command.FormattedUsage())
			utils.Check(err)
		} else {
			r = branch.ShortName()
		}
	} else {
		r = parseCompareRange(args.RemoveParam(args.ParamsSize() - 1))
		if args.IsParamsEmpty() {
			project, err = localRepo.CurrentProject()
			utils.Check(err)
		} else {
			project = github.NewProject(args.RemoveParam(args.ParamsSize()-1), "", "")
		}
	}

	r = strings.Replace(r, "/", ";", -1)
	subpage := utils.ConcatPaths("compare", r)
	url := project.WebURL("", "", subpage)
	launcher, err := utils.BrowserLauncher()
	utils.Check(err)

	if flagCompareURLOnly {
		args.Replace("echo", url)
	} else {
		args.Replace(launcher[0], "", launcher[1:]...)
		args.AppendParams(url)
	}
}

func parseCompareRange(r string) string {
	shaOrTag := fmt.Sprintf("((?:%s:)?\\w[\\w.-]+\\w)", OwnerRe)
	shaOrTagRange := fmt.Sprintf("^%s\\.\\.%s$", shaOrTag, shaOrTag)
	shaOrTagRangeRegexp := regexp.MustCompile(shaOrTagRange)
	return shaOrTagRangeRegexp.ReplaceAllString(r, "$1...$2")
}
<commit_msg>Compare compatible<commit_after>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdCompare = &Command{
	Run:   compare,
	Usage: "compare [-u] [USER] [<START>...]<END>",
	Short: "Open a compare page on GitHub",
	Long: `Open a GitHub compare view page in the system's default web browser.
<START> to <END> are branch names, tag names, or commit SHA1s specifying
the range of history to compare. If a range with two dots ("a..b") is given,
it will be transformed into one with three dots. If <START> is omitted,
GitHub will compare against the base branch (the default is "master").
If <END> is omitted, GitHub compare view is opened for the current branch.
With "-u", outputs the URL rather than opening the browser.
`,
}

var (
	flagCompareURLOnly bool
)

func init() {
	cmdCompare.Flag.BoolVarP(&flagCompareURLOnly, "url-only", "u", false, "URL only")

	CmdRunner.Use(cmdCompare)
}

/*
  $ gh compare refactor
  > open https://github.com/CURRENT_REPO/compare/refactor

  $ gh compare 1.0..1.1
  > open https://github.com/CURRENT_REPO/compare/1.0...1.1

  $ gh compare -u other-user patch
  > open https://github.com/other-user/REPO/compare/patch
*/
func compare(command *Command, args *Args) {
	localRepo, err := github.LocalRepo()
	utils.Check(err)

	var (
		branch  *github.Branch
		project *github.Project
		r       string
	)

	branch, project, err = localRepo.RemoteBranchAndProject("")
	utils.Check(err)

	if args.IsParamsEmpty() {
		if branch != nil && !branch.IsMaster() {
			r = branch.ShortName()
		} else {
			err = fmt.Errorf("Usage: hub compare [USER] [<START>...]<END>")
			utils.Check(err)
		}
	} else {
		r = parseCompareRange(args.RemoveParam(args.ParamsSize() - 1))
		if args.IsParamsEmpty() {
			project, err = localRepo.CurrentProject()
			utils.Check(err)
		} else {
			project = github.NewProject(args.RemoveParam(args.ParamsSize()-1), "", "")
		}
	}

	r = strings.Replace(r, "/", ";", -1)
	subpage := utils.ConcatPaths("compare", r)
	url := project.WebURL("", "", subpage)
	launcher, err := utils.BrowserLauncher()
	utils.Check(err)

	if flagCompareURLOnly {
		args.Replace("echo", url)
	} else {
		args.Replace(launcher[0], "", launcher[1:]...)
		args.AppendParams(url)
	}
}

func parseCompareRange(r string) string {
	shaOrTag := fmt.Sprintf("((?:%s:)?\\w[\\w.-]+\\w)", OwnerRe)
	shaOrTagRange := fmt.Sprintf("^%s\\.\\.%s$", shaOrTag, shaOrTag)
	shaOrTagRangeRegexp := regexp.MustCompile(shaOrTagRange)
	return shaOrTagRangeRegexp.ReplaceAllString(r, "$1...$2")
}
<|endoftext|>