<commit_before>// Package miniredis is a pure Go Redis test server, for use in Go unittests.
// There are no dependencies on system binaries, and every server you start
// will be empty.
//
// Start a server with `s, err := miniredis.Run()`.
// Stop it with `defer s.Close()`.
//
// Point your Redis client to `s.Addr()` or `s.Host(), s.Port()`.
//
// Set keys directly via s.Set(...) and similar commands, or use a Redis client.
//
// For direct use you can select a Redis database with either `s.Select(12);
// s.Get("foo")` or `s.DB(12).Get("foo")`.

package miniredis

import (
	"fmt"
	"net"
	"sync"

	"github.com/bsm/redeo"
)

type hashKey map[string]string
type listKey []string
type setKey map[string]struct{}

// RedisDB holds a single (numbered) Redis database.
type RedisDB struct {
	master        *sync.Mutex          // pointer to the lock in Miniredis
	id            int                  // db id
	keys          map[string]string    // Master map of keys with their type
	stringKeys    map[string]string    // GET/SET &c. keys
	hashKeys      map[string]hashKey   // MGET/MSET &c. keys
	listKeys      map[string]listKey   // LPUSH &c. keys
	setKeys       map[string]setKey    // SADD &c. keys
	sortedsetKeys map[string]sortedSet // ZADD &c. keys
	expire        map[string]int       // EXPIRE values
	keyVersion    map[string]uint      // used to watch values
}

// Miniredis is a Redis server implementation.
type Miniredis struct {
	sync.Mutex
	srv        *redeo.Server
	listenAddr string
	closed     chan struct{}
	listen     net.Listener
	dbs        map[int]*RedisDB
	selectedDB int // DB id used in the direct Get(), Set() &c.
}

type txCmd func(*redeo.Responder, *connCtx)

// database id + key combo
type dbKey struct {
	db  int
	key string
}

// connCtx has all state for a single connection.
type connCtx struct {
	selectedDB       int            // selected DB
	transaction      []txCmd        // transaction callbacks. Or nil.
	dirtyTransaction bool           // any error during QUEUEing.
	watch            map[dbKey]uint // WATCHed keys.
}

// NewMiniRedis makes a new, non-started, Miniredis object.
func NewMiniRedis() *Miniredis {
	return &Miniredis{
		closed: make(chan struct{}),
		dbs:    map[int]*RedisDB{},
	}
}

func newRedisDB(id int, l *sync.Mutex) RedisDB {
	return RedisDB{
		id:            id,
		master:        l,
		keys:          map[string]string{},
		stringKeys:    map[string]string{},
		hashKeys:      map[string]hashKey{},
		listKeys:      map[string]listKey{},
		setKeys:       map[string]setKey{},
		sortedsetKeys: map[string]sortedSet{},
		expire:        map[string]int{},
		keyVersion:    map[string]uint{},
	}
}

// Run creates and Start()s a Miniredis.
func Run() (*Miniredis, error) {
	m := NewMiniRedis()
	return m, m.Start()
}

// Restart restarts a Close()d server on the same port. Values will be
// preserved.
func (m *Miniredis) Restart() error {
	m.Lock()
	defer m.Unlock()

	l, err := listen(m.listenAddr)
	if err != nil {
		return err
	}
	m.listen = l

	go func() {
		m.srv.Serve(m.listen)
		m.closed <- struct{}{}
	}()

	return nil
}

// Start starts a server. It listens on a random port on localhost. See also
// Addr().
func (m *Miniredis) Start() error {
	m.Lock()
	defer m.Unlock()

	l, err := listen("127.0.0.1:0")
	if err != nil {
		return err
	}
	m.listen = l
	m.listenAddr = l.Addr().String()
	m.srv = redeo.NewServer(&redeo.Config{Addr: m.listenAddr})

	commandsConnection(m, m.srv)
	commandsGeneric(m, m.srv)
	commandsString(m, m.srv)
	commandsHash(m, m.srv)
	commandsList(m, m.srv)
	commandsSet(m, m.srv)
	commandsSortedSet(m, m.srv)
	commandsTransaction(m, m.srv)

	go func() {
		m.srv.Serve(m.listen)
		m.closed <- struct{}{}
	}()
	return nil
}

func listen(addr string) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		if l, err = net.Listen("tcp6", addr); err != nil {
			return nil, fmt.Errorf("failed to listen on a port: %v", err)
		}
	}
	return l, nil
}

// Close shuts down a Miniredis.
func (m *Miniredis) Close() {
	m.Lock()
	defer m.Unlock()
	if m.listen == nil {
		return
	}
	if m.listen.Close() != nil {
		return
	}
	m.srv.Close()
	<-m.closed
	m.listen = nil
}

// DB returns a DB by ID.
func (m *Miniredis) DB(i int) *RedisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(i)
}

// get DB. No locks!
func (m *Miniredis) db(i int) *RedisDB {
	if db, ok := m.dbs[i]; ok {
		return db
	}
	db := newRedisDB(i, &m.Mutex) // the DB has our lock.
	m.dbs[i] = &db
	return &db
}

// Addr returns '127.0.0.1:12345'. Can be given to a Dial(). See also Host()
// and Port(), which return the same things.
func (m *Miniredis) Addr() string {
	m.Lock()
	defer m.Unlock()
	return m.listenAddr
}

// Host returns the host part of Addr().
func (m *Miniredis) Host() string {
	m.Lock()
	defer m.Unlock()
	host, _, _ := net.SplitHostPort(m.listenAddr)
	return host
}

// Port returns the (random) port part of Addr().
func (m *Miniredis) Port() string {
	m.Lock()
	defer m.Unlock()
	_, port, _ := net.SplitHostPort(m.listenAddr)
	return port
}

// CommandCount returns the number of processed commands.
func (m *Miniredis) CommandCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.srv.Info().TotalCommands())
}

// CurrentConnectionCount returns the number of currently connected clients.
func (m *Miniredis) CurrentConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return m.srv.Info().ClientsLen()
}

// TotalConnectionCount returns the number of client connections since server start.
func (m *Miniredis) TotalConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.srv.Info().TotalConnections())
}

func getCtx(cl *redeo.Client) *connCtx {
	if cl.Ctx == nil {
		cl.Ctx = &connCtx{}
	}
	return cl.Ctx.(*connCtx)
}

func startTx(ctx *connCtx) {
	ctx.transaction = []txCmd{}
	ctx.dirtyTransaction = false
}

func stopTx(ctx *connCtx) {
	ctx.transaction = nil
	unwatch(ctx)
}

func inTx(ctx *connCtx) bool {
	return ctx.transaction != nil
}

func addTxCmd(ctx *connCtx, cb txCmd) {
	ctx.transaction = append(ctx.transaction, cb)
}

func dirtyTx(ctx *connCtx) bool {
	return ctx.dirtyTransaction
}

func watch(db *RedisDB, ctx *connCtx, key string) {
	if ctx.watch == nil {
		ctx.watch = map[dbKey]uint{}
	}
	ctx.watch[dbKey{db: db.id, key: key}] = db.keyVersion[key] // Can be 0.
}

func unwatch(ctx *connCtx) {
	ctx.watch = nil
}

// setDirty can be called even when not in an tx. Is an no-op then.
func setDirty(cl *redeo.Client) {
	if cl.Ctx == nil {
		// No transaction. Not relevant.
		return
	}
	getCtx(cl).dirtyTransaction = true
}
<commit_msg>fix package comment<commit_after>// Package miniredis is a pure Go Redis test server, for use in Go unittests.
// There are no dependencies on system binaries, and every server you start
// will be empty.
//
// Start a server with `s, err := miniredis.Run()`.
// Stop it with `defer s.Close()`.
//
// Point your Redis client to `s.Addr()` or `s.Host(), s.Port()`.
//
// Set keys directly via s.Set(...) and similar commands, or use a Redis client.
//
// For direct use you can select a Redis database with either `s.Select(12);
// s.Get("foo")` or `s.DB(12).Get("foo")`.
//
package miniredis

import (
	"fmt"
	"net"
	"sync"

	"github.com/bsm/redeo"
)

type hashKey map[string]string
type listKey []string
type setKey map[string]struct{}

// RedisDB holds a single (numbered) Redis database.
type RedisDB struct {
	master        *sync.Mutex          // pointer to the lock in Miniredis
	id            int                  // db id
	keys          map[string]string    // Master map of keys with their type
	stringKeys    map[string]string    // GET/SET &c. keys
	hashKeys      map[string]hashKey   // MGET/MSET &c. keys
	listKeys      map[string]listKey   // LPUSH &c. keys
	setKeys       map[string]setKey    // SADD &c. keys
	sortedsetKeys map[string]sortedSet // ZADD &c. keys
	expire        map[string]int       // EXPIRE values
	keyVersion    map[string]uint      // used to watch values
}

// Miniredis is a Redis server implementation.
type Miniredis struct {
	sync.Mutex
	srv        *redeo.Server
	listenAddr string
	closed     chan struct{}
	listen     net.Listener
	dbs        map[int]*RedisDB
	selectedDB int // DB id used in the direct Get(), Set() &c.
}

type txCmd func(*redeo.Responder, *connCtx)

// database id + key combo
type dbKey struct {
	db  int
	key string
}

// connCtx has all state for a single connection.
type connCtx struct {
	selectedDB       int            // selected DB
	transaction      []txCmd        // transaction callbacks. Or nil.
	dirtyTransaction bool           // any error during QUEUEing.
	watch            map[dbKey]uint // WATCHed keys.
}

// NewMiniRedis makes a new, non-started, Miniredis object.
func NewMiniRedis() *Miniredis {
	return &Miniredis{
		closed: make(chan struct{}),
		dbs:    map[int]*RedisDB{},
	}
}

func newRedisDB(id int, l *sync.Mutex) RedisDB {
	return RedisDB{
		id:            id,
		master:        l,
		keys:          map[string]string{},
		stringKeys:    map[string]string{},
		hashKeys:      map[string]hashKey{},
		listKeys:      map[string]listKey{},
		setKeys:       map[string]setKey{},
		sortedsetKeys: map[string]sortedSet{},
		expire:        map[string]int{},
		keyVersion:    map[string]uint{},
	}
}

// Run creates and Start()s a Miniredis.
func Run() (*Miniredis, error) {
	m := NewMiniRedis()
	return m, m.Start()
}

// Restart restarts a Close()d server on the same port. Values will be
// preserved.
func (m *Miniredis) Restart() error {
	m.Lock()
	defer m.Unlock()

	l, err := listen(m.listenAddr)
	if err != nil {
		return err
	}
	m.listen = l

	go func() {
		m.srv.Serve(m.listen)
		m.closed <- struct{}{}
	}()

	return nil
}

// Start starts a server. It listens on a random port on localhost. See also
// Addr().
func (m *Miniredis) Start() error {
	m.Lock()
	defer m.Unlock()

	l, err := listen("127.0.0.1:0")
	if err != nil {
		return err
	}
	m.listen = l
	m.listenAddr = l.Addr().String()
	m.srv = redeo.NewServer(&redeo.Config{Addr: m.listenAddr})

	commandsConnection(m, m.srv)
	commandsGeneric(m, m.srv)
	commandsString(m, m.srv)
	commandsHash(m, m.srv)
	commandsList(m, m.srv)
	commandsSet(m, m.srv)
	commandsSortedSet(m, m.srv)
	commandsTransaction(m, m.srv)

	go func() {
		m.srv.Serve(m.listen)
		m.closed <- struct{}{}
	}()
	return nil
}

func listen(addr string) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		if l, err = net.Listen("tcp6", addr); err != nil {
			return nil, fmt.Errorf("failed to listen on a port: %v", err)
		}
	}
	return l, nil
}

// Close shuts down a Miniredis.
func (m *Miniredis) Close() {
	m.Lock()
	defer m.Unlock()
	if m.listen == nil {
		return
	}
	if m.listen.Close() != nil {
		return
	}
	m.srv.Close()
	<-m.closed
	m.listen = nil
}

// DB returns a DB by ID.
func (m *Miniredis) DB(i int) *RedisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(i)
}

// get DB. No locks!
func (m *Miniredis) db(i int) *RedisDB {
	if db, ok := m.dbs[i]; ok {
		return db
	}
	db := newRedisDB(i, &m.Mutex) // the DB has our lock.
	m.dbs[i] = &db
	return &db
}

// Addr returns '127.0.0.1:12345'. Can be given to a Dial(). See also Host()
// and Port(), which return the same things.
func (m *Miniredis) Addr() string {
	m.Lock()
	defer m.Unlock()
	return m.listenAddr
}

// Host returns the host part of Addr().
func (m *Miniredis) Host() string {
	m.Lock()
	defer m.Unlock()
	host, _, _ := net.SplitHostPort(m.listenAddr)
	return host
}

// Port returns the (random) port part of Addr().
func (m *Miniredis) Port() string {
	m.Lock()
	defer m.Unlock()
	_, port, _ := net.SplitHostPort(m.listenAddr)
	return port
}

// CommandCount returns the number of processed commands.
func (m *Miniredis) CommandCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.srv.Info().TotalCommands())
}

// CurrentConnectionCount returns the number of currently connected clients.
func (m *Miniredis) CurrentConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return m.srv.Info().ClientsLen()
}

// TotalConnectionCount returns the number of client connections since server start.
func (m *Miniredis) TotalConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.srv.Info().TotalConnections())
}

func getCtx(cl *redeo.Client) *connCtx {
	if cl.Ctx == nil {
		cl.Ctx = &connCtx{}
	}
	return cl.Ctx.(*connCtx)
}

func startTx(ctx *connCtx) {
	ctx.transaction = []txCmd{}
	ctx.dirtyTransaction = false
}

func stopTx(ctx *connCtx) {
	ctx.transaction = nil
	unwatch(ctx)
}

func inTx(ctx *connCtx) bool {
	return ctx.transaction != nil
}

func addTxCmd(ctx *connCtx, cb txCmd) {
	ctx.transaction = append(ctx.transaction, cb)
}

func dirtyTx(ctx *connCtx) bool {
	return ctx.dirtyTransaction
}

func watch(db *RedisDB, ctx *connCtx, key string) {
	if ctx.watch == nil {
		ctx.watch = map[dbKey]uint{}
	}
	ctx.watch[dbKey{db: db.id, key: key}] = db.keyVersion[key] // Can be 0.
}

func unwatch(ctx *connCtx) {
	ctx.watch = nil
}

// setDirty can be called even when not in an tx. Is an no-op then.
func setDirty(cl *redeo.Client) {
	if cl.Ctx == nil {
		// No transaction. Not relevant.
		return
	}
	getCtx(cl).dirtyTransaction = true
}
<|endoftext|>


<commit_before>// Copyright 2014 go-dockerclient authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/fsouza/go-dockerclient/engine"
	"net/http"
	"net/url"
	"reflect"
	"testing"
)

type DockerVersion struct {
	Version   string
	GitCommit string
	GoVersion string
}

func TestVersion(t *testing.T) {
	body := `{
     "Version":"0.2.2",
     "GitCommit":"5a2a5cc+CHANGES",
     "GoVersion":"go1.0.3"
}`
	fakeRT := FakeRoundTripper{message: body, status: http.StatusOK}
	client := newTestClient(&fakeRT)
	expected := DockerVersion{
		Version:   "0.2.2",
		GitCommit: "5a2a5cc+CHANGES",
		GoVersion: "go1.0.3",
	}
	version, err := client.Version()
	if err != nil {
		t.Fatal(err)
	}

	if result := version.Get("Version"); result != expected.Version {
		t.Errorf("Version(): Wrong result. Want %#v. Got %#v.", expected.Version, version.Get("Version"))
	}
	if result := version.Get("GitCommit"); result != expected.GitCommit {
		t.Errorf("GitCommit(): Wrong result. Want %#v. Got %#v.", expected.GitCommit, version.Get("GitCommit"))
	}
	if result := version.Get("GoVersion"); result != expected.GoVersion {
		t.Errorf("GoVersion(): Wrong result. Want %#v. Got %#v.", expected.GoVersion, version.Get("GoVersion"))
	}
	req := fakeRT.requests[0]
	if req.Method != "GET" {
		t.Errorf("Version(): wrong request method. Want GET. Got %s.", req.Method)
	}
	u, _ := url.Parse(client.getURL("/version"))
	if req.URL.Path != u.Path {
		t.Errorf("Version(): wrong request path. Want %q. Got %q.", u.Path, req.URL.Path)
	}
}

func TestVersionError(t *testing.T) {
	fakeRT := &FakeRoundTripper{message: "internal error", status: http.StatusInternalServerError}
	client := newTestClient(fakeRT)
	version, err := client.Version()
	if version != nil {
		t.Errorf("Version(): expected <nil> value, got %#v.", version)
	}
	if err == nil {
		t.Error("Version(): unexpected <nil> error")
	}
}

func TestInfo(t *testing.T) {
	body := `{
     "Containers":11,
     "Images":16,
     "Debug":0,
     "NFd":11,
     "NGoroutines":21,
     "MemoryLimit":1,
     "SwapLimit":0
}`
	fakeRT := FakeRoundTripper{message: body, status: http.StatusOK}
	client := newTestClient(&fakeRT)
	expected := engine.Env{}
	expected.SetInt("Containers", 11)
	expected.SetInt("Images", 16)
	expected.SetBool("Debug", false)
	expected.SetInt("NFd", 11)
	expected.SetInt("NGoroutines", 21)
	expected.SetBool("MemoryLimit", true)
	expected.SetBool("SwapLimit", false)
	info, err := client.Info()
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(&expected, info) {
		t.Errorf("Info(): Wrong result. Want %#v. Got %#v.", expected, info)
	}
	req := fakeRT.requests[0]
	if req.Method != "GET" {
		t.Errorf("Info(): Wrong HTTP method. Want GET. Got %s.", req.Method)
	}
	u, _ := url.Parse(client.getURL("/info"))
	if req.URL.Path != u.Path {
		t.Errorf("Info(): Wrong request path. Want %q. Got %q.", u.Path, req.URL.Path)
	}
}

func TestInfoError(t *testing.T) {
	fakeRT := &FakeRoundTripper{message: "internal error", status: http.StatusInternalServerError}
	client := newTestClient(fakeRT)
	version, err := client.Info()
	if version != nil {
		t.Errorf("Info(): expected <nil> value, got %#v.", version)
	}
	if err == nil {
		t.Error("Info(): unexpected <nil> error")
	}
}
<commit_msg>misc_test: fix bug in assert<commit_after>// Copyright 2014 go-dockerclient authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/fsouza/go-dockerclient/engine"
	"net/http"
	"net/url"
	"reflect"
	"sort"
	"testing"
)

type DockerVersion struct {
	Version   string
	GitCommit string
	GoVersion string
}

func TestVersion(t *testing.T) {
	body := `{
     "Version":"0.2.2",
     "GitCommit":"5a2a5cc+CHANGES",
     "GoVersion":"go1.0.3"
}`
	fakeRT := FakeRoundTripper{message: body, status: http.StatusOK}
	client := newTestClient(&fakeRT)
	expected := DockerVersion{
		Version:   "0.2.2",
		GitCommit: "5a2a5cc+CHANGES",
		GoVersion: "go1.0.3",
	}
	version, err := client.Version()
	if err != nil {
		t.Fatal(err)
	}

	if result := version.Get("Version"); result != expected.Version {
		t.Errorf("Version(): Wrong result. Want %#v. Got %#v.", expected.Version, version.Get("Version"))
	}
	if result := version.Get("GitCommit"); result != expected.GitCommit {
		t.Errorf("GitCommit(): Wrong result. Want %#v. Got %#v.", expected.GitCommit, version.Get("GitCommit"))
	}
	if result := version.Get("GoVersion"); result != expected.GoVersion {
		t.Errorf("GoVersion(): Wrong result. Want %#v. Got %#v.", expected.GoVersion, version.Get("GoVersion"))
	}
	req := fakeRT.requests[0]
	if req.Method != "GET" {
		t.Errorf("Version(): wrong request method. Want GET. Got %s.", req.Method)
	}
	u, _ := url.Parse(client.getURL("/version"))
	if req.URL.Path != u.Path {
		t.Errorf("Version(): wrong request path. Want %q. Got %q.", u.Path, req.URL.Path)
	}
}

func TestVersionError(t *testing.T) {
	fakeRT := &FakeRoundTripper{message: "internal error", status: http.StatusInternalServerError}
	client := newTestClient(fakeRT)
	version, err := client.Version()
	if version != nil {
		t.Errorf("Version(): expected <nil> value, got %#v.", version)
	}
	if err == nil {
		t.Error("Version(): unexpected <nil> error")
	}
}

func TestInfo(t *testing.T) {
	body := `{
     "Containers":11,
     "Images":16,
     "Debug":0,
     "NFd":11,
     "NGoroutines":21,
     "MemoryLimit":1,
     "SwapLimit":0
}`
	fakeRT := FakeRoundTripper{message: body, status: http.StatusOK}
	client := newTestClient(&fakeRT)
	expected := engine.Env{}
	expected.SetInt("Containers", 11)
	expected.SetInt("Images", 16)
	expected.SetBool("Debug", false)
	expected.SetInt("NFd", 11)
	expected.SetInt("NGoroutines", 21)
	expected.SetBool("MemoryLimit", true)
	expected.SetBool("SwapLimit", false)
	info, err := client.Info()
	if err != nil {
		t.Fatal(err)
	}
	infoSlice := []string(*info)
	expectedSlice := []string(expected)
	sort.Strings(infoSlice)
	sort.Strings(expectedSlice)
	if !reflect.DeepEqual(expectedSlice, infoSlice) {
		t.Errorf("Info(): Wrong result.\nWant %#v.\nGot %#v.", expected, *info)
	}
	req := fakeRT.requests[0]
	if req.Method != "GET" {
		t.Errorf("Info(): Wrong HTTP method. Want GET. Got %s.", req.Method)
	}
	u, _ := url.Parse(client.getURL("/info"))
	if req.URL.Path != u.Path {
		t.Errorf("Info(): Wrong request path. Want %q. Got %q.", u.Path, req.URL.Path)
	}
}

func TestInfoError(t *testing.T) {
	fakeRT := &FakeRoundTripper{message: "internal error", status: http.StatusInternalServerError}
	client := newTestClient(fakeRT)
	version, err := client.Info()
	if version != nil {
		t.Errorf("Info(): expected <nil> value, got %#v.", version)
	}
	if err == nil {
		t.Error("Info(): unexpected <nil> error")
	}
}
<|endoftext|>


<commit_before>package sockjsclient

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/cookiejar"
	"sync"
)

// the implementation of New() doesn't have any error to be returned yet it
// returns, so it's totally safe to neglect the error
var cookieJar, _ = cookiejar.New(nil)

type XHRSession struct {
	mu sync.Mutex

	client     *http.Client
	sessionURL string
	sessionID  string
	messages   []string
	abort      chan struct{}
	opened     bool
	closed     bool
}

// NewXHRSession returns a new XHRSession, a SockJS client which supports
// xhr-polling
// http://sockjs.github.io/sockjs-protocol/sockjs-protocol-0.3.3.html#section-74
func NewXHRSession(opts *DialOptions) (*XHRSession, error) {
	client := opts.Client()

	// following /server_id/session_id should always be the same for every session
	serverID := threeDigits()
	sessionID := randomStringLength(20)
	sessionURL := opts.BaseURL + "/" + serverID + "/" + sessionID

	// start the initial session handshake
	sessionResp, err := client.Post(sessionURL+"/xhr", "text/plain", nil)
	if err != nil {
		return nil, err
	}
	defer sessionResp.Body.Close()

	if sessionResp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Starting new session failed. Want: %d Got: %d",
			http.StatusOK, sessionResp.StatusCode)
	}

	buf := bufio.NewReader(sessionResp.Body)
	frame, err := buf.ReadByte()
	if err != nil {
		return nil, err
	}

	if frame != 'o' {
		return nil, fmt.Errorf("can't start session, invalid frame: %s", frame)
	}

	return &XHRSession{
		client:     client,
		sessionID:  sessionID,
		sessionURL: sessionURL,
		opened:     true,
		abort:      make(chan struct{}, 1),
	}, nil
}

func (x *XHRSession) ID() string {
	return x.sessionID
}

func (x *XHRSession) Recv() (string, error) {
	type requestCanceler interface {
		CancelRequest(*http.Request)
	}

	// Return previously received messages if there is any.
	if len(x.messages) > 0 {
		msg := x.messages[0]
		x.messages = x.messages[1:]
		return msg, nil
	}

	if x.isClosed() {
		return "", ErrSessionClosed
	}

	// start to poll from the server until we receive something
	for {
		req, err := http.NewRequest("POST", x.sessionURL+"/xhr", nil)
		if err != nil {
			return "", fmt.Errorf("Receiving data failed: %s", err)
		}

		req.Header.Set("Content-Type", "text/plain")

		select {
		case <-x.abort:
			if cn, ok := x.client.Transport.(requestCanceler); ok {
				cn.CancelRequest(req)
			}

			return "", fmt.Errorf("session aborted by server")
		case res := <-x.do(req):
			if res.Error != nil {
				return "", fmt.Errorf("Receiving data failed: %s", res.Error)
			}

			msg, ok, err := x.handleResp(res.Response)
			if err != nil {
				return "", err
			}

			if ok {
				continue
			}

			return msg, nil
		}
	}
}

func (x *XHRSession) handleResp(resp *http.Response) (msg string, again bool, err error) {
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", false, fmt.Errorf("Receiving data failed. Want: %d Got: %d",
			http.StatusOK, resp.StatusCode)
	}

	buf := bufio.NewReader(resp.Body)

	// returns an error if buffer is empty
	frame, err := buf.ReadByte()
	if err != nil {
		return "", false, err
	}

	switch frame {
	case 'o':
		x.mu.Lock()
		x.opened = true
		x.mu.Unlock()

		return "", true, nil
	case 'a':
		// received an array of messages
		var messages []string
		if err := json.NewDecoder(buf).Decode(&messages); err != nil {
			return "", false, err
		}

		x.messages = append(x.messages, messages...)

		if len(x.messages) == 0 {
			return "", false, errors.New("no message")
		}

		// Return first message in slice, and remove it from the slice, so
		// next time the others will be picked
		msg := x.messages[0]
		x.messages = x.messages[1:]

		return msg, false, nil
	case 'h':
		// heartbeat received
		return "", true, nil
	case 'c':
		x.mu.Lock()
		x.opened = false
		x.mu.Unlock()

		return "", false, errors.New("session closed")
	default:
		return "", false, errors.New("invalid frame type")
	}
}

func (x *XHRSession) Send(frame string) error {
	if !x.isOpened() {
		return errors.New("session is not opened yet")
	}

	if x.isClosed() {
		return ErrSessionClosed
	}

	// Need's to be JSON encoded array of string messages (SockJS protocol
	// requirement)
	message := []string{frame}
	body, err := json.Marshal(&message)
	if err != nil {
		return err
	}

	resp, err := x.client.Post(x.sessionURL+"/xhr_send", "text/plain", bytes.NewReader(body))
	if err != nil {
		return err
	}

	if resp.StatusCode == http.StatusNotFound {
		x.Close(0, "") // invalidate session - see details: sockjs/sockjs-client#66

		return fmt.Errorf("XHR session does not exist: %s", x.sessionID)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("Sending data failed. Want: %d Got: %d",
			http.StatusOK, resp.StatusCode)
	}

	return nil
}

func (x *XHRSession) Close(status uint32, reason string) error {
	x.mu.Lock()
	x.opened = false
	x.closed = true
	x.mu.Unlock()

	select {
	case x.abort <- struct{}{}:
	default:
	}

	return nil
}

func (x *XHRSession) isOpened() bool {
	x.mu.Lock()
	defer x.mu.Unlock()

	return x.opened
}

func (x *XHRSession) isClosed() bool {
	x.mu.Lock()
	defer x.mu.Unlock()

	return x.closed
}

type doResult struct {
	Response *http.Response
	Error    error
}

func (x *XHRSession) do(req *http.Request) <-chan doResult {
	ch := make(chan doResult)

	go func() {
		var res doResult
		res.Response, res.Error = x.client.Do(req)
		ch <- res
	}()

	return ch
}
<commit_msg>kite: return proper error on Send when session is closed<commit_after>package sockjsclient

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/cookiejar"
	"sync"
)

// the implementation of New() doesn't have any error to be returned yet it
// returns, so it's totally safe to neglect the error
var cookieJar, _ = cookiejar.New(nil)

// XHRSession implements sockjs.Session with XHR transport.
type XHRSession struct {
	mu sync.Mutex

	client     *http.Client
	sessionURL string
	sessionID  string
	messages   []string
	abort      chan struct{}

	// TODO(rjeczalik): replace with single state field
	opened bool
	closed bool
}

// NewXHRSession returns a new XHRSession, a SockJS client which supports
// xhr-polling
// http://sockjs.github.io/sockjs-protocol/sockjs-protocol-0.3.3.html#section-74
func NewXHRSession(opts *DialOptions) (*XHRSession, error) {
	client := opts.Client()

	// following /server_id/session_id should always be the same for every session
	serverID := threeDigits()
	sessionID := randomStringLength(20)
	sessionURL := opts.BaseURL + "/" + serverID + "/" + sessionID

	// start the initial session handshake
	sessionResp, err := client.Post(sessionURL+"/xhr", "text/plain", nil)
	if err != nil {
		return nil, err
	}
	defer sessionResp.Body.Close()

	if sessionResp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Starting new session failed. Want: %d Got: %d",
			http.StatusOK, sessionResp.StatusCode)
	}

	buf := bufio.NewReader(sessionResp.Body)
	frame, err := buf.ReadByte()
	if err != nil {
		return nil, err
	}

	if frame != 'o' {
		return nil, fmt.Errorf("can't start session, invalid frame: %s", frame)
	}

	return &XHRSession{
		client:     client,
		sessionID:  sessionID,
		sessionURL: sessionURL,
		opened:     true,
		abort:      make(chan struct{}, 1),
	}, nil
}

func (x *XHRSession) ID() string {
	return x.sessionID
}

func (x *XHRSession) Recv() (string, error) {
	type requestCanceler interface {
		CancelRequest(*http.Request)
	}

	// Return previously received messages if there is any.
	if len(x.messages) > 0 {
		msg := x.messages[0]
		x.messages = x.messages[1:]
		return msg, nil
	}

	if x.isClosed() {
		return "", ErrSessionClosed
	}

	// start to poll from the server until we receive something
	for {
		req, err := http.NewRequest("POST", x.sessionURL+"/xhr", nil)
		if err != nil {
			return "", fmt.Errorf("Receiving data failed: %s", err)
		}

		req.Header.Set("Content-Type", "text/plain")

		select {
		case <-x.abort:
			if cn, ok := x.client.Transport.(requestCanceler); ok {
				cn.CancelRequest(req)
			}

			return "", fmt.Errorf("session aborted by server")
		case res := <-x.do(req):
			if res.Error != nil {
				return "", fmt.Errorf("Receiving data failed: %s", res.Error)
			}

			msg, ok, err := x.handleResp(res.Response)
			if err != nil {
				return "", err
			}

			if ok {
				continue
			}

			return msg, nil
		}
	}
}

func (x *XHRSession) handleResp(resp *http.Response) (msg string, again bool, err error) {
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", false, fmt.Errorf("Receiving data failed. Want: %d Got: %d",
			http.StatusOK, resp.StatusCode)
	}

	buf := bufio.NewReader(resp.Body)

	// returns an error if buffer is empty
	frame, err := buf.ReadByte()
	if err != nil {
		return "", false, err
	}

	switch frame {
	case 'o':
		x.mu.Lock()
		x.opened = true
		x.mu.Unlock()

		return "", true, nil
	case 'a':
		// received an array of messages
		var messages []string
		if err := json.NewDecoder(buf).Decode(&messages); err != nil {
			return "", false, err
		}

		x.messages = append(x.messages, messages...)

		if len(x.messages) == 0 {
			return "", false, errors.New("no message")
		}

		// Return first message in slice, and remove it from the slice, so
		// next time the others will be picked
		msg := x.messages[0]
		x.messages = x.messages[1:]

		return msg, false, nil
	case 'h':
		// heartbeat received
		return "", true, nil
	case 'c':
		x.mu.Lock()
		x.opened = false
		x.closed = true
		x.mu.Unlock()

		return "", false, ErrSessionClosed
	default:
		return "", false, errors.New("invalid frame type")
	}
}

func (x *XHRSession) Send(frame string) error {
	if x.isClosed() {
		return ErrSessionClosed
	}

	if !x.isOpened() {
		return errors.New("session is not opened yet")
	}

	// Need's to be JSON encoded array of string messages (SockJS protocol
	// requirement)
	message := []string{frame}
	body, err := json.Marshal(&message)
	if err != nil {
		return err
	}

	resp, err := x.client.Post(x.sessionURL+"/xhr_send", "text/plain", bytes.NewReader(body))
	if err != nil {
		return err
	}

	if resp.StatusCode == http.StatusNotFound {
		x.Close(0, "") // invalidate session - see details: sockjs/sockjs-client#66

		return fmt.Errorf("XHR session does not exist: %s", x.sessionID)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("Sending data failed. Want: %d Got: %d",
			http.StatusOK, resp.StatusCode)
	}

	return nil
}

func (x *XHRSession) Close(status uint32, reason string) error {
	x.mu.Lock()
	x.opened = false
	x.closed = true
	x.mu.Unlock()

	select {
	case x.abort <- struct{}{}:
	default:
	}

	return nil
}

func (x *XHRSession) isOpened() bool {
	x.mu.Lock()
	defer x.mu.Unlock()

	return x.opened
}

func (x *XHRSession) isClosed() bool {
	x.mu.Lock()
	defer x.mu.Unlock()

	return x.closed
}

type doResult struct {
	Response *http.Response
	Error    error
}

func (x *XHRSession) do(req *http.Request) <-chan doResult {
	ch := make(chan doResult)

	go func() {
		var res doResult
		res.Response, res.Error = x.client.Do(req)
		ch <- res
	}()

	return ch
}
<|endoftext|>


<commit_before>package goredis
<commit_msg>add some sorted sets test case<commit_after>package goredis

import (
	"testing"
)

func TestZAdd(t *testing.T) {
	r.Del("key")
	pairs := map[string]float64{
		"one":   1.0,
		"two":   1.0,
		"three": 3.0,
	}
	if n, err := r.ZAdd("key", pairs); err != nil {
		t.Error(err)
	} else if n != 3 {
		t.Fail()
	}
	if n, _ := r.ZAdd("key", map[string]float64{"two": 2.0}); n != 0 {
		t.Fail()
	}
}

func TestZCard(t *testing.T) {
	r.Del("key")
	pairs := map[string]float64{
		"one":   1.0,
		"two":   1.0,
		"three": 3.0,
	}
	r.ZAdd("key", pairs)
	if n, err := r.ZCard("key"); err != nil {
		t.Error(err)
	} else if n != 3 {
		t.Fail()
	}
}

func TestZCount(t *testing.T) {
	r.Del("key")
	pairs := map[string]float64{
		"one":   1.0,
		"two":   2.0,
		"three": 3.0,
	}
	r.ZAdd("key", pairs)
	if n, err := r.ZCount("key", "-inf", "+inf"); err != nil {
		t.Error(err)
	} else if n != 3 {
		t.Fail()
	}
	if n, _ := r.ZCount("key", "(1", "3"); n != 2 {
		t.Fail()
	}
}

func TestZIncrBy(t *testing.T) {
	r.Del("key")
	pairs := map[string]float64{
		"one":   1.0,
		"two":   1.0,
		"three": 3.0,
	}
	r.ZAdd("key", pairs)
	if n, err := r.ZIncrBy("key", 1.0, "two"); err != nil {
		t.Error(err)
	} else if n != 2.0 {
		t.Fail()
	}
}

func TestZRange(t *testing.T) {
	r.Del("key")
	pairs := map[string]float64{
		"one":   1.0,
		"two":   2.0,
		"three": 3.0,
	}
	r.ZAdd("key", pairs)
	if result, err := r.ZRange("key", 0, -1, false); err != nil {
		t.Error(err)
	} else if len(result) != 3 {
		t.Fail()
	} else if result[0] != "one" {
		t.Fail()
	}
	if result, err := r.ZRange("key", -2, -1, true); err != nil {
		t.Error(err)
	} else if len(result) != 4 {
		t.Fail()
	} else if result[0] != "two" {
		t.Fail()
	} else if result[1] != "2" {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/topicai/candy"
)

func TestCacheWithUpdate(t *testing.T) {
	mux := http.NewServeMux()
	srv := 0
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%05d", srv)
		srv++
	})

	ln, e := net.Listen("tcp", ":0")
	assert.Nil(t, e)
	go func() {
		candy.Must(http.Serve(ln, mux))
	}()

	url := fmt.Sprintf("http://%s/", ln.Addr())
	tmpdir, _ := ioutil.TempDir("", "")
	cache := New(url, path.Join(tmpdir, "cachefile"))

	v := string(cache.Get())
	for i := 0; i < 10; i++ {
		nv := string(cache.Get())
		assert.True(t, strings.Compare(v, nv) <= 0, fmt.Sprintf("%v should be <= %s", v, nv))
		v = nv
		time.Sleep(50 * time.Millisecond)
	}
}

func TestCacheWithConstantServer(t *testing.T) {
	mux := http.NewServeMux()
	srv := 0
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if srv == 0 {
			fmt.Fprintf(w, "%05d", srv)
			srv++
		} else {
			http.Error(w, "no longer works", http.StatusInternalServerError)
		}
	})

	ln, e := net.Listen("tcp", ":0")
	assert.Nil(t, e)
	go func() {
		candy.Must(http.Serve(ln, mux))
	}()

	url := fmt.Sprintf("http://%s/", ln.Addr())
	tmpdir, _ := ioutil.TempDir("", "")
	cache := New(url, path.Join(tmpdir, "cachefile"))

	for i := 0; i < 10; i++ {
		assert.Equal(t, "00000", string(cache.Get()))
		time.Sleep(50 * time.Millisecond)
	}
}
<commit_msg>An experiment with TraivsCI<commit_after>package cache

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"path"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/topicai/candy"
)

func TestCacheWithUpdate(t *testing.T) {
	mux := http.NewServeMux()
	srv := 0
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%05d", srv)
		srv++
	})

	ln, e := net.Listen("tcp", ":0")
	assert.Nil(t, e)
	go func() {
		candy.Must(http.Serve(ln, mux))
	}()

	url := fmt.Sprintf("http://%s/", ln.Addr())
	tmpdir, _ := ioutil.TempDir("", "")
	cache := New(url, path.Join(tmpdir, "cachefile"))

	for i := 0; i < 10; i++ {
		assert.Equal(t, fmt.Sprintf("%05d", i), string(cache.Get()))
		time.Sleep(550 * time.Millisecond)
	}
}

func TestCacheWithConstantServer(t *testing.T) {
	mux := http.NewServeMux()
	srv := 0
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if srv == 0 {
			fmt.Fprintf(w, "%05d", srv)
			srv++
		} else {
			http.Error(w, "no longer works", http.StatusInternalServerError)
		}
	})

	ln, e := net.Listen("tcp", ":0")
	assert.Nil(t, e)
	go func() {
		candy.Must(http.Serve(ln, mux))
	}()

	url := fmt.Sprintf("http://%s/", ln.Addr())
	tmpdir, _ := ioutil.TempDir("", "")
	cache := New(url, path.Join(tmpdir, "cachefile"))

	for i := 0; i < 10; i++ {
		assert.Equal(t, "00000", string(cache.Get()))
		time.Sleep(50 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/klog"
)

// DataDirectory provides utilities for initializing and backing up an
// etcd "data-dir" as well as managing a version.txt file to track the
// etcd server version and storage verion of the etcd data in the
// directory.
type DataDirectory struct {
	path        string
	versionFile *VersionFile
}

// OpenOrCreateDataDirectory opens a data directory, creating the directory
// if it doesn't not already exist.
func OpenOrCreateDataDirectory(path string) (*DataDirectory, error) {
	exists, err := exists(path)
	if err != nil {
		return nil, err
	}
	if !exists {
		klog.Infof("data directory '%s' does not exist, creating it", path)
		err := os.MkdirAll(path, 0777)
		if err != nil {
			return nil, fmt.Errorf("failed to create data directory %s: %v", path, err)
		}
	}
	versionFile := &VersionFile{
		path: filepath.Join(path, versionFilename),
	}
	return &DataDirectory{path, versionFile}, nil
}

// Initialize set the version.txt to the target version if the data
// directory is empty. If the data directory is non-empty, no
// version.txt file will be written since the actual version of etcd
// used to create the data is unknown.
func (d *DataDirectory) Initialize(target *EtcdVersionPair) error {
	isEmpty, err := d.IsEmpty()
	if err != nil {
		return err
	}
	if isEmpty {
		klog.Infof("data directory '%s' is empty, writing target version '%s' to version.txt", d.path, target)
		err = d.versionFile.Write(target)
		if err != nil {
			return fmt.Errorf("failed to write version.txt to '%s': %v", d.path, err)
		}
		return nil
	}
	return nil
}

// Backup creates a backup copy of data directory.
func (d *DataDirectory) Backup() error {
	backupDir := fmt.Sprintf("%s.bak", d.path)
	err := os.RemoveAll(backupDir)
	if err != nil {
		return err
	}
	err = os.MkdirAll(backupDir, 0777)
	if err != nil {
		return err
	}
	err = exec.Command("cp", "-r", d.path, backupDir).Run()
	if err != nil {
		return err
	}

	return nil
}

// IsEmpty returns true if the data directory is entirely empty.
func (d *DataDirectory) IsEmpty() (bool, error) {
	dir, err := os.Open(d.path)
	if err != nil {
		return false, fmt.Errorf("failed to open data directory %s: %v", d.path, err)
	}
	defer dir.Close()
	_, err = dir.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}

// String returns the data directory path.
func (d *DataDirectory) String() string {
	return d.path
}

// VersionFile provides utilities for reading and writing version.txt files
// to etcd "data-dir" for tracking the etcd server and storage verions
// of the data in the directory.
type VersionFile struct {
	path string
}

// Exists returns true if a version.txt file exists on the file system.
func (v *VersionFile) Exists() (bool, error) {
	return exists(v.path)
}

// Read parses the version.txt file and returns it's contents.
func (v *VersionFile) Read() (*EtcdVersionPair, error) {
	data, err := ioutil.ReadFile(v.path)
	if err != nil {
		return nil, fmt.Errorf("failed to read version file %s: %v", v.path, err)
	}
	txt := strings.TrimSpace(string(data))
	vp, err := ParseEtcdVersionPair(txt)
	if err != nil {
		return nil, fmt.Errorf("failed to parse etcd '<version>/<storage-version>' string from version.txt file contents '%s': %v", txt, err)
	}
	return vp, nil
}

// Write creates or overwrites the contents of the version.txt file with the given EtcdVersionPair.
func (v *VersionFile) Write(vp *EtcdVersionPair) error {
	data := []byte(fmt.Sprintf("%s/%s", vp.version, vp.storageVersion))
	return ioutil.WriteFile(v.path, data, 0666)
}

func exists(path string) (bool, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}
<commit_msg>Correcting spelling errors<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/klog"
)

// DataDirectory provides utilities for initializing and backing up an
// etcd "data-dir" as well as managing a version.txt file to track the
// etcd server version and storage version of the etcd data in the
// directory.
type DataDirectory struct {
	path        string
	versionFile *VersionFile
}

// OpenOrCreateDataDirectory opens a data directory, creating the directory
// if it doesn't not already exist.
func OpenOrCreateDataDirectory(path string) (*DataDirectory, error) {
	exists, err := exists(path)
	if err != nil {
		return nil, err
	}
	if !exists {
		klog.Infof("data directory '%s' does not exist, creating it", path)
		err := os.MkdirAll(path, 0777)
		if err != nil {
			return nil, fmt.Errorf("failed to create data directory %s: %v", path, err)
		}
	}
	versionFile := &VersionFile{
		path: filepath.Join(path, versionFilename),
	}
	return &DataDirectory{path, versionFile}, nil
}

// Initialize set the version.txt to the target version if the data
// directory is empty. If the data directory is non-empty, no
// version.txt file will be written since the actual version of etcd
// used to create the data is unknown.
func (d *DataDirectory) Initialize(target *EtcdVersionPair) error {
	isEmpty, err := d.IsEmpty()
	if err != nil {
		return err
	}
	if isEmpty {
		klog.Infof("data directory '%s' is empty, writing target version '%s' to version.txt", d.path, target)
		err = d.versionFile.Write(target)
		if err != nil {
			return fmt.Errorf("failed to write version.txt to '%s': %v", d.path, err)
		}
		return nil
	}
	return nil
}

// Backup creates a backup copy of data directory.
func (d *DataDirectory) Backup() error {
	backupDir := fmt.Sprintf("%s.bak", d.path)
	err := os.RemoveAll(backupDir)
	if err != nil {
		return err
	}
	err = os.MkdirAll(backupDir, 0777)
	if err != nil {
		return err
	}
	err = exec.Command("cp", "-r", d.path, backupDir).Run()
	if err != nil {
		return err
	}

	return nil
}

// IsEmpty returns true if the data directory is entirely empty.
func (d *DataDirectory) IsEmpty() (bool, error) {
	dir, err := os.Open(d.path)
	if err != nil {
		return false, fmt.Errorf("failed to open data directory %s: %v", d.path, err)
	}
	defer dir.Close()
	_, err = dir.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}

// String returns the data directory path.
func (d *DataDirectory) String() string {
	return d.path
}

// VersionFile provides utilities for reading and writing version.txt files
// to etcd "data-dir" for tracking the etcd server and storage verions
// of the data in the directory.
type VersionFile struct {
	path string
}

// Exists returns true if a version.txt file exists on the file system.
func (v *VersionFile) Exists() (bool, error) {
	return exists(v.path)
}

// Read parses the version.txt file and returns it's contents.
func (v *VersionFile) Read() (*EtcdVersionPair, error) {
	data, err := ioutil.ReadFile(v.path)
	if err != nil {
		return nil, fmt.Errorf("failed to read version file %s: %v", v.path, err)
	}
	txt := strings.TrimSpace(string(data))
	vp, err := ParseEtcdVersionPair(txt)
	if err != nil {
		return nil, fmt.Errorf("failed to parse etcd '<version>/<storage-version>' string from version.txt file contents '%s': %v", txt, err)
	}
	return vp, nil
}

// Write creates or overwrites the contents of the version.txt file with the given EtcdVersionPair.
func (v *VersionFile) Write(vp *EtcdVersionPair) error {
	data := []byte(fmt.Sprintf("%s/%s", vp.version, vp.storageVersion))
	return ioutil.WriteFile(v.path, data, 0666)
}

func exists(path string) (bool, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/techjanitor/pram-post/config"
	e "github.com/techjanitor/pram-post/errors"
	"github.com/techjanitor/pram-post/models"
	u "github.com/techjanitor/pram-post/utils"
)

// Input from new reply form
type replyForm struct {
	Key     string `form:"askey" binding:"required"`
	Comment string `form:"comment"`
	Thread  uint   `form:"thread" binding:"required"`
}

// ReplyController handles the creation of new threads
func ReplyController(c *gin.Context) {
	var err error
	var rf replyForm
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(u.User)

	// check size of content
	if req.ContentLength > config.Settings.Limits.ImageMaxSize {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize)
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	err = c.Bind(&rf)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInvalidParam))
		c.Error(err)
		return
	}

	// Test for antispam key from Prim
	antispam := rf.Key
	if antispam != config.Settings.Antispam.AntispamKey {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrInvalidKey.Error()})
		c.Error(e.ErrInvalidKey)
		return
	}

	// Set parameters to ReplyModel
	m := models.ReplyModel{
		Uid:     userdata.Id,
		Ip:      c.ClientIP(),
		Comment: rf.Comment,
		Thread:  rf.Thread,
		Image:   true,
	}

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		m.Image = false
	}

	// Validate input parameters
	err = m.ValidateInput()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err)
		return
	}

	// Check thread status
	err = m.Status()
	if err == e.ErrThreadClosed {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err)
		return
	} else if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	if m.Comment != "" {

		// Check comment in SFS and Akismet
		check := u.CheckComment{
			Ip:      m.Ip,
			Name:    userdata.Name,
			Ua:      req.UserAgent(),
			Referer: req.Referer(),
			Comment: m.Comment,
		}

		err = check.Get()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err)
			return
		}

	}

	if m.Image {

		// process the uploaded file, this creates an md5
		err = image.ProcessFile()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err)
			return
		}

		// Set MD5 from results
		m.MD5 = image.MD5

		// Initialize check duplicate
		duplicate := u.CheckDuplicate{
			Ib:  m.Ib,
			MD5: m.MD5,
		}

		// Check database for duplicate image hashes
		err = duplicate.Get()
		if err == e.ErrDuplicateImage {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error(), "thread": duplicate.Thread, "post": duplicate.Post})
			c.Error(err)
			return
		} else if err != nil {
			c.JSON(e.ErrorMessage(e.ErrInternalError))
			c.Error(err)
			return
		}

		if image.Ext == ".webm" {

			// Save the webm to a file
			err = image.SaveWebM()
			if err != nil {
				c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
				c.Error(err)
				return
			}

		} else {

			// Save the image to a file
			err = image.SaveImage()
			if err != nil {
				c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
				c.Error(err)
				return
			}

		}

		m.OrigWidth = image.OrigWidth
		m.OrigHeight = image.OrigHeight
		m.ThumbWidth = image.ThumbWidth
		m.ThumbHeight = image.ThumbHeight
		m.Filename = image.Filename
		m.Thumbnail = image.Thumbnail

	}

	// Post data
	err = m.Post()
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	// Initialize cache handle
	cache := u.RedisCache

	// Delete redis stuff
	index_key := fmt.Sprintf("%s:%d", "index", m.Ib)
	directory_key := fmt.Sprintf("%s:%d", "directory", m.Ib)
	thread_key := fmt.Sprintf("%s:%d:%d", "thread", m.Ib, m.Thread)
	image_key := fmt.Sprintf("%s:%d", "image", m.Ib)

	err = cache.Delete(index_key, directory_key, thread_key, image_key)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	c.Redirect(303, req.Referer())

	audit := u.Audit{
		User:   userdata.Id,
		Ib:     m.Ib,
		Ip:     m.Ip,
		Action: u.AuditReply,
		Info:   fmt.Sprintf("%d/%d", m.Thread, m.PostNum),
	}

	err = audit.Submit()
	if err != nil {
		c.Error(err)
	}

	return

}
<commit_msg>add maxbytesreader<commit_after>package controllers

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/techjanitor/pram-post/config"
	e "github.com/techjanitor/pram-post/errors"
	"github.com/techjanitor/pram-post/models"
	u "github.com/techjanitor/pram-post/utils"
)

// Input from new reply form
type replyForm struct {
	Key     string `form:"askey" binding:"required"`
	Comment string `form:"comment"`
	Thread  uint   `form:"thread" binding:"required"`
}

// ReplyController handles the creation of new threads
func ReplyController(c *gin.Context) {
	var err error
	var rf replyForm
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(u.User)

	// check size of content
	if req.ContentLength > int64(config.Settings.Limits.ImageMaxSize) {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize)
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	err = c.Bind(&rf)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInvalidParam))
		c.Error(err)
		return
	}

	// Test for antispam key from Prim
	antispam := rf.Key
	if antispam != config.Settings.Antispam.AntispamKey {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrInvalidKey.Error()})
		c.Error(e.ErrInvalidKey)
		return
	}

	// Set parameters to ReplyModel
	m := models.ReplyModel{
		Uid:     userdata.Id,
		Ip:      c.ClientIP(),
		Comment: rf.Comment,
		Thread:  rf.Thread,
		Image:   true,
	}

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		m.Image = false
	}

	// Validate input parameters
	err = m.ValidateInput()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err)
		return
	}

	// Check thread status
	err = m.Status()
	if err == e.ErrThreadClosed {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err)
		return
	} else if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	if m.Comment != "" {

		// Check comment in SFS and Akismet
		check := u.CheckComment{
			Ip:      m.Ip,
			Name:    userdata.Name,
			Ua:      req.UserAgent(),
			Referer: req.Referer(),
			Comment: m.Comment,
		}

		err = check.Get()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err)
			return
		}

	}

	if m.Image {

		// process the uploaded file, this creates an md5
		err = image.ProcessFile()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err)
			return
		}

		// Set MD5 from results
		m.MD5 = image.MD5

		// Initialize check duplicate
		duplicate := u.CheckDuplicate{
			Ib:  m.Ib,
			MD5: m.MD5,
		}

		// Check database for duplicate image hashes
		err = duplicate.Get()
		if err == e.ErrDuplicateImage {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error(), "thread": duplicate.Thread, "post": duplicate.Post})
			c.Error(err)
			return
		} else if err != nil {
			c.JSON(e.ErrorMessage(e.ErrInternalError))
			c.Error(err)
			return
		}

		if image.Ext == ".webm" {

			// Save the webm to a file
			err = image.SaveWebM()
			if err != nil {
				c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
				c.Error(err)
				return
			}

		} else {

			// Save the image to a file
			err = image.SaveImage()
			if err != nil {
				c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
				c.Error(err)
				return
			}

		}

		m.OrigWidth = image.OrigWidth
		m.OrigHeight = image.OrigHeight
		m.ThumbWidth = image.ThumbWidth
		m.ThumbHeight = image.ThumbHeight
		m.Filename = image.Filename
		m.Thumbnail = image.Thumbnail

	}

	// Post data
	err = m.Post()
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	// Initialize cache handle
	cache := u.RedisCache

	// Delete redis stuff
	index_key := fmt.Sprintf("%s:%d", "index", m.Ib)
	directory_key := fmt.Sprintf("%s:%d", "directory", m.Ib)
	thread_key := fmt.Sprintf("%s:%d:%d", "thread", m.Ib, m.Thread)
	image_key := fmt.Sprintf("%s:%d", "image", m.Ib)

	err = cache.Delete(index_key, directory_key, thread_key, image_key)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err)
		return
	}

	c.Redirect(303, req.Referer())

	audit := u.Audit{
		User:   userdata.Id,
		Ib:     m.Ib,
		Ip:     m.Ip,
		Action: u.AuditReply,
		Info:   fmt.Sprintf("%d/%d", m.Thread, m.PostNum),
	}

	err = audit.Submit()
	if err != nil {
		c.Error(err)
	}

	return

}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"

	"github.com/golang/glog"
	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"
)

var (
	output string
)

var profileListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all minikube profiles.",
	Long:  "Lists all valid minikube profiles and detects all possible invalid profiles.",
	Run: func(cmd *cobra.Command, args []string) {

		switch strings.ToLower(output) {
		case "json":
			printProfilesJSON()
		case "table":
			printProfilesTable()
		default:
			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'table', 'json'", output))
		}

	},
}

var printProfilesTable = func() {

	var validData [][]string
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Profile", "VM Driver", "Container Runtime", "NodeIP", "API Port", "Kubernetes Version", "Status"})
	table.SetAutoFormatHeaders(false)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")
	validProfiles, invalidProfiles, err := config.ListProfiles()

	if len(validProfiles) == 0 || err != nil {
		exit.UsageT("No minikube profile was found. You can create one using `minikube start`.")
	}
	api, err := machine.NewAPIClient()
	if err != nil {
		glog.Infof("failed to get machine api client %v", err)
	}
	defer api.Close()

	for _, p := range validProfiles {
		p.Status, err = cluster.GetHostStatus(api, p.Name)
		if err != nil {
			glog.Infof("error getting host status for %v", err)
		}
		cp, err := config.PrimaryControlPlane(*p.Config)
		if err != nil {
			exit.WithError("profile has no control plane", err)
		}
		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
	}

	table.AppendBulk(validData)
	table.Render()

	if invalidProfiles != nil {
		out.T(out.WarningType, "Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
		for _, p := range invalidProfiles {
			out.T(out.Empty, "\t "+p.Name)
		}
		out.T(out.Tip, "You can delete them using the following command(s): ")
		for _, p := range invalidProfiles {
			out.String(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
		}

	}

	if err != nil {
		exit.WithCodeT(exit.Config, fmt.Sprintf("error loading profiles: %v", err))
	}

}

var printProfilesJSON = func() {
	api, err := machine.NewAPIClient()
	if err != nil {
		glog.Infof("failed to get machine api client %v", err)
	}
	defer api.Close()

	validProfiles, invalidProfiles, err := config.ListProfiles()
	for _, v := range validProfiles {
		status, err := cluster.GetHostStatus(api, v.Name)
		if err != nil {
			glog.Infof("error getting host status for  %v", err)
		}
		v.Status = status
	}

	var valid []*config.Profile
	var invalid []*config.Profile

	if validProfiles != nil {
		valid = validProfiles
	} else {
		valid = []*config.Profile{}
	}

	if invalidProfiles != nil {
		invalid = invalidProfiles
	} else {
		invalid = []*config.Profile{}
	}

	var body = map[string]interface{}{}

	if err == nil || os.IsNotExist(err) {
		body["valid"] = valid
		body["invalid"] = invalid
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
	} else {
		body["error"] = err
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
		os.Exit(exit.Failure)
	}
}

func init() {
	profileListCmd.Flags().StringVarP(&output, "output", "o", "table", "The output format. One of 'json', 'table'")
	ProfileCmd.AddCommand(profileListCmd)
}
<commit_msg>make it less than 80<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"

	"github.com/golang/glog"
	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"
)

var (
	output string
)

var profileListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all minikube profiles.",
	Long:  "Lists all valid minikube profiles and detects all possible invalid profiles.",
	Run: func(cmd *cobra.Command, args []string) {

		switch strings.ToLower(output) {
		case "json":
			printProfilesJSON()
		case "table":
			printProfilesTable()
		default:
			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'table', 'json'", output))
		}

	},
}

var printProfilesTable = func() {

	var validData [][]string
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status"})
	table.SetAutoFormatHeaders(false)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")
	validProfiles, invalidProfiles, err := config.ListProfiles()

	if len(validProfiles) == 0 || err != nil {
		exit.UsageT("No minikube profile was found. You can create one using `minikube start`.")
	}
	api, err := machine.NewAPIClient()
	if err != nil {
		glog.Infof("failed to get machine api client %v", err)
	}
	defer api.Close()

	for _, p := range validProfiles {
		p.Status, err = cluster.GetHostStatus(api, p.Name)
		if err != nil {
			glog.Infof("error getting host status for %v", err)
		}
		cp, err := config.PrimaryControlPlane(*p.Config)
		if err != nil {
			exit.WithError("profile has no control plane", err)
		}
		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
	}

	table.AppendBulk(validData)
	table.Render()

	if invalidProfiles != nil {
		out.T(out.WarningType, "Found {{.number}} invalid profile(s) ! ", out.V{"number": len(invalidProfiles)})
		for _, p := range invalidProfiles {
			out.T(out.Empty, "\t "+p.Name)
		}
		out.T(out.Tip, "You can delete them using the following command(s): ")
		for _, p := range invalidProfiles {
			out.String(fmt.Sprintf("\t $ minikube delete -p %s \n", p.Name))
		}

	}

	if err != nil {
		exit.WithCodeT(exit.Config, fmt.Sprintf("error loading profiles: %v", err))
	}

}

var printProfilesJSON = func() {
	api, err := machine.NewAPIClient()
	if err != nil {
		glog.Infof("failed to get machine api client %v", err)
	}
	defer api.Close()

	validProfiles, invalidProfiles, err := config.ListProfiles()
	for _, v := range validProfiles {
		status, err := cluster.GetHostStatus(api, v.Name)
		if err != nil {
			glog.Infof("error getting host status for  %v", err)
		}
		v.Status = status
	}

	var valid []*config.Profile
	var invalid []*config.Profile

	if validProfiles != nil {
		valid = validProfiles
	} else {
		valid = []*config.Profile{}
	}

	if invalidProfiles != nil {
		invalid = invalidProfiles
	} else {
		invalid = []*config.Profile{}
	}

	var body = map[string]interface{}{}

	if err == nil || os.IsNotExist(err) {
		body["valid"] = valid
		body["invalid"] = invalid
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
	} else {
		body["error"] = err
		jsonString, _ := json.Marshal(body)
		out.String(string(jsonString))
		os.Exit(exit.Failure)
	}
}

func init() {
	profileListCmd.Flags().StringVarP(&output, "output", "o", "table", "The output format. One of 'json', 'table'")
	ProfileCmd.AddCommand(profileListCmd)
}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"

	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/metadata"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/opentsdb"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/util"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_iostat_linux})
	collectors = append(collectors, &IntervalCollector{F: c_dfstat_blocks_linux})
	collectors = append(collectors, &IntervalCollector{F: c_dfstat_inodes_linux})
}

var diskLinuxFields = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
	desc string
}{
	{"read_requests", metadata.Counter, metadata.Count, "Total number of reads completed successfully."},
	{"read_merged", metadata.Counter, metadata.Count, "Adjacent read requests merged in a single req."},
	{"read_sectors", metadata.Counter, metadata.Count, "Total number of sectors read successfully."},
	{"msec_read", metadata.Counter, metadata.MilliSecond, "Total number of ms spent by all reads."},
	{"write_requests", metadata.Counter, metadata.Count, "Total number of writes completed successfully."},
	{"write_merged", metadata.Counter, metadata.Count, " Adjacent write requests merged in a single req."},
	{"write_sectors", metadata.Counter, metadata.Count, "Total number of sectors written successfully."},
	{"msec_write", metadata.Counter, metadata.MilliSecond, "Total number of ms spent by all writes."},
	{"ios_in_progress", metadata.Counter, metadata.Count, "Number of actual I/O requests currently in flight."},
	{"msec_total", metadata.Counter, metadata.MilliSecond, "Amount of time during which ios_in_progress >= 1."},
	{"msec_weighted_total", metadata.Gauge, metadata.MilliSecond, "Measure of recent I/O completion time and backlog."},
}

var diskLinuxFieldsPart = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
}{
	{"read_issued", metadata.Counter, metadata.Count},
	{"read_sectors", metadata.Counter, metadata.Count},
	{"write_issued", metadata.Counter, metadata.Count},
	{"write_sectors", metadata.Counter, metadata.Count},
}

func removable(major, minor string) bool {
	//We don't return an error, because removable may not exist for partitions of a removable device
	//So this is really "best effort" and we will have to see how it works in practice.
	b, err := ioutil.ReadFile("/sys/dev/block/" + major + ":" + minor + "/removable")
	if err != nil {
		return false
	}
	return strings.Trim(string(b), "\n") == "1"
}

var sdiskRE = regexp.MustCompile(`/dev/(sd[a-z])[0-9]?`)

func removable_fs(name string) bool {
	s := sdiskRE.FindStringSubmatch(name)
	if len(s) > 1 {
		b, err := ioutil.ReadFile("/sys/block/" + s[1] + "/removable")
		if err != nil {
			return false
		}
		return strings.Trim(string(b), "\n") == "1"
	}
	return false
}

func c_iostat_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var removables []string
	err := readLine("/proc/diskstats", func(s string) error {
		values := strings.Fields(s)
		if len(values) < 4 {
			return nil
		} else if values[3] == "0" {
			// Skip disks that haven't done a single read.
			return nil
		}
		metric := "linux.disk.part."
		i0, _ := strconv.Atoi(values[0])
		i1, _ := strconv.Atoi(values[1])
		if i1%16 == 0 && i0 > 1 {
			metric = "linux.disk."
		}
		device := values[2]
		ts := opentsdb.TagSet{"dev": device}
		if removable(values[0], values[1]) {
			removables = append(removables, device)
		}
		for _, r := range removables {
			if strings.HasPrefix(device, r) {
				metric += "rem."
			}
		}
		if len(values) == 14 {
			var read_sectors, msec_read, write_sectors, msec_write float64
			for i, v := range values[3:] {
				switch diskLinuxFields[i].key {
				case "read_sectors":
					read_sectors, _ = strconv.ParseFloat(v, 64)
				case "msec_read":
					msec_read, _ = strconv.ParseFloat(v, 64)
				case "write_sectors":
					write_sectors, _ = strconv.ParseFloat(v, 64)
				case "msec_write":
					msec_write, _ = strconv.ParseFloat(v, 64)
				}
				Add(&md, metric+diskLinuxFields[i].key, v, ts, diskLinuxFields[i].rate, diskLinuxFields[i].unit, diskLinuxFields[i].desc)
			}
			if read_sectors != 0 && msec_read != 0 {
				Add(&md, metric+"time_per_read", read_sectors/msec_read, ts, metadata.Rate, metadata.MilliSecond, "")
			}
			if write_sectors != 0 && msec_write != 0 {
				Add(&md, metric+"time_per_write", write_sectors/msec_write, ts, metadata.Rate, metadata.MilliSecond, "")
			}
		} else if len(values) == 7 {
			for i, v := range values[3:] {
				Add(&md, metric+diskLinuxFieldsPart[i].key, v, ts, diskLinuxFieldsPart[i].rate, diskLinuxFieldsPart[i].unit, "")
			}
		} else {
			return fmt.Errorf("cannot parse")
		}
		return nil
	})
	return md, err
}

func c_dfstat_blocks_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	err := util.ReadCommand(func(line string) error {
		fields := strings.Fields(line)
		// TODO: support mount points with spaces in them. They mess up the field order
		// currently due to df's columnar output.
		if len(fields) != 6 || !IsDigit(fields[2]) {
			return nil
		}
		fs := fields[0]
		mount := fields[5]
		tags := opentsdb.TagSet{"mount": mount}
		os_tags := opentsdb.TagSet{"disk": mount}
		metric := "linux.disk.fs."
		ometric := "os.disk.fs."
		if removable_fs(fs) {
			metric += "rem."
			ometric += "rem."
		}
		Add(&md, metric+"space_total", fields[1], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, metric+"space_used", fields[2], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, metric+"space_free", fields[3], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_total", fields[1], os_tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_used", fields[2], os_tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_free", fields[3], os_tags, metadata.Gauge, metadata.KBytes, "")
		st, _ := strconv.ParseFloat(fields[1], 64)
		sf, _ := strconv.ParseFloat(fields[3], 64)
		if st != 0 {
			Add(&md, osDiskPctFree, sf/st*100, os_tags, metadata.Gauge, metadata.Pct, "")
		}
		return nil
	}, "df", "-lP", "--block-size", "1")
	return md, err
}

func c_dfstat_inodes_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	err := util.ReadCommand(func(line string) error {
		fields := strings.Fields(line)
		if len(fields) != 6 || !IsDigit(fields[2]) {
			return nil
		}
		mount := fields[5]
		fs := fields[0]
		tags := opentsdb.TagSet{"mount": mount}
		metric := "linux.disk.fs."
		if removable_fs(fs) {
			metric += "rem."
		}
		Add(&md, metric+"inodes_total", fields[1], tags, metadata.Gauge, metadata.Count, "")
		Add(&md, metric+"inodes_used", fields[2], tags, metadata.Gauge, metadata.Count, "")
		Add(&md, metric+"inodes_free", fields[3], tags, metadata.Gauge, metadata.Count, "")
		return nil
	}, "df", "-liP")
	return md, err
}
<commit_msg>cmd/scollector: ios_in_progress should be Gauge instead of Counter<commit_after>package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"

	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/metadata"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/opentsdb"
	"github.com/bosun-monitor/scollector/_third_party/github.com/bosun-monitor/util"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_iostat_linux})
	collectors = append(collectors, &IntervalCollector{F: c_dfstat_blocks_linux})
	collectors = append(collectors, &IntervalCollector{F: c_dfstat_inodes_linux})
}

var diskLinuxFields = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
	desc string
}{
	{"read_requests", metadata.Counter, metadata.Count, "Total number of reads completed successfully."},
	{"read_merged", metadata.Counter, metadata.Count, "Adjacent read requests merged in a single req."},
	{"read_sectors", metadata.Counter, metadata.Count, "Total number of sectors read successfully."},
	{"msec_read", metadata.Counter, metadata.MilliSecond, "Total number of ms spent by all reads."},
	{"write_requests", metadata.Counter, metadata.Count, "Total number of writes completed successfully."},
	{"write_merged", metadata.Counter, metadata.Count, " Adjacent write requests merged in a single req."},
	{"write_sectors", metadata.Counter, metadata.Count, "Total number of sectors written successfully."},
	{"msec_write", metadata.Counter, metadata.MilliSecond, "Total number of ms spent by all writes."},
	{"ios_in_progress", metadata.Gauge, metadata.Operation, "Number of actual I/O requests currently in flight."},
	{"msec_total", metadata.Counter, metadata.MilliSecond, "Amount of time during which ios_in_progress >= 1."},
	{"msec_weighted_total", metadata.Gauge, metadata.MilliSecond, "Measure of recent I/O completion time and backlog."},
}

var diskLinuxFieldsPart = []struct {
	key  string
	rate metadata.RateType
	unit metadata.Unit
}{
	{"read_issued", metadata.Counter, metadata.Count},
	{"read_sectors", metadata.Counter, metadata.Count},
	{"write_issued", metadata.Counter, metadata.Count},
	{"write_sectors", metadata.Counter, metadata.Count},
}

func removable(major, minor string) bool {
	//We don't return an error, because removable may not exist for partitions of a removable device
	//So this is really "best effort" and we will have to see how it works in practice.
	b, err := ioutil.ReadFile("/sys/dev/block/" + major + ":" + minor + "/removable")
	if err != nil {
		return false
	}
	return strings.Trim(string(b), "\n") == "1"
}

var sdiskRE = regexp.MustCompile(`/dev/(sd[a-z])[0-9]?`)

func removable_fs(name string) bool {
	s := sdiskRE.FindStringSubmatch(name)
	if len(s) > 1 {
		b, err := ioutil.ReadFile("/sys/block/" + s[1] + "/removable")
		if err != nil {
			return false
		}
		return strings.Trim(string(b), "\n") == "1"
	}
	return false
}

func c_iostat_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var removables []string
	err := readLine("/proc/diskstats", func(s string) error {
		values := strings.Fields(s)
		if len(values) < 4 {
			return nil
		} else if values[3] == "0" {
			// Skip disks that haven't done a single read.
			return nil
		}
		metric := "linux.disk.part."
		i0, _ := strconv.Atoi(values[0])
		i1, _ := strconv.Atoi(values[1])
		if i1%16 == 0 && i0 > 1 {
			metric = "linux.disk."
		}
		device := values[2]
		ts := opentsdb.TagSet{"dev": device}
		if removable(values[0], values[1]) {
			removables = append(removables, device)
		}
		for _, r := range removables {
			if strings.HasPrefix(device, r) {
				metric += "rem."
			}
		}
		if len(values) == 14 {
			var read_sectors, msec_read, write_sectors, msec_write float64
			for i, v := range values[3:] {
				switch diskLinuxFields[i].key {
				case "read_sectors":
					read_sectors, _ = strconv.ParseFloat(v, 64)
				case "msec_read":
					msec_read, _ = strconv.ParseFloat(v, 64)
				case "write_sectors":
					write_sectors, _ = strconv.ParseFloat(v, 64)
				case "msec_write":
					msec_write, _ = strconv.ParseFloat(v, 64)
				}
				Add(&md, metric+diskLinuxFields[i].key, v, ts, diskLinuxFields[i].rate, diskLinuxFields[i].unit, diskLinuxFields[i].desc)
			}
			if read_sectors != 0 && msec_read != 0 {
				Add(&md, metric+"time_per_read", read_sectors/msec_read, ts, metadata.Rate, metadata.MilliSecond, "")
			}
			if write_sectors != 0 && msec_write != 0 {
				Add(&md, metric+"time_per_write", write_sectors/msec_write, ts, metadata.Rate, metadata.MilliSecond, "")
			}
		} else if len(values) == 7 {
			for i, v := range values[3:] {
				Add(&md, metric+diskLinuxFieldsPart[i].key, v, ts, diskLinuxFieldsPart[i].rate, diskLinuxFieldsPart[i].unit, "")
			}
		} else {
			return fmt.Errorf("cannot parse")
		}
		return nil
	})
	return md, err
}

func c_dfstat_blocks_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	err := util.ReadCommand(func(line string) error {
		fields := strings.Fields(line)
		// TODO: support mount points with spaces in them. They mess up the field order
		// currently due to df's columnar output.
		if len(fields) != 6 || !IsDigit(fields[2]) {
			return nil
		}
		fs := fields[0]
		mount := fields[5]
		tags := opentsdb.TagSet{"mount": mount}
		os_tags := opentsdb.TagSet{"disk": mount}
		metric := "linux.disk.fs."
		ometric := "os.disk.fs."
		if removable_fs(fs) {
			metric += "rem."
			ometric += "rem."
		}
		Add(&md, metric+"space_total", fields[1], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, metric+"space_used", fields[2], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, metric+"space_free", fields[3], tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_total", fields[1], os_tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_used", fields[2], os_tags, metadata.Gauge, metadata.KBytes, "")
		Add(&md, ometric+"space_free", fields[3], os_tags, metadata.Gauge, metadata.KBytes, "")
		st, _ := strconv.ParseFloat(fields[1], 64)
		sf, _ := strconv.ParseFloat(fields[3], 64)
		if st != 0 {
			Add(&md, osDiskPctFree, sf/st*100, os_tags, metadata.Gauge, metadata.Pct, "")
		}
		return nil
	}, "df", "-lP", "--block-size", "1")
	return md, err
}

func c_dfstat_inodes_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	err := util.ReadCommand(func(line string) error {
		fields := strings.Fields(line)
		if len(fields) != 6 || !IsDigit(fields[2]) {
			return nil
		}
		mount := fields[5]
		fs := fields[0]
		tags := opentsdb.TagSet{"mount": mount}
		metric := "linux.disk.fs."
		if removable_fs(fs) {
			metric += "rem."
		}
		Add(&md, metric+"inodes_total", fields[1], tags, metadata.Gauge, metadata.Count, "")
		Add(&md, metric+"inodes_used", fields[2], tags, metadata.Gauge, metadata.Count, "")
		Add(&md, metric+"inodes_free", fields[3], tags, metadata.Gauge, metadata.Count, "")
		return nil
	}, "df", "-liP")
	return md, err
}
<|endoftext|>


<commit_before>// +build darwin linux

package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
	"time"

	"github.com/garyburd/redigo/redis"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_redis_linux, init: redisInit})
}

var redisFields = map[string]bool{
	"aof_enabled":                  true,
	"aof_rewrite_in_progress":      true,
	"aof_rewrite_scheduled":        true,
	"aof_last_rewrite_time_sec":    true,
	"aof_current_rewrite_time_sec": true,
	"aof_last_bgrewrite_status":    true,
	"bgrewriteaof_in_progress":     true,
	"bgsave_in_progress":           true,
	"blocked_clients":              true,
	"changes_since_last_save":      true,
	"client_biggest_input_buf":     true,
	"client_longest_output_list":   true,
	"connected_clients":            true,
	"connected_slaves":             true,
	"evicted_keys":                 true,
	"expired_keys":                 true,
	"hash_max_zipmap_entries":      true,
	"hash_max_zipmap_value":        true,
	"keyspace_hits":                true,
	"keyspace_misses":              true,
	"master_link_status":           true,
	"master_sync_in_progress":      true,
	"master_last_io_seconds_ago":   true,
	"master_sync_left_bytes":       true,
	"mem_fragmentation_ratio":      true,
	"pubsub_channels":              true,
	"pubsub_patterns":              true,
	"rdb_changes_since_last_save":  true,
	"rdb_bgsave_in_progress":       true,
	"rdb_last_save_time":           true,
	"rdb_last_bgsave_status":       true,
	"rdb_last_bgsave_time_sec":     true,
	"rdb_current_bgsave_time_sec":  true,
	"total_commands_processed":     true,
	"total_connections_received":   true,
	"uptime_in_seconds":            true,
	"used_cpu_sys":                 true,
	"used_cpu_user":                true,
	"used_memory":                  true,
	"used_memory_rss":              true,
}

//For master_link_status
var redisMlsMap = map[string]string{
	"up":   "1",
	"down": "0",
}

//For aof_last_bgrewrite_status, rdb_last_bgsave_status
func status(s string) string {
	if s == "ok" {
		return "1"
	}
	return "0"
}

var (
	tcRE           = regexp.MustCompile(`^\s*#\s*scollector.(\w+)\s*=\s*(.+)$`)
	redisInstances map[string]string
)

func redisInit() {
	update := func() {
		ri := make(map[string]string)
		oldRedis := false
		add := func(port, pid string) {
			cluster := fmt.Sprintf("port-%s", port)
			defer func() {
				ri[port] = cluster
			}()
			f, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid))
			if err != nil {
				return
			}
			fsp := strings.Split(strings.Split(string(f), "\n")[0], "\u0000")
			if len(fsp) < 2 {
				return
			}
			cfg := fsp[len(fsp)-2]
			if len(cfg) == 0 {
				return
			}
			readLine(cfg, func(cfgline string) {
				result := tcRE.FindStringSubmatch(cfgline)
				if len(result) > 2 && strings.ToLower(result[0]) == "cluster" {
					cluster = strings.ToLower(result[1])
				}
			})
		}
		readCommand(func(line string) {
			sp := strings.Fields(line)
			if len(sp) != 3 || !strings.HasSuffix(sp[1], "redis-server") {
				return
			}
			if !strings.Contains(sp[2], ":") {
				oldRedis = true
				return
			}
			pid := sp[0]
			port := strings.Split(sp[2], ":")[1]
			add(port, pid)
		}, "ps", "-e", "-o", "pid,args")
		if oldRedis {
			readCommand(func(line string) {
				if !strings.Contains(line, "redis-server") {
					return
				}
				sp := strings.Fields(line)
				if len(sp) < 7 || !strings.Contains(sp[3], ":") {
					return
				}
				pid := strings.Split(sp[6], "/")[0]
				port := strings.Split(sp[3], ":")[1]
				add(port, pid)
			}, "netstat", "-tnlp")
		}
		redisInstances = ri
	}
	update()
	go func() {
		for _ = range time.Tick(time.Minute * 5) {
			update()
		}
	}()
}

func c_redis_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	for port, cluster := range redisInstances {
		c, err := redis.Dial("tcp", fmt.Sprintf(":%s", port))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		defer c.Close()
		tags := opentsdb.TagSet{
			"cluster": cluster,
			"port":    port,
		}
		lines, err := c.Do("INFO")
		if err != nil {
			slog.Infoln(err)
			continue
		}
		_ = tags
		for _, line := range strings.Split(string(lines.([]uint8)), "\n") {
			line = strings.TrimSpace(line)
			sp := strings.Split(line, ":")
			if len(sp) < 2 || !redisFields[sp[0]] {
				continue
			}
			if sp[0] == "master_link_status" {
				Add(&md, "redis."+sp[0], redisMlsMap[sp[1]], tags)
				continue
			}
			if sp[0] == "aof_last_bgrewrite_status" || sp[0] == "rdb_last_bgsave_status" {
				Add(&md, "redis."+sp[0], status(sp[1]), tags)
				continue
			}
			Add(&md, "redis."+sp[0], sp[1], tags)
		}
	}
	return md
}
<commit_msg>cmd/scollector: Sentences<commit_after>// +build darwin linux

package collectors

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
	"time"

	"github.com/garyburd/redigo/redis"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_redis_linux, init: redisInit})
}

var redisFields = map[string]bool{
	"aof_enabled":                  true,
	"aof_rewrite_in_progress":      true,
	"aof_rewrite_scheduled":        true,
	"aof_last_rewrite_time_sec":    true,
	"aof_current_rewrite_time_sec": true,
	"aof_last_bgrewrite_status":    true,
	"bgrewriteaof_in_progress":     true,
	"bgsave_in_progress":           true,
	"blocked_clients":              true,
	"changes_since_last_save":      true,
	"client_biggest_input_buf":     true,
	"client_longest_output_list":   true,
	"connected_clients":            true,
	"connected_slaves":             true,
	"evicted_keys":                 true,
	"expired_keys":                 true,
	"hash_max_zipmap_entries":      true,
	"hash_max_zipmap_value":        true,
	"keyspace_hits":                true,
	"keyspace_misses":              true,
	"master_link_status":           true,
	"master_sync_in_progress":      true,
	"master_last_io_seconds_ago":   true,
	"master_sync_left_bytes":       true,
	"mem_fragmentation_ratio":      true,
	"pubsub_channels":              true,
	"pubsub_patterns":              true,
	"rdb_changes_since_last_save":  true,
	"rdb_bgsave_in_progress":       true,
	"rdb_last_save_time":           true,
	"rdb_last_bgsave_status":       true,
	"rdb_last_bgsave_time_sec":     true,
	"rdb_current_bgsave_time_sec":  true,
	"total_commands_processed":     true,
	"total_connections_received":   true,
	"uptime_in_seconds":            true,
	"used_cpu_sys":                 true,
	"used_cpu_user":                true,
	"used_memory":                  true,
	"used_memory_rss":              true,
}

// For master_link_status.
var redisMlsMap = map[string]string{
	"up":   "1",
	"down": "0",
}

// For aof_last_bgrewrite_status, rdb_last_bgsave_status.
func status(s string) string {
	if s == "ok" {
		return "1"
	}
	return "0"
}

var (
	tcRE           = regexp.MustCompile(`^\s*#\s*scollector.(\w+)\s*=\s*(.+)$`)
	redisInstances map[string]string
)

func redisInit() {
	update := func() {
		ri := make(map[string]string)
		oldRedis := false
		add := func(port, pid string) {
			cluster := fmt.Sprintf("port-%s", port)
			defer func() {
				ri[port] = cluster
			}()
			f, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid))
			if err != nil {
				return
			}
			fsp := strings.Split(strings.Split(string(f), "\n")[0], "\u0000")
			if len(fsp) < 2 {
				return
			}
			cfg := fsp[len(fsp)-2]
			if len(cfg) == 0 {
				return
			}
			readLine(cfg, func(cfgline string) {
				result := tcRE.FindStringSubmatch(cfgline)
				if len(result) > 2 && strings.ToLower(result[0]) == "cluster" {
					cluster = strings.ToLower(result[1])
				}
			})
		}
		readCommand(func(line string) {
			sp := strings.Fields(line)
			if len(sp) != 3 || !strings.HasSuffix(sp[1], "redis-server") {
				return
			}
			if !strings.Contains(sp[2], ":") {
				oldRedis = true
				return
			}
			pid := sp[0]
			port := strings.Split(sp[2], ":")[1]
			add(port, pid)
		}, "ps", "-e", "-o", "pid,args")
		if oldRedis {
			readCommand(func(line string) {
				if !strings.Contains(line, "redis-server") {
					return
				}
				sp := strings.Fields(line)
				if len(sp) < 7 || !strings.Contains(sp[3], ":") {
					return
				}
				pid := strings.Split(sp[6], "/")[0]
				port := strings.Split(sp[3], ":")[1]
				add(port, pid)
			}, "netstat", "-tnlp")
		}
		redisInstances = ri
	}
	update()
	go func() {
		for _ = range time.Tick(time.Minute * 5) {
			update()
		}
	}()
}

func c_redis_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	for port, cluster := range redisInstances {
		c, err := redis.Dial("tcp", fmt.Sprintf(":%s", port))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		defer c.Close()
		tags := opentsdb.TagSet{
			"cluster": cluster,
			"port":    port,
		}
		lines, err := c.Do("INFO")
		if err != nil {
			slog.Infoln(err)
			continue
		}
		_ = tags
		for _, line := range strings.Split(string(lines.([]uint8)), "\n") {
			line = strings.TrimSpace(line)
			sp := strings.Split(line, ":")
			if len(sp) < 2 || !redisFields[sp[0]] {
				continue
			}
			if sp[0] == "master_link_status" {
				Add(&md, "redis."+sp[0], redisMlsMap[sp[1]], tags)
				continue
			}
			if sp[0] == "aof_last_bgrewrite_status" || sp[0] == "rdb_last_bgsave_status" {
				Add(&md, "redis."+sp[0], status(sp[1]), tags)
				continue
			}
			Add(&md, "redis."+sp[0], sp[1], tags)
		}
	}
	return md
}
<|endoftext|>


<commit_before>package commands

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"os"
	gopath "path"
	"strings"

	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"

	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	path "github.com/ipfs/go-ipfs/path"
	tar "github.com/ipfs/go-ipfs/thirdparty/tar"
	uarchive "github.com/ipfs/go-ipfs/unixfs/archive"
)

var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9")

var GetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Download IPFS objects.",
		ShortDescription: `
Retrieves the object named by <ipfs-or-ipns-path> and stores the data to disk.

By default, the output will be stored at ./<ipfs-path>, but an alternate path
can be specified with '--output=<path>' or '-o=<path>'.

To output a TAR archive instead of unpacked files, use '--archive' or '-a'.

To compress the output with GZIP compression, use '--compress' or '-C'. You
may also specify the level of compression by specifying '-l=<1-9>'.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted.").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.StringOption("output", "o", "The path where output should be stored."),
		cmds.BoolOption("archive", "a", "Output a TAR archive."),
		cmds.BoolOption("compress", "C", "Compress the output with GZIP compression."),
		cmds.IntOption("compression-level", "l", "The level of compression (1-9)."),
	},
	PreRun: func(req cmds.Request) error {
		_, err := getCompressOptions(req)
		return err
	},
	Run: func(req cmds.Request, res cmds.Response) {
		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		p := path.Path(req.Arguments()[0])
		ctx := req.Context()
		dn, err := core.Resolve(ctx, node, p)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		archive, _, _ := req.Option("archive").Bool()
		reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		res.SetOutput(reader)
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Output() == nil {
			return
		}
		outReader := res.Output().(io.Reader)
		res.SetOutput(nil)

		outPath, _, _ := req.Option("output").String()
		if len(outPath) == 0 {
			_, outPath = gopath.Split(req.Arguments()[0])
			outPath = gopath.Clean(outPath)
		}

		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		archive, _, _ := req.Option("archive").Bool()

		gw := getWriter{
			Out:         os.Stdout,
			Err:         os.Stderr,
			Archive:     archive,
			Compression: cmplvl,
		}

		if err := gw.Write(outReader, outPath); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

type clearlineReader struct {
	io.Reader
	out io.Writer
}

func (r *clearlineReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	if err == io.EOF {
		// callback
		fmt.Fprintf(r.out, "\033[2K\r") // clear progress bar line on EOF
	}
	return
}

func progressBarForReader(out io.Writer, r io.Reader, l int64) (*pb.ProgressBar, io.Reader) {
	// setup bar reader
	// TODO: get total length of files
	bar := pb.New64(l).SetUnits(pb.U_BYTES)
	bar.Output = out

	// the progress bar lib doesn't give us a way to get the width of the output,
	// so as a hack we just use a callback to measure the output, then git rid of it
	bar.Callback = func(line string) {
		terminalWidth := len(line)
		bar.Callback = nil
		log.Infof("terminal width: %v\n", terminalWidth)
	}
	barR := bar.NewProxyReader(r)
	return bar, &clearlineReader{barR, out}
}

type getWriter struct {
	Out io.Writer // for output to user
	Err io.Writer // for progress bar output

	Archive     bool
	Compression int
}

func (gw *getWriter) Write(r io.Reader, fpath string) error {
	if gw.Archive || gw.Compression != gzip.NoCompression {
		return gw.writeArchive(r, fpath)
	}
	return gw.writeExtracted(r, fpath)
}

func (gw *getWriter) writeArchive(r io.Reader, fpath string) error {
	// adjust file name if tar
	if gw.Archive {
		if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") {
			fpath += ".tar"
		}
	}

	// adjust file name if gz
	if gw.Compression != gzip.NoCompression {
		if !strings.HasSuffix(fpath, ".gz") {
			fpath += ".gz"
		}
	}

	// create file
	file, err := os.Create(fpath)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r, 0)
	bar.Start()
	defer bar.Finish()

	_, err = io.Copy(file, barR)
	return err
}

func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error {
	fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r, 0)
	bar.Start()
	defer bar.Finish()

	extractor := &tar.Extractor{fpath}
	return extractor.Extract(barR)
}

func getCompressOptions(req cmds.Request) (int, error) {
	cmprs, _, _ := req.Option("compress").Bool()
	cmplvl, cmplvlFound, _ := req.Option("compression-level").Int()
	switch {
	case !cmprs:
		return gzip.NoCompression, nil
	case cmprs && !cmplvlFound:
		return gzip.DefaultCompression, nil
	case cmprs && cmplvlFound && (cmplvl < 1 || cmplvl > 9):
		return gzip.NoCompression, ErrInvalidCompressionLevel
	}
	return cmplvl, nil
}
<commit_msg>Added ticks, `the`, and defaults to `get`<commit_after>package commands

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"os"
	gopath "path"
	"strings"

	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"

	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	path "github.com/ipfs/go-ipfs/path"
	tar "github.com/ipfs/go-ipfs/thirdparty/tar"
	uarchive "github.com/ipfs/go-ipfs/unixfs/archive"
)

var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9")

var GetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Download IPFS objects.",
		ShortDescription: `
Retrieves the object named by <ipfs-or-ipns-path> and stores the data to disk.

By default, the output will be stored at './<ipfs-path>', but an alternate path
can be specified with '--output=<path>' or '-o=<path>'.

To output a TAR archive instead of unpacked files, use '--archive' or '-a'.

To compress the output with GZIP compression, use '--compress' or '-C'. You
may also specify the level of compression by specifying '-l=<1-9>'.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted.").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.StringOption("output", "o", "The path where the output should be stored."),
		cmds.BoolOption("archive", "a", "Output a TAR archive. Default: false."),
		cmds.BoolOption("compress", "C", "Compress the output with GZIP compression. Default: false."),
		cmds.IntOption("compression-level", "l", "The level of compression (1-9). Default: -1."),
	},
	PreRun: func(req cmds.Request) error {
		_, err := getCompressOptions(req)
		return err
	},
	Run: func(req cmds.Request, res cmds.Response) {
		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		p := path.Path(req.Arguments()[0])
		ctx := req.Context()
		dn, err := core.Resolve(ctx, node, p)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		archive, _, _ := req.Option("archive").Bool()
		reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		res.SetOutput(reader)
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Output() == nil {
			return
		}
		outReader := res.Output().(io.Reader)
		res.SetOutput(nil)

		outPath, _, _ := req.Option("output").String()
		if len(outPath) == 0 {
			_, outPath = gopath.Split(req.Arguments()[0])
			outPath = gopath.Clean(outPath)
		}

		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		archive, _, _ := req.Option("archive").Bool()

		gw := getWriter{
			Out:         os.Stdout,
			Err:         os.Stderr,
			Archive:     archive,
			Compression: cmplvl,
		}

		if err := gw.Write(outReader, outPath); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

type clearlineReader struct {
	io.Reader
	out io.Writer
}

func (r *clearlineReader) Read(p []byte) (n int, err error) {
	n, err = r.Reader.Read(p)
	if err == io.EOF {
		// callback
		fmt.Fprintf(r.out, "\033[2K\r") // clear progress bar line on EOF
	}
	return
}

func progressBarForReader(out io.Writer, r io.Reader, l int64) (*pb.ProgressBar, io.Reader) {
	// setup bar reader
	// TODO: get total length of files
	bar := pb.New64(l).SetUnits(pb.U_BYTES)
	bar.Output = out

	// the progress bar lib doesn't give us a way to get the width of the output,
	// so as a hack we just use a callback to measure the output, then git rid of it
	bar.Callback = func(line string) {
		terminalWidth := len(line)
		bar.Callback = nil
		log.Infof("terminal width: %v\n", terminalWidth)
	}
	barR := bar.NewProxyReader(r)
	return bar, &clearlineReader{barR, out}
}

type getWriter struct {
	Out io.Writer // for output to user
	Err io.Writer // for progress bar output

	Archive     bool
	Compression int
}

func (gw *getWriter) Write(r io.Reader, fpath string) error {
	if gw.Archive || gw.Compression != gzip.NoCompression {
		return gw.writeArchive(r, fpath)
	}
	return gw.writeExtracted(r, fpath)
}

func (gw *getWriter) writeArchive(r io.Reader, fpath string) error {
	// adjust file name if tar
	if gw.Archive {
		if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") {
			fpath += ".tar"
		}
	}

	// adjust file name if gz
	if gw.Compression != gzip.NoCompression {
		if !strings.HasSuffix(fpath, ".gz") {
			fpath += ".gz"
		}
	}

	// create file
	file, err := os.Create(fpath)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r, 0)
	bar.Start()
	defer bar.Finish()

	_, err = io.Copy(file, barR)
	return err
}

func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error {
	fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r, 0)
	bar.Start()
	defer bar.Finish()

	extractor := &tar.Extractor{fpath}
	return extractor.Extract(barR)
}

func getCompressOptions(req cmds.Request) (int, error) {
	cmprs, _, _ := req.Option("compress").Bool()
	cmplvl, cmplvlFound, _ := req.Option("compression-level").Int()
	switch {
	case !cmprs:
		return gzip.NoCompression, nil
	case cmprs && !cmplvlFound:
		return gzip.DefaultCompression, nil
	case cmprs && cmplvlFound && (cmplvl < 1 || cmplvl > 9):
		return gzip.NoCompression, ErrInvalidCompressionLevel
	}
	return cmplvl, nil
}
<|endoftext|>