<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"
)

var (
	vsn = "0.0.1-rc2"
)

type Remote interface {
	Channel() chan []byte
	Close() error
	Connect() error
	Recv(timeout time.Duration) ([]byte, error)
	Write(buf []byte, timeout time.Duration) (int, error)
}

type Flags struct {
	help    bool
	list    bool
	version bool

	verify        bool
	nid           Address
	sid           Address
	uid           Address
	autoConfigure bool

	stdio     bool
	remote    string
	tls       bool
	reconnect bool
}

func parseFlags() Flags {
	flags := new(Flags)

	// commands
	listFlag := flag.Bool("list", false, "List available serialports")
	helpFlag := flag.Bool("help", false, "Show help text")
	versionFlag := flag.Bool("version", false, "Show version")

	// link flags
	verifyFlag := flag.Bool("verify", true, "validate IDs according to -nid, -sid, and -uid flags")
	autoConfigureFlag := flag.Bool("auto-configure", false, "Automatically configure gateway operation and ID's; use -nid, -sid, and -uid flags")
	nidFlag := flag.String("nid", "::", "32bit Network ID in hexadecimal (ie, aa:bb:cc:dd)")
	sidFlag := flag.String("sid", "::", "32bit System ID in hexadecimal (ie, aa:bb:cc:dd)")
	uidFlag := flag.String("uid", "::", "32bit Unique ID in hexadecimal (ie, aa:bb:cc:dd)")

	// communication flags
	stdioFlag := flag.Bool("stdio", false, "Use stdio for communication instead of remote")
	remoteFlag := flag.String("remote", "tcp.cloud.tiny-mesh.com:7002", "The upstream url to connect to")
	usetlsFlag := flag.Bool("tls", true, "Controll use of TLS with -remote")
	reconnectFlag := flag.Bool("reconnect", true, "Automatically re-establish communication on failure")

	flag.Parse()

	flags.help = *helpFlag
	flags.list = *listFlag
	flags.version = *versionFlag

	flags.verify = *verifyFlag
	flags.autoConfigure = *autoConfigureFlag
	flags.nid = parseAddr(*nidFlag)
	flags.sid = parseAddr(*sidFlag)
	flags.uid = parseAddr(*uidFlag)

	if len(flags.nid) == 0 {
		log.Fatalf("failed to parse -nid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -nid 01:02:03:04\n")
	}
	if len(flags.sid) == 0 {
		log.Fatalf("failed to parse -sid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -sid 01:02:03:04\n")
	}
	if len(flags.uid) == 0 {
		log.Fatalf("failed to parse -uid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -uid 01:02:03:04\n")
	}

	flags.stdio = *stdioFlag
	flags.remote = *remoteFlag
	flags.tls = *usetlsFlag
	flags.reconnect = *reconnectFlag

	return *flags
}

func pickUpstream(flags Flags) (Remote, error) {
	if true == flags.stdio {
		// stdio
		return ConnectStdio(os.Stdin, os.Stdout)
	} else if true == flags.tls {
		// tls
		return ConnectTLS(flags.remote)
	}

	return ConnectTCP(flags.remote)
}

func forward(remote Remote, t time.Duration) chan []byte {
	ch := make(chan []byte, 256)

	go func() {
		for {
			buf, err := remote.Recv(t)

			if nil != err {
				log.Printf("forward-err: %v\n", err)
				close(ch)
				return
			}

			ch <- buf
		}
	}()

	return ch
}

func loop(from Remote, to Remote, flags Flags) {
	upstream := forward(from, 500*time.Millisecond)
	downstream := forward(to, 2*time.Millisecond)

	upoff := &Backoff{
		initial: 1 * time.Second,
		wait:    1 * time.Second,
		delay:   2.5,
		max:     5 * time.Minute,
	}

	downoff := &Backoff{
		initial: 1 * time.Second,
		wait:    1 * time.Second,
		delay:   2.5,
		max:     5 * time.Minute,
	}

	for {
		select {
		case buf, state := <-upstream:
			if false == state {
				if !flags.reconnect {
					log.Fatalf("upstream:close, exiting")
				}

				log.Printf("upstream:close, reconnecting\n")
				from.Close()
				if err := from.Connect(); nil != err {
					log.Printf("upstream:open: %v\n", err)
					upoff.Fail()
				} else {
					upoff.Success()
					upstream = forward(from, 500*time.Millisecond)
				}
			} else if len(buf) > 0 {
				log.Printf("upstream:recv[%v] %v\n", state, buf)
				if len(buf) > 10 && 6 == buf[0] {
					to.Write(buf[:1], -1)
					to.Write(buf[1:], -1)
				} else {
					to.Write(buf, -1)
				}
			}

		case buf, state := <-downstream:
			if false == state {
				if !flags.reconnect {
					log.Fatalf("upstream:close, exiting")
				}

				to.Close()
				log.Printf("downstream:close, reconnecting\n")
				if err := to.Connect(); nil != err {
					log.Printf("downstream:open: %v\n", err)
					downoff.Fail()
				} else {
					downoff.Success()
					downstream = forward(to, 2*time.Millisecond)
				}
			} else if len(buf) > 0 {
				log.Printf("downstream:recv[%v] %v\n", state, buf)
				from.Write(buf, -1)
			}
		}
	}
}

func main() {

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	flags := parseFlags()

	if true == flags.help {
		flag.PrintDefaults()
		return
	} else if true == flags.list {
		PrintPortList()
		return
	} else if true == flags.version {
		fmt.Printf("%v\n", vsn)
		return
	}

	path := flag.Arg(0)

	if "" == path {
		log.Fatal(errors.New("1st argument, tty path, missing"))
	}

	var upstream Remote
	var downstream Remote
	var err error

	log.Printf("guri - version %v\n", vsn)

	if downstream, err = ConnectSerial(path, flags); nil != err {
		log.Fatal(err)
	}

	if upstream, err = pickUpstream(flags); nil != err {
		log.Fatalf("failed to connect to upstream; %v\n", err)
	} else {
		loop(upstream, downstream, flags)
	}
}
<commit_msg>Fix vsn tag<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"
)

var (
	vsn = "0.0.1-rc3"
)

type Remote interface {
	Channel() chan []byte
	Close() error
	Connect() error
	Recv(timeout time.Duration) ([]byte, error)
	Write(buf []byte, timeout time.Duration) (int, error)
}

type Flags struct {
	help    bool
	list    bool
	version bool

	verify        bool
	nid           Address
	sid           Address
	uid           Address
	autoConfigure bool

	stdio     bool
	remote    string
	tls       bool
	reconnect bool
}

func parseFlags() Flags {
	flags := new(Flags)

	// commands
	listFlag := flag.Bool("list", false, "List available serialports")
	helpFlag := flag.Bool("help", false, "Show help text")
	versionFlag := flag.Bool("version", false, "Show version")

	// link flags
	verifyFlag := flag.Bool("verify", true, "validate IDs according to -nid, -sid, and -uid flags")
	autoConfigureFlag := flag.Bool("auto-configure", false, "Automatically configure gateway operation and ID's; use -nid, -sid, and -uid flags")
	nidFlag := flag.String("nid", "::", "32bit Network ID in hexadecimal (ie, aa:bb:cc:dd)")
	sidFlag := flag.String("sid", "::", "32bit System ID in hexadecimal (ie, aa:bb:cc:dd)")
	uidFlag := flag.String("uid", "::", "32bit Unique ID in hexadecimal (ie, aa:bb:cc:dd)")

	// communication flags
	stdioFlag := flag.Bool("stdio", false, "Use stdio for communication instead of remote")
	remoteFlag := flag.String("remote", "tcp.cloud.tiny-mesh.com:7002", "The upstream url to connect to")
	usetlsFlag := flag.Bool("tls", true, "Controll use of TLS with -remote")
	reconnectFlag := flag.Bool("reconnect", true, "Automatically re-establish communication on failure")

	flag.Parse()

	flags.help = *helpFlag
	flags.list = *listFlag
	flags.version = *versionFlag

	flags.verify = *verifyFlag
	flags.autoConfigure = *autoConfigureFlag
	flags.nid = parseAddr(*nidFlag)
	flags.sid = parseAddr(*sidFlag)
	flags.uid = parseAddr(*uidFlag)

	if len(flags.nid) == 0 {
		log.Fatalf("failed to parse -nid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -nid 01:02:03:04\n")
	}
	if len(flags.sid) == 0 {
		log.Fatalf("failed to parse -sid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -sid 01:02:03:04\n")
	}
	if len(flags.uid) == 0 {
		log.Fatalf("failed to parse -uid value, value must be 4 bytes encoded as hexadecimals with : as a separator\nexample: -uid 01:02:03:04\n")
	}

	flags.stdio = *stdioFlag
	flags.remote = *remoteFlag
	flags.tls = *usetlsFlag
	flags.reconnect = *reconnectFlag

	return *flags
}

func pickUpstream(flags Flags) (Remote, error) {
	if true == flags.stdio {
		// stdio
		return ConnectStdio(os.Stdin, os.Stdout)
	} else if true == flags.tls {
		// tls
		return ConnectTLS(flags.remote)
	}

	return ConnectTCP(flags.remote)
}

func forward(remote Remote, t time.Duration) chan []byte {
	ch := make(chan []byte, 256)

	go func() {
		for {
			buf, err := remote.Recv(t)

			if nil != err {
				log.Printf("forward-err: %v\n", err)
				close(ch)
				return
			}

			ch <- buf
		}
	}()

	return ch
}

func loop(from Remote, to Remote, flags Flags) {
	upstream := forward(from, 500*time.Millisecond)
	downstream := forward(to, 2*time.Millisecond)

	upoff := &Backoff{
		initial: 1 * time.Second,
		wait:    1 * time.Second,
		delay:   2.5,
		max:     5 * time.Minute,
	}

	downoff := &Backoff{
		initial: 1 * time.Second,
		wait:    1 * time.Second,
		delay:   2.5,
		max:     5 * time.Minute,
	}

	for {
		select {
		case buf, state := <-upstream:
			if false == state {
				if !flags.reconnect {
					log.Fatalf("upstream:close, exiting")
				}

				log.Printf("upstream:close, reconnecting\n")
				from.Close()
				if err := from.Connect(); nil != err {
					log.Printf("upstream:open: %v\n", err)
					upoff.Fail()
				} else {
					upoff.Success()
					upstream = forward(from, 500*time.Millisecond)
				}
			} else if len(buf) > 0 {
				log.Printf("upstream:recv[%v] %v\n", state, buf)
				if len(buf) > 10 && 6 == buf[0] {
					to.Write(buf[:1], -1)
					to.Write(buf[1:], -1)
				} else {
					to.Write(buf, -1)
				}
			}

		case buf, state := <-downstream:
			if false == state {
				if !flags.reconnect {
					log.Fatalf("upstream:close, exiting")
				}

				to.Close()
				log.Printf("downstream:close, reconnecting\n")
				if err := to.Connect(); nil != err {
					log.Printf("downstream:open: %v\n", err)
					downoff.Fail()
				} else {
					downoff.Success()
					downstream = forward(to, 2*time.Millisecond)
				}
			} else if len(buf) > 0 {
				log.Printf("downstream:recv[%v] %v\n", state, buf)
				from.Write(buf, -1)
			}
		}
	}
}

func main() {

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	flags := parseFlags()

	if true == flags.help {
		flag.PrintDefaults()
		return
	} else if true == flags.list {
		PrintPortList()
		return
	} else if true == flags.version {
		fmt.Printf("%v\n", vsn)
		return
	}

	path := flag.Arg(0)

	if "" == path {
		log.Fatal(errors.New("1st argument, tty path, missing"))
	}

	var upstream Remote
	var downstream Remote
	var err error

	log.Printf("guri - version %v\n", vsn)

	if downstream, err = ConnectSerial(path, flags); nil != err {
		log.Fatal(err)
	}

	if upstream, err = pickUpstream(flags); nil != err {
		log.Fatalf("failed to connect to upstream; %v\n", err)
	} else {
		loop(upstream, downstream, flags)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017, Jonsen Yang.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	. "github.com/forease/ebase"
	"io"
	"os"
	"strings"
	"time"
)

type queue struct {
	expiry time.Time
	last   time.Time
	days   int64
}

var (
	cache    = make(map[string]queue)
	smtp     *Smtp
	mailAddr string
	retry    int
)

//
// check rules:
// more than 120 days, check every 30 days.
// more than 60 days, check every 7 days.
// more than 30 days, check every 3 days.
// less than 30 days, check every day.
func scan(fileName string) {
	Log.Debug("start scan domain")
	// get domain from file
	f, err := os.Open(fileName)
	if err != nil {
		Log.Error(err)
		return
	}
	defer f.Close()

	now := time.Now()
	nowUnix := now.Unix()

	mailList := make(map[string]int64)
	count := 0

	buf := bufio.NewReader(f)
	for {

		domain, err := buf.ReadString('\n')

		if err != nil {
			if err == io.EOF {
				break
			}
			Log.Error(err)
			break
		}

		domain = strings.TrimSpace(domain)
		if domain == "" {
			continue
		}

		extDomain, ok := cache[domain]
		if ok {
			switch {
			case extDomain.days > 120 && !extDomain.last.Before(now.Add(-24*time.Hour*30)):
				fallthrough
			case extDomain.days >= 60 && !extDomain.last.Before(now.Add(-24*time.Hour*7)):
				fallthrough
			case extDomain.days >= 30 && !extDomain.last.Before(now.Add(-24*time.Hour*3)):
				Log.Debugf("domain %s not need scan. expiry day %d, last scan %s.",
					domain, extDomain.days, extDomain.last)
				continue
			default:
			}

		}

		var ss []byte

		for i := 0; i < retry; i++ {
			ss, err = Whois(domain)
			if err == nil {
				break
			}
		}

		if err != nil {
			Log.Errorf("domain %s error %s", domain, err)
			continue
		}

		d := parse(ss)
		count++

		// Check expiry time
		exDay := (d.Expiry.Unix() - nowUnix) / 86400
		if ok {
			extDomain.last = now
			extDomain.days = exDay
		} else {
			cache[domain] = queue{expiry: d.Expiry, last: now, days: exDay}
		}

		mailList[domain] = exDay

		time.Sleep(time.Second * 3)
	}

	// Send email
	total := len(mailList)
	if len(mailList) > 0 {
		subject := fmt.Sprintf("%d domains status report at %s", total,
			now.Format("2006-01-02 15:04:05"))

		content := "Hi,\n\n"

		list := NewMapSorter(mailList)
		for sn, data := range list {
			date := cache[data.Key].expiry.Format("2006-01-02")
			content += fmt.Sprintf("   %d. %s will expire on %d (%s) days.\n",
				sn+1, data.Key, data.Val, date)
		}

		content += "\n\n    report by dmonitord\n\n"

		content += "\r\n\r\n"

		//  MailSender(subject, content, to, cc, bcc string)
		if err := smtp.MailSender(subject, content, mailAddr, "", ""); err != nil {
			Log.Error(err)
		}
	}

	Log.Debugf("end scan domain. with %d domain, using %s.", count, time.Now().Sub(now))

}

func main() {
	EbaseInit()

	/*
		model, err := NewDefaultModels()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		defer model.Close()
	*/

	dFile, err := Config.String("common.dfile", "")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = os.Stat(dFile)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	smtp = NewSmtp()
	mailAddr, _ = Config.String("common.adminer", "")
	retry, _ = Config.Int("common.retry", 3)

	Log.Info("dmonitord running now...")

	go func() {

		for {

			time.Sleep(time.Second)
			now := time.Now()

			hour := now.Hour()
			minute := now.Minute()
			second := now.Second()

			if hour == 3 && minute == 0 && second == 0 {
				scan(dFile)
			}
		}

	}()

	SignalHandle(map[string]interface{}{})
}
<commit_msg>change set checking time<commit_after>// Copyright 2017, Jonsen Yang.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	. "github.com/forease/ebase"
	"io"
	"os"
	"strings"
	"time"
)

type queue struct {
	expiry time.Time
	last   time.Time
	days   int64
}

var (
	cache    = make(map[string]queue)
	smtp     *Smtp
	mailAddr string
	retry    int
)

//
// check rules:
// more than 120 days, check every 30 days.
// more than 60 days, check every 7 days.
// more than 30 days, check every 3 days.
// less than 30 days, check every day.
func scan(fileName string) {
	Log.Debug("start scan domain")
	// get domain from file
	f, err := os.Open(fileName)
	if err != nil {
		Log.Error(err)
		return
	}
	defer f.Close()

	now := time.Now()
	nowUnix := now.Unix()

	mailList := make(map[string]int64)
	count := 0

	buf := bufio.NewReader(f)
	for {

		domain, err := buf.ReadString('\n')

		if err != nil {
			if err == io.EOF {
				break
			}
			Log.Error(err)
			break
		}

		domain = strings.TrimSpace(domain)
		if domain == "" {
			continue
		}

		extDomain, ok := cache[domain]
		if ok {
			switch {
			case extDomain.days > 120 && !extDomain.last.Before(now.Add(-24*time.Hour*30)):
				fallthrough
			case extDomain.days >= 60 && !extDomain.last.Before(now.Add(-24*time.Hour*7)):
				fallthrough
			case extDomain.days >= 30 && !extDomain.last.Before(now.Add(-24*time.Hour*3)):
				Log.Debugf("domain %s not need scan. expiry day %d, last scan %s.",
					domain, extDomain.days, extDomain.last)
				continue
			default:
			}

		}

		var ss []byte

		for i := 0; i < retry; i++ {
			ss, err = Whois(domain)
			if err == nil {
				break
			}
		}

		if err != nil {
			Log.Errorf("domain %s error %s", domain, err)
			continue
		}

		d := parse(ss)
		count++

		// Check expiry time
		exDay := (d.Expiry.Unix() - nowUnix) / 86400
		if ok {
			extDomain.last = now
			extDomain.days = exDay
		} else {
			cache[domain] = queue{expiry: d.Expiry, last: now, days: exDay}
		}

		mailList[domain] = exDay

		time.Sleep(time.Second * 3)
	}

	// Send email
	total := len(mailList)
	if len(mailList) > 0 {
		subject := fmt.Sprintf("%d domains status report at %s", total,
			now.Format("2006-01-02 15:04:05"))

		content := "Hi,\n\n"

		list := NewMapSorter(mailList)
		for sn, data := range list {
			date := cache[data.Key].expiry.Format("2006-01-02")
			content += fmt.Sprintf("   %d. %s will expire on %d (%s) days.\n",
				sn+1, data.Key, data.Val, date)
		}

		content += "\n\n    report by dmonitord\n\n"

		content += "\r\n\r\n"

		//  MailSender(subject, content, to, cc, bcc string)
		if err := smtp.MailSender(subject, content, mailAddr, "", ""); err != nil {
			Log.Error(err)
		}
	}

	Log.Debugf("end scan domain. with %d domain, using %s.", count, time.Now().Sub(now))

}

func main() {
	EbaseInit()

	/*
		model, err := NewDefaultModels()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		defer model.Close()
	*/

	dFile, err := Config.String("common.dfile", "")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = os.Stat(dFile)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	smtp = NewSmtp()
	mailAddr, _ = Config.String("common.adminer", "")
	retry, _ = Config.Int("common.retry", 3)
	cHour, _ := Config.Int("check.hour", 5)
	cMinute, _ := Config.Int("check.minute", 0)
	cSecond, _ := Config.Int("check.second", 0)

	Log.Info("dmonitord running now...")

	go func() {

		for {

			time.Sleep(time.Second)
			now := time.Now()

			hour := now.Hour()
			minute := now.Minute()
			second := now.Second()

			if hour == cHour && minute == cMinute && second == cSecond {
				scan(dFile)
			}
		}

	}()

	SignalHandle(map[string]interface{}{})
}
<|endoftext|>


<commit_before>package termite

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"rand"
	"crypto/hmac"
	"sync"
	"io"
)

const challengeLength = 20

func sign(conn net.Conn, challenge []byte, secret []byte, local bool) []byte {
	h := hmac.NewSHA1(secret)
	h.Write(challenge)
	l := conn.LocalAddr()
	r := conn.RemoteAddr()
	if local {
		h.Write([]byte(fmt.Sprintf("%v-%v", l, r)))
	} else {
		h.Write([]byte(fmt.Sprintf("%v-%v", r, l)))
	}
	return h.Sum()
}

// Symmetrical authentication using HMAC-SHA1.
//
// To authenticate, we do  the following:
//
// * Receive 20-byte random challenge
// * Using the secret, sign (challenge + remote address + local address)
// * Return the signature
//
// TODO - should probably use SSL/TLS? Figure out what is useful and
// necessary here.
func Authenticate(conn net.Conn, secret []byte) os.Error {
	challenge := RandomBytes(challengeLength)

	_, err := conn.Write(challenge)
	if err != nil {
		return err
	}
	expected := sign(conn, challenge, secret, true)

	remoteChallenge := make([]byte, challengeLength)
	n, err := conn.Read(remoteChallenge)
	if err != nil {
		return err
	}
	remoteChallenge = remoteChallenge[:n]
	_, err = conn.Write(sign(conn, remoteChallenge, secret, false))

	response := make([]byte, len(expected))
	n, err = conn.Read(response)
	if err != nil {
		return err
	}
	response = response[:n]

	if bytes.Compare(response, expected) != 0 {
		log.Println("Authentication failure from", conn.RemoteAddr())
		conn.Close()
		return os.NewError("Mismatch in response")
	}

	expectAck := []byte("OK")
	conn.Write(expectAck)

	ack := make([]byte, len(expectAck))
	n, err = conn.Read(ack)
	if err != nil {
		return err
	}

	ack = ack[:n]
	if bytes.Compare(expectAck, ack) != 0 {
		fmt.Println(expectAck, ack)
		return os.NewError("Missing ack reply")
	}

	return nil
}

func SetupServer(port int, secret []byte, output chan net.Conn) {
	host, _ := os.Hostname()
	addr := fmt.Sprintf("%s:%d", host, port)
	// TODO - also listen on localhost.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal("net.Listen", err)
	}
	log.Println("Listening to", addr)

	for {
		conn, err := listener.Accept()
		if err != nil {
			panic(err)
		}

		err = Authenticate(conn, secret)
		if err != nil {
			log.Println("Authentication error: ", err)
			conn.Close()
			continue
		}
		output <- conn
	}
}

// ids:
//
const (
	RPC_CHANNEL = "rpc....."
	_ID_FMT     = "id%06d"
	HEADER_LEN  = 8
)

func ConnectionId() string {
	id := rand.Intn(1e6)
	return fmt.Sprintf(_ID_FMT, id)
}

type PendingConnection struct {
	Id    string
	Ready sync.Cond
	Conn  net.Conn
}

// PendingConnections manages a list of connections, indexed by ID.
// The id is sent as the first 8 bytes, after the authentication.
type PendingConnections struct {
	connectionsMutex sync.Mutex
	connections      map[string]*PendingConnection
}

func NewPendingConnections() *PendingConnections {
	return &PendingConnections{
		connections: make(map[string]*PendingConnection),
	}
}

func (me *PendingConnections) newPendingConnection(id string) *PendingConnection {
	p := &PendingConnection{
		Id: id,
	}
	p.Ready.L = &me.connectionsMutex
	return p
}

func (me *PendingConnections) WaitConnection(id string) net.Conn {
	me.connectionsMutex.Lock()
	defer me.connectionsMutex.Unlock()
	p := me.connections[id]
	if p == nil {
		p = me.newPendingConnection(id)
		me.connections[id] = p
	}

	for p.Conn == nil {
		p.Ready.Wait()
	}

	me.connections[id] = nil
	return p.Conn
}

func (me *PendingConnections) Accept(conn net.Conn) bool {
	idBytes := make([]byte, HEADER_LEN)
	n, err := conn.Read(idBytes)
	if n != HEADER_LEN || err != nil {
		// TODO -an error?
		conn.Close()
		return true
	}
	id := string(idBytes)
	if id == RPC_CHANNEL {
		return false
	}

	me.connectionsMutex.Lock()
	defer me.connectionsMutex.Unlock()
	p := me.connections[id]
	if p == nil {
		p = me.newPendingConnection(id)
		me.connections[id] = p
	}
	if p.Conn != nil {
		panic(fmt.Sprintf("accepted the same connection id twice: %s", id))
	}
	p.Conn = conn
	p.Ready.Signal()
	return true
}

func DialTypedConnection(addr string, id string, secret []byte) (net.Conn, os.Error) {
	if len(id) != HEADER_LEN {
		log.Fatal("id != 8", id, len(id))
	}
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	err = Authenticate(conn, secret)
	if err != nil {
		return nil, err
	}
	_, err = io.WriteString(conn, id)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func OpenSocketConnection(socket string, channel string) net.Conn {
	conn, err := net.Dial("unix", socket)
	if err != nil {
		log.Fatal("Dial:", err)
	}
	if len(channel) != HEADER_LEN {
		panic(channel)
	}
	_, err = io.WriteString(conn, channel)
	if err != nil {
		log.Fatal("WriteString", err)
	}
	return conn
}
<commit_msg>Use exponential backoff if unix socket connect returns EAGAIN.<commit_after>package termite

import (
	"bytes"
	"crypto/hmac"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"rand"
	"sync"
	"time"
)

const challengeLength = 20

func sign(conn net.Conn, challenge []byte, secret []byte, local bool) []byte {
	h := hmac.NewSHA1(secret)
	h.Write(challenge)
	l := conn.LocalAddr()
	r := conn.RemoteAddr()
	if local {
		h.Write([]byte(fmt.Sprintf("%v-%v", l, r)))
	} else {
		h.Write([]byte(fmt.Sprintf("%v-%v", r, l)))
	}
	return h.Sum()
}

// Symmetrical authentication using HMAC-SHA1.
//
// To authenticate, we do  the following:
//
// * Receive 20-byte random challenge
// * Using the secret, sign (challenge + remote address + local address)
// * Return the signature
//
// TODO - should probably use SSL/TLS? Figure out what is useful and
// necessary here.
func Authenticate(conn net.Conn, secret []byte) os.Error {
	challenge := RandomBytes(challengeLength)

	_, err := conn.Write(challenge)
	if err != nil {
		return err
	}
	expected := sign(conn, challenge, secret, true)

	remoteChallenge := make([]byte, challengeLength)
	n, err := conn.Read(remoteChallenge)
	if err != nil {
		return err
	}
	remoteChallenge = remoteChallenge[:n]
	_, err = conn.Write(sign(conn, remoteChallenge, secret, false))

	response := make([]byte, len(expected))
	n, err = conn.Read(response)
	if err != nil {
		return err
	}
	response = response[:n]

	if bytes.Compare(response, expected) != 0 {
		log.Println("Authentication failure from", conn.RemoteAddr())
		conn.Close()
		return os.NewError("Mismatch in response")
	}

	expectAck := []byte("OK")
	conn.Write(expectAck)

	ack := make([]byte, len(expectAck))
	n, err = conn.Read(ack)
	if err != nil {
		return err
	}

	ack = ack[:n]
	if bytes.Compare(expectAck, ack) != 0 {
		fmt.Println(expectAck, ack)
		return os.NewError("Missing ack reply")
	}

	return nil
}

func SetupServer(port int, secret []byte, output chan net.Conn) {
	host, _ := os.Hostname()
	addr := fmt.Sprintf("%s:%d", host, port)
	// TODO - also listen on localhost.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal("net.Listen", err)
	}
	log.Println("Listening to", addr)

	for {
		conn, err := listener.Accept()
		if err != nil {
			panic(err)
		}

		err = Authenticate(conn, secret)
		if err != nil {
			log.Println("Authentication error: ", err)
			conn.Close()
			continue
		}
		output <- conn
	}
}

// ids:
//
const (
	RPC_CHANNEL = "rpc....."
	_ID_FMT     = "id%06d"
	HEADER_LEN  = 8
)

func ConnectionId() string {
	id := rand.Intn(1e6)
	return fmt.Sprintf(_ID_FMT, id)
}

type PendingConnection struct {
	Id    string
	Ready sync.Cond
	Conn  net.Conn
}

// PendingConnections manages a list of connections, indexed by ID.
// The id is sent as the first 8 bytes, after the authentication.
type PendingConnections struct {
	connectionsMutex sync.Mutex
	connections      map[string]*PendingConnection
}

func NewPendingConnections() *PendingConnections {
	return &PendingConnections{
		connections: make(map[string]*PendingConnection),
	}
}

func (me *PendingConnections) newPendingConnection(id string) *PendingConnection {
	p := &PendingConnection{
		Id: id,
	}
	p.Ready.L = &me.connectionsMutex
	return p
}

func (me *PendingConnections) WaitConnection(id string) net.Conn {
	me.connectionsMutex.Lock()
	defer me.connectionsMutex.Unlock()
	p := me.connections[id]
	if p == nil {
		p = me.newPendingConnection(id)
		me.connections[id] = p
	}

	for p.Conn == nil {
		p.Ready.Wait()
	}

	me.connections[id] = nil
	return p.Conn
}

func (me *PendingConnections) Accept(conn net.Conn) bool {
	idBytes := make([]byte, HEADER_LEN)
	n, err := conn.Read(idBytes)
	if n != HEADER_LEN || err != nil {
		// TODO -an error?
		conn.Close()
		return true
	}
	id := string(idBytes)
	if id == RPC_CHANNEL {
		return false
	}

	me.connectionsMutex.Lock()
	defer me.connectionsMutex.Unlock()
	p := me.connections[id]
	if p == nil {
		p = me.newPendingConnection(id)
		me.connections[id] = p
	}
	if p.Conn != nil {
		panic(fmt.Sprintf("accepted the same connection id twice: %s", id))
	}
	p.Conn = conn
	p.Ready.Signal()
	return true
}

func DialTypedConnection(addr string, id string, secret []byte) (net.Conn, os.Error) {
	if len(id) != HEADER_LEN {
		log.Fatal("id != 8", id, len(id))
	}
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	err = Authenticate(conn, secret)
	if err != nil {
		return nil, err
	}
	_, err = io.WriteString(conn, id)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

const _MAXTRY = 3

func OpenSocketConnection(socket string, channel string) net.Conn {
	delay := int64(0)
	conn, err := net.Dial("unix", socket)
	for try := 0; err != nil && try < _MAXTRY; try++ {
		if err == os.EAGAIN {
			delay = int64(1.0 + rand.Float64()) * delay + 0.01e9
			time.Sleep(int64(delay))
			conn, err = net.Dial("unix", socket)
			continue
		}
		if err != nil {
			log.Fatal("Dial:", err)
		}
	}
	if len(channel) != HEADER_LEN {
		panic(channel)
	}
	_, err = io.WriteString(conn, channel)
	if err != nil {
		log.Fatal("WriteString", err)
	}
	return conn
}
<|endoftext|>


<commit_before>package suite

import (
	"testing"
)

// TestingSuite can store and return the current *testing.T context
// generated by 'go test'.
type TestingSuite interface {
	T() *testing.T
	SetT(*testing.T)
}

// SetupAllSuite has a SetupSuite method, which will run before the
// tests in the suite are run.
type SetupAllSuite interface {
	SetupSuite()
}

// SetupTestSuite has a SetupTest method, which will run before each
// test in the suite.
type SetupTestSuite interface {
	SetupTest()
}

// TearDownAllSuite has a TearDownSuite method, which will run after
// all the tests in the suite have been run.
type TearDownAllSuite interface {
	TearDownSuite()
}

// TearDownTestSuite has a TearDownTest method, which will run after
// each test in the suite.
type TearDownTestSuite interface {
	TearDownTest()
}

// BeforeTest has a function to be executed right before the test
// starts and receives the suite and test names as input
type BeforeTest interface {
	BeforeTest(suiteName, testName string)
}

// AfterTest has a function to be executed right after the test
// finishes and receives the suite and test names as input
type AfterTest interface {
	AfterTest(suiteName, testName string)
}

// WithStats is a suite which measures some stats for its tests.
type WithStats interface {
	HandleStats(suiteName string, stats *SuiteInformation)
}
<commit_msg>Update documentation<commit_after>package suite

import (
	"testing"
)

// TestingSuite can store and return the current *testing.T context
// generated by 'go test'.
type TestingSuite interface {
	T() *testing.T
	SetT(*testing.T)
}

// SetupAllSuite has a SetupSuite method, which will run before the
// tests in the suite are run.
type SetupAllSuite interface {
	SetupSuite()
}

// SetupTestSuite has a SetupTest method, which will run before each
// test in the suite.
type SetupTestSuite interface {
	SetupTest()
}

// TearDownAllSuite has a TearDownSuite method, which will run after
// all the tests in the suite have been run.
type TearDownAllSuite interface {
	TearDownSuite()
}

// TearDownTestSuite has a TearDownTest method, which will run after
// each test in the suite.
type TearDownTestSuite interface {
	TearDownTest()
}

// BeforeTest has a function to be executed right before the test
// starts and receives the suite and test names as input
type BeforeTest interface {
	BeforeTest(suiteName, testName string)
}

// AfterTest has a function to be executed right after the test
// finishes and receives the suite and test names as input
type AfterTest interface {
	AfterTest(suiteName, testName string)
}

// WithStats implements HandleStats, a function that will be executed
// when a test suite is finished. The stats contain information about
// the execution of that suite and its tests.
type WithStats interface {
	HandleStats(suiteName string, stats *SuiteInformation)
}
<|endoftext|>


<commit_before>package svnwatch

import (
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"strings"

	shellwords "github.com/mattn/go-shellwords"
	"github.com/pkg/errors"

	"github.com/jackwilsdon/svnwatch/svn"
)

var commandTypes = make(map[string]CommandType)

// A CommandType is a method of passing data about a revision into a command.
type CommandType func(cmd *exec.Cmd, repository Repository, revision svn.Revision) error

// RegisterCommandType registers the specified command type under the name provided.
func RegisterCommandType(name string, commandType CommandType) {
	if commandType == nil {
		panic(fmt.Sprintf("type \"%s\" is nil", name))
	}

	if _, registered := commandTypes[name]; registered {
		panic(fmt.Sprintf("type \"%s\" is already registered", name))
	}

	commandTypes[name] = commandType
}

// Command represents a command that is executed when a change is detected.
type Command struct {
	XMLName xml.Name `xml:"command"`
	Type    string   `xml:"type,attr,omitempty"`
	Command string   `xml:",chardata"`
}

// Execute the command on the specified repository and revision.
func (command Command) Execute(repository Repository, revision svn.Revision) error {
	pieces, err := shellwords.Parse(command.Command)

	if err != nil {
		return errors.Wrapf(err, "failed to parse \"%s\"", command.Command)
	}

	if len(pieces) == 0 {
		return fmt.Errorf("failed to parse \"%s\"", command.Command)
	}

	cmd := exec.Command(pieces[0], pieces[1:]...)
	cmd.Env = os.Environ()

	commandType, registered := commandTypes[command.Type]

	if !registered {
		keys := make([]string, 0, len(commandTypes))

		for key := range commandTypes {
			keys = append(keys, key)
		}

		return fmt.Errorf("invalid type \"%s\" for \"%s\" (supported types: %s)", command.Type, command.Command, strings.Join(keys, ", "))
	}

	if err := commandType(cmd, repository, revision); err != nil {
		return errors.Wrapf(err, "failed to use command type \"%s\"", command.Type)
	}

	if err := cmd.Run(); err != nil {
		return errors.Wrapf(err, "failed to execute \"%s\"", command.Command)
	}

	return nil
}

// UnmarshalXML unmarshals the command from XML whilst providing some extra
// validation.
func (command *Command) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	cmd := struct {
		Type    *string `xml:"type,attr,omitempty"`
		Command string  `xml:",chardata"`
	}{}

	if err := decoder.DecodeElement(&cmd, &start); err != nil {
		return err
	}

	if cmd.Type == nil {
		return fmt.Errorf("missing type for \"%s\"", cmd.Command)
	}

	if _, registered := commandTypes[*cmd.Type]; !registered {
		keys := make([]string, 0, len(commandTypes))

		for key := range commandTypes {
			keys = append(keys, key)
		}

		return fmt.Errorf("invalid type \"%s\" for \"%s\" (supported types: %s)", *cmd.Type, cmd.Command, strings.Join(keys, ", "))
	}

	command.Type = *cmd.Type
	command.Command = cmd.Command

	return nil
}
<commit_msg>Forward stdout and stderr from executing commands<commit_after>package svnwatch

import (
	"encoding/xml"
	"fmt"
	"os"
	"os/exec"
	"strings"

	shellwords "github.com/mattn/go-shellwords"
	"github.com/pkg/errors"

	"github.com/jackwilsdon/svnwatch/svn"
)

var commandTypes = make(map[string]CommandType)

// A CommandType is a method of passing data about a revision into a command.
type CommandType func(cmd *exec.Cmd, repository Repository, revision svn.Revision) error

// RegisterCommandType registers the specified command type under the name provided.
func RegisterCommandType(name string, commandType CommandType) {
	if commandType == nil {
		panic(fmt.Sprintf("type \"%s\" is nil", name))
	}

	if _, registered := commandTypes[name]; registered {
		panic(fmt.Sprintf("type \"%s\" is already registered", name))
	}

	commandTypes[name] = commandType
}

// Command represents a command that is executed when a change is detected.
type Command struct {
	XMLName xml.Name `xml:"command"`
	Type    string   `xml:"type,attr,omitempty"`
	Command string   `xml:",chardata"`
}

// Execute the command on the specified repository and revision.
func (command Command) Execute(repository Repository, revision svn.Revision) error {
	pieces, err := shellwords.Parse(command.Command)

	if err != nil {
		return errors.Wrapf(err, "failed to parse \"%s\"", command.Command)
	}

	if len(pieces) == 0 {
		return fmt.Errorf("failed to parse \"%s\"", command.Command)
	}

	cmd := exec.Command(pieces[0], pieces[1:]...)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	commandType, registered := commandTypes[command.Type]

	if !registered {
		keys := make([]string, 0, len(commandTypes))

		for key := range commandTypes {
			keys = append(keys, key)
		}

		return fmt.Errorf("invalid type \"%s\" for \"%s\" (supported types: %s)", command.Type, command.Command, strings.Join(keys, ", "))
	}

	if err := commandType(cmd, repository, revision); err != nil {
		return errors.Wrapf(err, "failed to use command type \"%s\"", command.Type)
	}

	if err := cmd.Run(); err != nil {
		return errors.Wrapf(err, "failed to execute \"%s\"", command.Command)
	}

	return nil
}

// UnmarshalXML unmarshals the command from XML whilst providing some extra
// validation.
func (command *Command) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	cmd := struct {
		Type    *string `xml:"type,attr,omitempty"`
		Command string  `xml:",chardata"`
	}{}

	if err := decoder.DecodeElement(&cmd, &start); err != nil {
		return err
	}

	if cmd.Type == nil {
		return fmt.Errorf("missing type for \"%s\"", cmd.Command)
	}

	if _, registered := commandTypes[*cmd.Type]; !registered {
		keys := make([]string, 0, len(commandTypes))

		for key := range commandTypes {
			keys = append(keys, key)
		}

		return fmt.Errorf("invalid type \"%s\" for \"%s\" (supported types: %s)", *cmd.Type, cmd.Command, strings.Join(keys, ", "))
	}

	command.Type = *cmd.Type
	command.Command = cmd.Command

	return nil
}
<|endoftext|>


<commit_before>package cloud

import (
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/appscode/go/log"
	"golang.org/x/crypto/ssh"
)

func ExecuteTCPCommand(command, addr string, config *ssh.ClientConfig) (string, error) {
	conn, err := ssh.Dial("tcp", addr, config)
	if err != nil {
		return "", err
	}
	defer conn.Close()
	session, err := conn.NewSession()
	if err != nil {
		return "", err
	}
	session.Stdout = DefaultWriter
	session.Stderr = DefaultWriter
	session.Stdin = os.Stdin
	session.Run(command)
	output := DefaultWriter.Output()
	session.Close()
	return output, nil
}

func ExecuteSSHCommand(name string, arg []string, stdIn io.Reader) (string, error) {
	cmd := exec.Command(name, arg...)
	cmd.Stdin = stdIn
	cmd.Stdout = DefaultWriter
	cmd.Stderr = DefaultWriter
	err := cmd.Run()
	output := DefaultWriter.Output()
	return output, err
}

var DefaultWriter = &StringWriter{
	data: make([]byte, 0),
}

type StringWriter struct {
	data []byte
}

func (s *StringWriter) Flush() {
	s.data = make([]byte, 0)
}

func (s *StringWriter) Output() string {
	return string(s.data)
}

func (s *StringWriter) Write(b []byte) (int, error) {
	log.Infoln("$ ", string(b))
	s.data = append(s.data, b...)
	return len(b), nil
}

func NewStringReader(ss []string) io.Reader {
	formattedString := strings.Join(ss, "\n")
	reader := strings.NewReader(formattedString)
	return reader
}
<commit_msg>ssh with not root user fix (#389)<commit_after>package cloud

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/appscode/go/log"
	"golang.org/x/crypto/ssh"
)

func ExecuteTCPCommand(command, addr string, config *ssh.ClientConfig) (string, error) {
	conn, err := ssh.Dial("tcp", addr, config)
	if err != nil {
		return "", err
	}
	defer conn.Close()
	session, err := conn.NewSession()
	if err != nil {
		return "", err
	}
	session.Stdout = DefaultWriter
	session.Stderr = DefaultWriter
	session.Stdin = os.Stdin
	if config.User != "root" {
		command = fmt.Sprintf("sudo %s" + command)
	}
	session.Run(command)
	output := DefaultWriter.Output()
	session.Close()
	return output, nil
}

func ExecuteSSHCommand(name string, arg []string, stdIn io.Reader) (string, error) {
	cmd := exec.Command(name, arg...)
	cmd.Stdin = stdIn
	cmd.Stdout = DefaultWriter
	cmd.Stderr = DefaultWriter
	err := cmd.Run()
	output := DefaultWriter.Output()
	return output, err
}

var DefaultWriter = &StringWriter{
	data: make([]byte, 0),
}

type StringWriter struct {
	data []byte
}

func (s *StringWriter) Flush() {
	s.data = make([]byte, 0)
}

func (s *StringWriter) Output() string {
	return string(s.data)
}

func (s *StringWriter) Write(b []byte) (int, error) {
	log.Infoln("$ ", string(b))
	s.data = append(s.data, b...)
	return len(b), nil
}

func NewStringReader(ss []string) io.Reader {
	formattedString := strings.Join(ss, "\n")
	reader := strings.NewReader(formattedString)
	return reader
}
<|endoftext|>


<commit_before>// vim: tabstop=2 shiftwidth=2

package main

import (
	"fmt"
	"os"
	"strings"
	"path"
	"io/ioutil"
	"net/mail"
	"crypto/sha256"
	"encoding/hex"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/idlog"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

// poolRead returns a dynamic Mix of filenames from the outbound pool.
// If flag_send is true, all files will be returned in random order,
// otherwise the configured pool size and rate rules will be applied.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size && ! flag_send {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else if flag_send {
		Info.Println("Flushing outbound pool")
		numToSend = poolSize
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	Trace.Printf("Processing %d pool messages.\n", poolSize)
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) (err error) {
	// Delete a pool file
	err = os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	}
	return
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(public *keymgr.Pubring, secret *keymgr.Secring, id idlog.IDLog) (err error) {
	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	if len(keys) > 0 {
		Trace.Printf("Reading %d messages from %s\n", len(keys), cfg.Files.Maildir)
	}
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err != nil {
				Info.Println(err)
			}
		} else {
			// It's not a remailer-foo request so assume a remailer message
			var mailMsg *mail.Message
			mailMsg, err := dir.Message(key)
			if err != nil {
				Warn.Printf("%s: Reading message failed with: %s", key, err)
				continue
			}
			var msg []byte
			// Convert the armored Yamn message to its byte components
			msg, err = stripArmor(mailMsg.Body)
			if err != nil {
				Info.Println(err)
			}
			if msg == nil {
				Warn.Println("Dearmor returned zero bytes")
				continue
			}
			err = decodeMsg(msg, public, secret, id)
			if err != nil {
				Info.Println(err)
			}
		} // End of remailer-foo or remailer message
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(prefix string, public *keymgr.Pubring, secret *keymgr.Secring, id idlog.IDLog) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, public, secret, id)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// inPoolWrite writes a raw Byte Yamn message to the Inbound pool
func inPoolWrite(yamnMsg []byte) (err error) {
	outfileName := randPoolFilename("i")
	err = ioutil.WriteFile(outfileName, yamnMsg, 0600)
	if err != nil {
	Warn.Printf("Failed to write raw message to inbound pool: %s", err)
		return
	}
	return
}

// outPoolWrite writes a raw Byte Yamn message to the Outbound pool.
// Oubound message files are prefixed with the recipient address.
func outPoolWrite(yamnMsg []byte) {
	digest := sha256.New()
	digest.Write(yamnMsg)
	poolFileName := "m" + hex.EncodeToString(digest.Sum(nil))[:14]
	fqfn := path.Join(cfg.Files.Pooldir, poolFileName)
	err := ioutil.WriteFile(fqfn, yamnMsg, 0600)
	if err != nil {
		Error.Printf("Failed to write payload to pool file: %s", err)
	}
}
<commit_msg>Don't log message count when there aren't any<commit_after>// vim: tabstop=2 shiftwidth=2

package main

import (
	"fmt"
	"os"
	"strings"
	"path"
	"io/ioutil"
	"net/mail"
	"crypto/sha256"
	"encoding/hex"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/idlog"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

// poolRead returns a dynamic Mix of filenames from the outbound pool.
// If flag_send is true, all files will be returned in random order,
// otherwise the configured pool size and rate rules will be applied.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size && ! flag_send {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else if flag_send {
		Info.Println("Flushing outbound pool")
		numToSend = poolSize
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	if poolSize > 0 {
		Trace.Printf("Processing %d pool messages.\n", poolSize)
	}
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) (err error) {
	// Delete a pool file
	err = os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	}
	return
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(public *keymgr.Pubring, secret *keymgr.Secring, id idlog.IDLog) (err error) {
	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	if len(keys) > 0 {
		Trace.Printf("Reading %d messages from %s\n", len(keys), cfg.Files.Maildir)
	}
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err != nil {
				Info.Println(err)
			}
		} else {
			// It's not a remailer-foo request so assume a remailer message
			var mailMsg *mail.Message
			mailMsg, err := dir.Message(key)
			if err != nil {
				Warn.Printf("%s: Reading message failed with: %s", key, err)
				continue
			}
			var msg []byte
			// Convert the armored Yamn message to its byte components
			msg, err = stripArmor(mailMsg.Body)
			if err != nil {
				Info.Println(err)
			}
			if msg == nil {
				Warn.Println("Dearmor returned zero bytes")
				continue
			}
			err = decodeMsg(msg, public, secret, id)
			if err != nil {
				Info.Println(err)
			}
		} // End of remailer-foo or remailer message
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(prefix string, public *keymgr.Pubring, secret *keymgr.Secring, id idlog.IDLog) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, public, secret, id)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// inPoolWrite writes a raw Byte Yamn message to the Inbound pool
func inPoolWrite(yamnMsg []byte) (err error) {
	outfileName := randPoolFilename("i")
	err = ioutil.WriteFile(outfileName, yamnMsg, 0600)
	if err != nil {
	Warn.Printf("Failed to write raw message to inbound pool: %s", err)
		return
	}
	return
}

// outPoolWrite writes a raw Byte Yamn message to the Outbound pool.
// Oubound message files are prefixed with the recipient address.
func outPoolWrite(yamnMsg []byte) {
	digest := sha256.New()
	digest.Write(yamnMsg)
	poolFileName := "m" + hex.EncodeToString(digest.Sum(nil))[:14]
	fqfn := path.Join(cfg.Files.Pooldir, poolFileName)
	err := ioutil.WriteFile(fqfn, yamnMsg, 0600)
	if err != nil {
		Error.Printf("Failed to write payload to pool file: %s", err)
	}
}
<|endoftext|>


<commit_before>package redis

import (
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"gopkg.in/bsm/ratelimit.v1"
)

var (
	errClosed      = errors.New("redis: client is closed")
	errPoolTimeout = errors.New("redis: connection pool timeout")
)

// PoolStats contains pool state information and accumulated stats.
type PoolStats struct {
	Requests uint64 // number of times a connection was requested by the pool
	Waits    uint64 // number of times our pool had to wait for a connection
	Timeouts uint64 // number of times a wait timeout occurred

	TotalConns uint64 // the number of total connections in the pool
	FreeConns  uint64 // the number of free connections in the pool
}

type pool interface {
	First() *conn
	Get() (*conn, bool, error)
	Put(*conn) error
	Remove(*conn, error) error
	Len() int
	FreeLen() int
	Close() error
	Stats() *PoolStats
}

type connList struct {
	cns  []*conn
	mx   sync.Mutex
	len  int32 // atomic
	size int32
}

func newConnList(size int) *connList {
	return &connList{
		cns:  make([]*conn, 0, size),
		size: int32(size),
	}
}

func (l *connList) Len() int {
	return int(atomic.LoadInt32(&l.len))
}

// Reserve reserves place in the list and returns true on success. The
// caller must add or remove connection if place was reserved.
func (l *connList) Reserve() bool {
	len := atomic.AddInt32(&l.len, 1)
	reserved := len <= l.size
	if !reserved {
		atomic.AddInt32(&l.len, -1)
	}
	return reserved
}

// Add adds connection to the list. The caller must reserve place first.
func (l *connList) Add(cn *conn) {
	l.mx.Lock()
	l.cns = append(l.cns, cn)
	l.mx.Unlock()
}

// Remove closes connection and removes it from the list.
func (l *connList) Remove(cn *conn) error {
	defer l.mx.Unlock()
	l.mx.Lock()

	if cn == nil {
		atomic.AddInt32(&l.len, -1)
		return nil
	}

	for i, c := range l.cns {
		if c == cn {
			l.cns = append(l.cns[:i], l.cns[i+1:]...)
			atomic.AddInt32(&l.len, -1)
			return cn.Close()
		}
	}

	if l.closed() {
		return nil
	}
	panic("conn not found in the list")
}

func (l *connList) Replace(cn, newcn *conn) error {
	defer l.mx.Unlock()
	l.mx.Lock()

	for i, c := range l.cns {
		if c == cn {
			l.cns[i] = newcn
			return cn.Close()
		}
	}

	if l.closed() {
		return newcn.Close()
	}
	panic("conn not found in the list")
}

func (l *connList) Close() (retErr error) {
	l.mx.Lock()
	for _, c := range l.cns {
		if err := c.Close(); err != nil {
			retErr = err
		}
	}
	l.cns = nil
	atomic.StoreInt32(&l.len, 0)
	l.mx.Unlock()
	return retErr
}

func (l *connList) closed() bool {
	return l.cns == nil
}

type connPool struct {
	dialer func() (*conn, error)

	rl        *ratelimit.RateLimiter
	opt       *Options
	conns     *connList
	freeConns chan *conn
	stats     PoolStats

	_closed int32

	lastErr atomic.Value
}

func newConnPool(opt *Options) *connPool {
	p := &connPool{
		dialer: newConnDialer(opt),

		rl:        ratelimit.New(3*opt.getPoolSize(), time.Second),
		opt:       opt,
		conns:     newConnList(opt.getPoolSize()),
		freeConns: make(chan *conn, opt.getPoolSize()),
	}
	if p.opt.getIdleTimeout() > 0 {
		go p.reaper()
	}
	return p
}

func (p *connPool) closed() bool {
	return atomic.LoadInt32(&p._closed) == 1
}

func (p *connPool) isIdle(cn *conn) bool {
	return p.opt.getIdleTimeout() > 0 && time.Since(cn.usedAt) > p.opt.getIdleTimeout()
}

// First returns first non-idle connection from the pool or nil if
// there are no connections.
func (p *connPool) First() *conn {
	for {
		select {
		case cn := <-p.freeConns:
			if p.isIdle(cn) {
				var err error
				cn, err = p.replace(cn)
				if err != nil {
					Logger.Printf("pool.replace failed: %s", err)
					continue
				}
			}
			return cn
		default:
			return nil
		}
	}
	panic("not reached")
}

// wait waits for free non-idle connection. It returns nil on timeout.
func (p *connPool) wait() *conn {
	deadline := time.After(p.opt.getPoolTimeout())
	for {
		select {
		case cn := <-p.freeConns:
			if p.isIdle(cn) {
				var err error
				cn, err = p.replace(cn)
				if err != nil {
					Logger.Printf("pool.replace failed: %s", err)
					continue
				}
			}
			return cn
		case <-deadline:
			return nil
		}
	}
	panic("not reached")
}

// Establish a new connection
func (p *connPool) new() (*conn, error) {
	if p.rl.Limit() {
		err := fmt.Errorf(
			"redis: you open connections too fast (last_error=%q)",
			p.loadLastErr(),
		)
		return nil, err
	}

	cn, err := p.dialer()
	if err != nil {
		p.storeLastErr(err.Error())
		return nil, err
	}

	return cn, nil
}

// Get returns existed connection from the pool or creates a new one.
func (p *connPool) Get() (cn *conn, isNew bool, err error) {
	if p.closed() {
		err = errClosed
		return
	}

	atomic.AddUint64(&p.stats.Requests, 1)

	// Fetch first non-idle connection, if available.
	if cn = p.First(); cn != nil {
		return
	}

	// Try to create a new one.
	if p.conns.Reserve() {
		cn, err = p.new()
		if err != nil {
			p.conns.Remove(nil)
			return
		}
		p.conns.Add(cn)
		isNew = true
		return
	}

	// Otherwise, wait for the available connection.
	atomic.AddUint64(&p.stats.Waits, 1)
	if cn = p.wait(); cn != nil {
		return
	}

	atomic.AddUint64(&p.stats.Timeouts, 1)
	err = errPoolTimeout
	return
}

func (p *connPool) Put(cn *conn) error {
	if cn.rd.Buffered() != 0 {
		b, _ := cn.rd.Peek(cn.rd.Buffered())
		err := fmt.Errorf("connection has unread data: %q", b)
		Logger.Print(err)
		return p.Remove(cn, err)
	}
	if p.opt.getIdleTimeout() > 0 {
		cn.usedAt = time.Now()
	}
	p.freeConns <- cn
	return nil
}

func (p *connPool) replace(cn *conn) (*conn, error) {
	newcn, err := p.new()
	if err != nil {
		_ = p.conns.Remove(cn)
		return nil, err
	}
	_ = p.conns.Replace(cn, newcn)
	return newcn, nil
}

func (p *connPool) Remove(cn *conn, reason error) error {
	p.storeLastErr(reason.Error())

	// Replace existing connection with new one and unblock waiter.
	newcn, err := p.replace(cn)
	if err != nil {
		return err
	}
	p.freeConns <- newcn
	return nil
}

// Len returns total number of connections.
func (p *connPool) Len() int {
	return p.conns.Len()
}

// FreeLen returns number of free connections.
func (p *connPool) FreeLen() int {
	return len(p.freeConns)
}

func (p *connPool) Stats() *PoolStats {
	stats := p.stats
	stats.Requests = atomic.LoadUint64(&p.stats.Requests)
	stats.Waits = atomic.LoadUint64(&p.stats.Waits)
	stats.Timeouts = atomic.LoadUint64(&p.stats.Timeouts)
	stats.TotalConns = uint64(p.Len())
	stats.FreeConns = uint64(p.FreeLen())
	return &stats
}

func (p *connPool) Close() (retErr error) {
	if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) {
		return errClosed
	}
	// Wait for app to free connections, but don't close them immediately.
	for i := 0; i < p.Len(); i++ {
		if cn := p.wait(); cn == nil {
			break
		}
	}
	// Close all connections.
	if err := p.conns.Close(); err != nil {
		retErr = err
	}
	return retErr
}

func (p *connPool) reaper() {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for _ = range ticker.C {
		if p.closed() {
			break
		}

		// pool.First removes idle connections from the pool and
		// returns first non-idle connection. So just put returned
		// connection back.
		if cn := p.First(); cn != nil {
			p.Put(cn)
		}
	}
}

func (p *connPool) storeLastErr(err string) {
	p.lastErr.Store(err)
}

func (p *connPool) loadLastErr() string {
	if v := p.lastErr.Load(); v != nil {
		return v.(string)
	}
	return ""
}

//------------------------------------------------------------------------------

type singleConnPool struct {
	cn *conn
}

func newSingleConnPool(cn *conn) *singleConnPool {
	return &singleConnPool{
		cn: cn,
	}
}

func (p *singleConnPool) First() *conn {
	return p.cn
}

func (p *singleConnPool) Get() (*conn, bool, error) {
	return p.cn, false, nil
}

func (p *singleConnPool) Put(cn *conn) error {
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *singleConnPool) Remove(cn *conn, _ error) error {
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *singleConnPool) Len() int {
	return 1
}

func (p *singleConnPool) FreeLen() int {
	return 0
}

func (p *singleConnPool) Stats() *PoolStats { return nil }

func (p *singleConnPool) Close() error {
	return nil
}

//------------------------------------------------------------------------------

type stickyConnPool struct {
	pool     pool
	reusable bool

	cn     *conn
	closed bool
	mx     sync.Mutex
}

func newStickyConnPool(pool pool, reusable bool) *stickyConnPool {
	return &stickyConnPool{
		pool:     pool,
		reusable: reusable,
	}
}

func (p *stickyConnPool) First() *conn {
	p.mx.Lock()
	cn := p.cn
	p.mx.Unlock()
	return cn
}

func (p *stickyConnPool) Get() (cn *conn, isNew bool, err error) {
	defer p.mx.Unlock()
	p.mx.Lock()

	if p.closed {
		err = errClosed
		return
	}
	if p.cn != nil {
		cn = p.cn
		return
	}

	cn, isNew, err = p.pool.Get()
	if err != nil {
		return
	}
	p.cn = cn
	return
}

func (p *stickyConnPool) put() (err error) {
	err = p.pool.Put(p.cn)
	p.cn = nil
	return err
}

func (p *stickyConnPool) Put(cn *conn) error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *stickyConnPool) remove(reason error) (err error) {
	err = p.pool.Remove(p.cn, reason)
	p.cn = nil
	return err
}

func (p *stickyConnPool) Remove(cn *conn, _ error) error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	if p.cn == nil {
		panic("p.cn == nil")
	}
	if cn != nil && p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *stickyConnPool) Len() int {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.cn == nil {
		return 0
	}
	return 1
}

func (p *stickyConnPool) FreeLen() int {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.cn == nil {
		return 1
	}
	return 0
}

func (p *stickyConnPool) Stats() *PoolStats { return nil }

func (p *stickyConnPool) Reset(reason error) (err error) {
	p.mx.Lock()
	if p.cn != nil {
		err = p.remove(reason)
	}
	p.mx.Unlock()
	return err
}

func (p *stickyConnPool) Close() error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	p.closed = true
	var err error
	if p.cn != nil {
		if p.reusable {
			err = p.put()
		} else {
			reason := errors.New("redis: sticky not reusable connection")
			err = p.remove(reason)
		}
	}
	return err
}
<commit_msg>Use uint32 because uint64 requires manual alignment on 386 arch. Fixes #256.<commit_after>package redis

import (
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"gopkg.in/bsm/ratelimit.v1"
)

var (
	errClosed      = errors.New("redis: client is closed")
	errPoolTimeout = errors.New("redis: connection pool timeout")
)

// PoolStats contains pool state information and accumulated stats.
type PoolStats struct {
	Requests uint32 // number of times a connection was requested by the pool
	Waits    uint32 // number of times our pool had to wait for a connection
	Timeouts uint32 // number of times a wait timeout occurred

	TotalConns uint32 // the number of total connections in the pool
	FreeConns  uint32 // the number of free connections in the pool
}

type pool interface {
	First() *conn
	Get() (*conn, bool, error)
	Put(*conn) error
	Remove(*conn, error) error
	Len() int
	FreeLen() int
	Close() error
	Stats() *PoolStats
}

type connList struct {
	cns  []*conn
	mx   sync.Mutex
	len  int32 // atomic
	size int32
}

func newConnList(size int) *connList {
	return &connList{
		cns:  make([]*conn, 0, size),
		size: int32(size),
	}
}

func (l *connList) Len() int {
	return int(atomic.LoadInt32(&l.len))
}

// Reserve reserves place in the list and returns true on success. The
// caller must add or remove connection if place was reserved.
func (l *connList) Reserve() bool {
	len := atomic.AddInt32(&l.len, 1)
	reserved := len <= l.size
	if !reserved {
		atomic.AddInt32(&l.len, -1)
	}
	return reserved
}

// Add adds connection to the list. The caller must reserve place first.
func (l *connList) Add(cn *conn) {
	l.mx.Lock()
	l.cns = append(l.cns, cn)
	l.mx.Unlock()
}

// Remove closes connection and removes it from the list.
func (l *connList) Remove(cn *conn) error {
	defer l.mx.Unlock()
	l.mx.Lock()

	if cn == nil {
		atomic.AddInt32(&l.len, -1)
		return nil
	}

	for i, c := range l.cns {
		if c == cn {
			l.cns = append(l.cns[:i], l.cns[i+1:]...)
			atomic.AddInt32(&l.len, -1)
			return cn.Close()
		}
	}

	if l.closed() {
		return nil
	}
	panic("conn not found in the list")
}

func (l *connList) Replace(cn, newcn *conn) error {
	defer l.mx.Unlock()
	l.mx.Lock()

	for i, c := range l.cns {
		if c == cn {
			l.cns[i] = newcn
			return cn.Close()
		}
	}

	if l.closed() {
		return newcn.Close()
	}
	panic("conn not found in the list")
}

func (l *connList) Close() (retErr error) {
	l.mx.Lock()
	for _, c := range l.cns {
		if err := c.Close(); err != nil {
			retErr = err
		}
	}
	l.cns = nil
	atomic.StoreInt32(&l.len, 0)
	l.mx.Unlock()
	return retErr
}

func (l *connList) closed() bool {
	return l.cns == nil
}

type connPool struct {
	dialer func() (*conn, error)

	rl        *ratelimit.RateLimiter
	opt       *Options
	conns     *connList
	freeConns chan *conn
	stats     PoolStats

	_closed int32

	lastErr atomic.Value
}

func newConnPool(opt *Options) *connPool {
	p := &connPool{
		dialer: newConnDialer(opt),

		rl:        ratelimit.New(3*opt.getPoolSize(), time.Second),
		opt:       opt,
		conns:     newConnList(opt.getPoolSize()),
		freeConns: make(chan *conn, opt.getPoolSize()),
	}
	if p.opt.getIdleTimeout() > 0 {
		go p.reaper()
	}
	return p
}

func (p *connPool) closed() bool {
	return atomic.LoadInt32(&p._closed) == 1
}

func (p *connPool) isIdle(cn *conn) bool {
	return p.opt.getIdleTimeout() > 0 && time.Since(cn.usedAt) > p.opt.getIdleTimeout()
}

// First returns first non-idle connection from the pool or nil if
// there are no connections.
func (p *connPool) First() *conn {
	for {
		select {
		case cn := <-p.freeConns:
			if p.isIdle(cn) {
				var err error
				cn, err = p.replace(cn)
				if err != nil {
					Logger.Printf("pool.replace failed: %s", err)
					continue
				}
			}
			return cn
		default:
			return nil
		}
	}
	panic("not reached")
}

// wait waits for free non-idle connection. It returns nil on timeout.
func (p *connPool) wait() *conn {
	deadline := time.After(p.opt.getPoolTimeout())
	for {
		select {
		case cn := <-p.freeConns:
			if p.isIdle(cn) {
				var err error
				cn, err = p.replace(cn)
				if err != nil {
					Logger.Printf("pool.replace failed: %s", err)
					continue
				}
			}
			return cn
		case <-deadline:
			return nil
		}
	}
	panic("not reached")
}

// Establish a new connection
func (p *connPool) new() (*conn, error) {
	if p.rl.Limit() {
		err := fmt.Errorf(
			"redis: you open connections too fast (last_error=%q)",
			p.loadLastErr(),
		)
		return nil, err
	}

	cn, err := p.dialer()
	if err != nil {
		p.storeLastErr(err.Error())
		return nil, err
	}

	return cn, nil
}

// Get returns existed connection from the pool or creates a new one.
func (p *connPool) Get() (cn *conn, isNew bool, err error) {
	if p.closed() {
		err = errClosed
		return
	}

	atomic.AddUint32(&p.stats.Requests, 1)

	// Fetch first non-idle connection, if available.
	if cn = p.First(); cn != nil {
		return
	}

	// Try to create a new one.
	if p.conns.Reserve() {
		cn, err = p.new()
		if err != nil {
			p.conns.Remove(nil)
			return
		}
		p.conns.Add(cn)
		isNew = true
		return
	}

	// Otherwise, wait for the available connection.
	atomic.AddUint32(&p.stats.Waits, 1)
	if cn = p.wait(); cn != nil {
		return
	}

	atomic.AddUint32(&p.stats.Timeouts, 1)
	err = errPoolTimeout
	return
}

func (p *connPool) Put(cn *conn) error {
	if cn.rd.Buffered() != 0 {
		b, _ := cn.rd.Peek(cn.rd.Buffered())
		err := fmt.Errorf("connection has unread data: %q", b)
		Logger.Print(err)
		return p.Remove(cn, err)
	}
	if p.opt.getIdleTimeout() > 0 {
		cn.usedAt = time.Now()
	}
	p.freeConns <- cn
	return nil
}

func (p *connPool) replace(cn *conn) (*conn, error) {
	newcn, err := p.new()
	if err != nil {
		_ = p.conns.Remove(cn)
		return nil, err
	}
	_ = p.conns.Replace(cn, newcn)
	return newcn, nil
}

func (p *connPool) Remove(cn *conn, reason error) error {
	p.storeLastErr(reason.Error())

	// Replace existing connection with new one and unblock waiter.
	newcn, err := p.replace(cn)
	if err != nil {
		return err
	}
	p.freeConns <- newcn
	return nil
}

// Len returns total number of connections.
func (p *connPool) Len() int {
	return p.conns.Len()
}

// FreeLen returns number of free connections.
func (p *connPool) FreeLen() int {
	return len(p.freeConns)
}

func (p *connPool) Stats() *PoolStats {
	stats := p.stats
	stats.Requests = atomic.LoadUint32(&p.stats.Requests)
	stats.Waits = atomic.LoadUint32(&p.stats.Waits)
	stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts)
	stats.TotalConns = uint32(p.Len())
	stats.FreeConns = uint32(p.FreeLen())
	return &stats
}

func (p *connPool) Close() (retErr error) {
	if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) {
		return errClosed
	}
	// Wait for app to free connections, but don't close them immediately.
	for i := 0; i < p.Len(); i++ {
		if cn := p.wait(); cn == nil {
			break
		}
	}
	// Close all connections.
	if err := p.conns.Close(); err != nil {
		retErr = err
	}
	return retErr
}

func (p *connPool) reaper() {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for _ = range ticker.C {
		if p.closed() {
			break
		}

		// pool.First removes idle connections from the pool and
		// returns first non-idle connection. So just put returned
		// connection back.
		if cn := p.First(); cn != nil {
			p.Put(cn)
		}
	}
}

func (p *connPool) storeLastErr(err string) {
	p.lastErr.Store(err)
}

func (p *connPool) loadLastErr() string {
	if v := p.lastErr.Load(); v != nil {
		return v.(string)
	}
	return ""
}

//------------------------------------------------------------------------------

type singleConnPool struct {
	cn *conn
}

func newSingleConnPool(cn *conn) *singleConnPool {
	return &singleConnPool{
		cn: cn,
	}
}

func (p *singleConnPool) First() *conn {
	return p.cn
}

func (p *singleConnPool) Get() (*conn, bool, error) {
	return p.cn, false, nil
}

func (p *singleConnPool) Put(cn *conn) error {
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *singleConnPool) Remove(cn *conn, _ error) error {
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *singleConnPool) Len() int {
	return 1
}

func (p *singleConnPool) FreeLen() int {
	return 0
}

func (p *singleConnPool) Stats() *PoolStats { return nil }

func (p *singleConnPool) Close() error {
	return nil
}

//------------------------------------------------------------------------------

type stickyConnPool struct {
	pool     pool
	reusable bool

	cn     *conn
	closed bool
	mx     sync.Mutex
}

func newStickyConnPool(pool pool, reusable bool) *stickyConnPool {
	return &stickyConnPool{
		pool:     pool,
		reusable: reusable,
	}
}

func (p *stickyConnPool) First() *conn {
	p.mx.Lock()
	cn := p.cn
	p.mx.Unlock()
	return cn
}

func (p *stickyConnPool) Get() (cn *conn, isNew bool, err error) {
	defer p.mx.Unlock()
	p.mx.Lock()

	if p.closed {
		err = errClosed
		return
	}
	if p.cn != nil {
		cn = p.cn
		return
	}

	cn, isNew, err = p.pool.Get()
	if err != nil {
		return
	}
	p.cn = cn
	return
}

func (p *stickyConnPool) put() (err error) {
	err = p.pool.Put(p.cn)
	p.cn = nil
	return err
}

func (p *stickyConnPool) Put(cn *conn) error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	if p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *stickyConnPool) remove(reason error) (err error) {
	err = p.pool.Remove(p.cn, reason)
	p.cn = nil
	return err
}

func (p *stickyConnPool) Remove(cn *conn, _ error) error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	if p.cn == nil {
		panic("p.cn == nil")
	}
	if cn != nil && p.cn != cn {
		panic("p.cn != cn")
	}
	return nil
}

func (p *stickyConnPool) Len() int {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.cn == nil {
		return 0
	}
	return 1
}

func (p *stickyConnPool) FreeLen() int {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.cn == nil {
		return 1
	}
	return 0
}

func (p *stickyConnPool) Stats() *PoolStats { return nil }

func (p *stickyConnPool) Reset(reason error) (err error) {
	p.mx.Lock()
	if p.cn != nil {
		err = p.remove(reason)
	}
	p.mx.Unlock()
	return err
}

func (p *stickyConnPool) Close() error {
	defer p.mx.Unlock()
	p.mx.Lock()
	if p.closed {
		return errClosed
	}
	p.closed = true
	var err error
	if p.cn != nil {
		if p.reusable {
			err = p.put()
		} else {
			reason := errors.New("redis: sticky not reusable connection")
			err = p.remove(reason)
		}
	}
	return err
}
<|endoftext|>


<commit_before>package spool

import (
	"bytes"
	"context"
	"crypto/sha256"
	"fmt"
	"strings"
	"time"

	"cloud.google.com/go/spanner"
	admin "cloud.google.com/go/spanner/admin/database/apiv1"
	"github.com/cloudspannerecosystem/spool/model"
	databasepb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
)

// State represents a state of the database.
type State int64

const (
	// StateIdle represents a idle state.
	StateIdle State = iota
	// StateBusy represents a busy state.
	StateBusy
)

// Int64 returns s as int64.
func (s State) Int64() int64 {
	return int64(s)
}

// String returns a string representing the state.
func (s State) String() string {
	switch s {
	case StateIdle:
		return "idle"
	case StateBusy:
		return "busy"
	}
	return "unknown"
}

// Pool represents a spanner database pool.
type Pool struct {
	client        *spanner.Client
	adminClient   *admin.DatabaseAdminClient
	conf          *Config
	ddlStatements []string
	checksum      string
}

// NewPool creates a new Pool.
func NewPool(ctx context.Context, conf *Config, ddl []byte) (*Pool, error) {
	client, err := spanner.NewClient(ctx, conf.Database(), conf.ClientOptions()...)
	if err != nil {
		return nil, err
	}
	adminClient, err := admin.NewDatabaseAdminClient(ctx, conf.ClientOptions()...)
	if err != nil {
		return nil, err
	}
	pool := &Pool{
		client:        client,
		adminClient:   adminClient,
		conf:          conf,
		ddlStatements: ddlToStatements(ddl),
		checksum:      checksum(ddl),
	}
	return pool, nil
}

func ddlToStatements(ddl []byte) []string {
	ddls := bytes.Split(ddl, []byte(";"))
	ddlStatements := make([]string, 0, len(ddls))
	for _, s := range ddls {
		if stmt := strings.TrimSpace(string(s)); stmt != "" {
			ddlStatements = append(ddlStatements, stmt)
		}
	}
	return ddlStatements
}

func checksum(ddl []byte) string {
	return fmt.Sprintf("%x", sha256.Sum256(ddl))
}

// Create creates a new database and adds to the pool.
func (p *Pool) Create(ctx context.Context, dbNamePrefix string) (*model.SpoolDatabase, error) {
	dbName := fmt.Sprintf("%s-%d", dbNamePrefix, time.Now().Unix())
	sdb := &model.SpoolDatabase{
		DatabaseName: dbName,
		Checksum:     p.checksum,
		State:        StateIdle.Int64(),
		CreatedAt:    spanner.CommitTimestamp,
		UpdatedAt:    spanner.CommitTimestamp,
	}
	return p.create(ctx, sdb)
}

func (p *Pool) create(ctx context.Context, sdb *model.SpoolDatabase) (*model.SpoolDatabase, error) {
	op, err := p.adminClient.CreateDatabase(ctx, &databasepb.CreateDatabaseRequest{
		Parent:          p.conf.Instance(),
		CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", sdb.DatabaseName),
		ExtraStatements: p.ddlStatements,
	})
	if err != nil {
		return nil, err
	}
	if _, err := op.Wait(ctx); err != nil {
		return nil, err
	}
	ts, err := p.client.Apply(ctx, []*spanner.Mutation{sdb.Insert(ctx)})
	if err != nil {
		_ = dropDatabase(ctx, p.conf.WithDatabaseID(sdb.DatabaseName))
		return nil, err
	}
	sdb.CreatedAt = ts
	sdb.UpdatedAt = ts
	return sdb, nil
}

// Get gets a idle database from the pool.
func (p *Pool) Get(ctx context.Context) (*model.SpoolDatabase, error) {
	var sdb *model.SpoolDatabase
	if _, err := p.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
		var err error
		sdb, err = model.FindSpoolDatabaseByChecksumState(ctx, txn, p.checksum, StateIdle.Int64())
		if err != nil {
			return err
		}
		sdb.ChangeState(StateBusy.Int64())
		if err := txn.BufferWrite([]*spanner.Mutation{sdb.Update(ctx)}); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return sdb, nil
}

// GetOrCreate gets a idle database or creates a new database.
func (p *Pool) GetOrCreate(ctx context.Context, dbNamePrefix string) (*model.SpoolDatabase, error) {
	sdb, err := p.Get(ctx)
	if err == nil {
		return sdb, nil
	}
	if !isErrNotFound(err) {
		return nil, err
	}
	dbName := fmt.Sprintf("%s-%d", dbNamePrefix, time.Now().Unix())
	sdb = &model.SpoolDatabase{
		DatabaseName: dbName,
		Checksum:     p.checksum,
		State:        StateBusy.Int64(),
		CreatedAt:    spanner.CommitTimestamp,
		UpdatedAt:    spanner.CommitTimestamp,
	}
	return p.create(ctx, sdb)
}

// List gets all databases from the pool.
func (p *Pool) List(ctx context.Context) ([]*model.SpoolDatabase, error) {
	return model.FindSpoolDatabasesByChecksum(ctx, p.client.ReadOnlyTransaction(), p.checksum)
}

// Put adds a database to the pool.
func (p *Pool) Put(ctx context.Context, dbName string) error {
	if _, err := p.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
		sdb, err := model.FindSpoolDatabase(ctx, txn, dbName)
		if err != nil {
			return err
		}
		sdb.ChangeState(StateIdle.Int64())
		if err := txn.BufferWrite([]*spanner.Mutation{sdb.Update(ctx)}); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}
	return nil
}

// Clean removes all idle databases.
func (p *Pool) Clean(ctx context.Context, filters ...func(sdb *model.SpoolDatabase) bool) error {
	return clean(ctx, p.client, p.conf, func(ctx context.Context, txn *spanner.ReadWriteTransaction) ([]*model.SpoolDatabase, error) {
		sdbs, err := model.FindSpoolDatabasesByChecksumState(ctx, txn, p.checksum, StateIdle.Int64())
		if err != nil {
			return nil, err
		}
		return filter(sdbs, filters...), nil
	})
}
<commit_msg>Nop-commit to trigger CircleCI rebuild.<commit_after>package spool

import (
	"bytes"
	"context"
	"crypto/sha256"
	"fmt"
	"strings"
	"time"

	"cloud.google.com/go/spanner"
	admin "cloud.google.com/go/spanner/admin/database/apiv1"
	"github.com/cloudspannerecosystem/spool/model"
	databasepb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
)

// State represents a state of the database.
type State int64

const (
	// StateIdle represents a idle state.
	StateIdle State = iota
	// StateBusy represents a busy state.
	StateBusy
)

// Int64 returns s as int64.
func (s State) Int64() int64 {
	return int64(s)
}

// String returns a string representing the state.
func (s State) String() string {
	switch s {
	case StateIdle:
		return "idle"
	case StateBusy:
		return "busy"
	}
	return "unknown"
}

// Pool represents a Spanner database pool.
type Pool struct {
	client        *spanner.Client
	adminClient   *admin.DatabaseAdminClient
	conf          *Config
	ddlStatements []string
	checksum      string
}

// NewPool creates a new Pool.
func NewPool(ctx context.Context, conf *Config, ddl []byte) (*Pool, error) {
	client, err := spanner.NewClient(ctx, conf.Database(), conf.ClientOptions()...)
	if err != nil {
		return nil, err
	}
	adminClient, err := admin.NewDatabaseAdminClient(ctx, conf.ClientOptions()...)
	if err != nil {
		return nil, err
	}
	pool := &Pool{
		client:        client,
		adminClient:   adminClient,
		conf:          conf,
		ddlStatements: ddlToStatements(ddl),
		checksum:      checksum(ddl),
	}
	return pool, nil
}

func ddlToStatements(ddl []byte) []string {
	ddls := bytes.Split(ddl, []byte(";"))
	ddlStatements := make([]string, 0, len(ddls))
	for _, s := range ddls {
		if stmt := strings.TrimSpace(string(s)); stmt != "" {
			ddlStatements = append(ddlStatements, stmt)
		}
	}
	return ddlStatements
}

func checksum(ddl []byte) string {
	return fmt.Sprintf("%x", sha256.Sum256(ddl))
}

// Create creates a new database and adds to the pool.
func (p *Pool) Create(ctx context.Context, dbNamePrefix string) (*model.SpoolDatabase, error) {
	dbName := fmt.Sprintf("%s-%d", dbNamePrefix, time.Now().Unix())
	sdb := &model.SpoolDatabase{
		DatabaseName: dbName,
		Checksum:     p.checksum,
		State:        StateIdle.Int64(),
		CreatedAt:    spanner.CommitTimestamp,
		UpdatedAt:    spanner.CommitTimestamp,
	}
	return p.create(ctx, sdb)
}

func (p *Pool) create(ctx context.Context, sdb *model.SpoolDatabase) (*model.SpoolDatabase, error) {
	op, err := p.adminClient.CreateDatabase(ctx, &databasepb.CreateDatabaseRequest{
		Parent:          p.conf.Instance(),
		CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", sdb.DatabaseName),
		ExtraStatements: p.ddlStatements,
	})
	if err != nil {
		return nil, err
	}
	if _, err := op.Wait(ctx); err != nil {
		return nil, err
	}
	ts, err := p.client.Apply(ctx, []*spanner.Mutation{sdb.Insert(ctx)})
	if err != nil {
		_ = dropDatabase(ctx, p.conf.WithDatabaseID(sdb.DatabaseName))
		return nil, err
	}
	sdb.CreatedAt = ts
	sdb.UpdatedAt = ts
	return sdb, nil
}

// Get gets a idle database from the pool.
func (p *Pool) Get(ctx context.Context) (*model.SpoolDatabase, error) {
	var sdb *model.SpoolDatabase
	if _, err := p.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
		var err error
		sdb, err = model.FindSpoolDatabaseByChecksumState(ctx, txn, p.checksum, StateIdle.Int64())
		if err != nil {
			return err
		}
		sdb.ChangeState(StateBusy.Int64())
		if err := txn.BufferWrite([]*spanner.Mutation{sdb.Update(ctx)}); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return sdb, nil
}

// GetOrCreate gets a idle database or creates a new database.
func (p *Pool) GetOrCreate(ctx context.Context, dbNamePrefix string) (*model.SpoolDatabase, error) {
	sdb, err := p.Get(ctx)
	if err == nil {
		return sdb, nil
	}
	if !isErrNotFound(err) {
		return nil, err
	}
	dbName := fmt.Sprintf("%s-%d", dbNamePrefix, time.Now().Unix())
	sdb = &model.SpoolDatabase{
		DatabaseName: dbName,
		Checksum:     p.checksum,
		State:        StateBusy.Int64(),
		CreatedAt:    spanner.CommitTimestamp,
		UpdatedAt:    spanner.CommitTimestamp,
	}
	return p.create(ctx, sdb)
}

// List gets all databases from the pool.
func (p *Pool) List(ctx context.Context) ([]*model.SpoolDatabase, error) {
	return model.FindSpoolDatabasesByChecksum(ctx, p.client.ReadOnlyTransaction(), p.checksum)
}

// Put adds a database to the pool.
func (p *Pool) Put(ctx context.Context, dbName string) error {
	if _, err := p.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
		sdb, err := model.FindSpoolDatabase(ctx, txn, dbName)
		if err != nil {
			return err
		}
		sdb.ChangeState(StateIdle.Int64())
		if err := txn.BufferWrite([]*spanner.Mutation{sdb.Update(ctx)}); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}
	return nil
}

// Clean removes all idle databases.
func (p *Pool) Clean(ctx context.Context, filters ...func(sdb *model.SpoolDatabase) bool) error {
	return clean(ctx, p.client, p.conf, func(ctx context.Context, txn *spanner.ReadWriteTransaction) ([]*model.SpoolDatabase, error) {
		sdbs, err := model.FindSpoolDatabasesByChecksumState(ctx, txn, p.checksum, StateIdle.Int64())
		if err != nil {
			return nil, err
		}
		return filter(sdbs, filters...), nil
	})
}
<|endoftext|>


<commit_before>// Package pop3 provides an implementation of the Post Office
// Protocol - Version 3.
package pop3

import (
	"bufio"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/mail"
	"strconv"
	"strings"
)

// Dial connects to the address on the named network.
func Dial(address string) (c *Client, err error) {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return
	}
	return NewClient(conn)
}

// DialTLS connects to the address on the named network using tls.
func DialTLS(address string) (c *Client, err error) {
	conn, err := tls.Dial("tcp", address, nil)
	if err != nil {
		return
	}
	return NewClient(conn)
}

// NewClient returns a new client object using an existing connection.
func NewClient(conn net.Conn) (c *Client, err error) {
	c = &Client{
		conn: conn,
		r:    bufio.NewReader(conn),
		w:    bufio.NewWriter(conn),
	}

	// Make sure we receive the server greeting
	line, err := c.ReadLine()
	if err != nil {
		return
	}
	if !IsOK(line) {
		return nil, errors.New("pop3: Server did not respond with +OK")
	}
	return
}

// ReadLine reads a single line from the buffer.
func (c *Client) ReadLine() (line string, err error) {
	b, _, err := c.r.ReadLine()
	if err == io.EOF {
		return
	}
	if err != nil {
		return
	}
	line = string(b)
	return
}

// ReadLines reads from the buffer until it hits the message end dot (".").
func (c *Client) ReadLines() (lines []string, err error) {
	for {
		line, err := c.ReadLine()
		if err != nil {
			return nil, err
		}

		// Look for a dot to indicate the end of a message
		// from the server.
		if line == "." {
			break
		}
		lines = append(lines, line)
	}
	return
}

// Send writes a command to the buffer and flushes it. Does not return any
// lines from the buffer.
func (c *Client) Send(format string, args ...interface{}) (err error) {
	if _, err = c.w.WriteString(fmt.Sprintf(format, args...)); err != nil {
		return
	}
	return c.w.Flush()
}

// Cmd sends a command to the server and returns a single line from the buffer.
func (c *Client) Cmd(format string,
	args ...interface{}) (line string, err error) {
	if err = c.Send(format, args...); err != nil {
		return
	}

	line, err = c.ReadLine()
	if err != nil {
		return
	}
	if !IsOK(line) {
		return "", errors.New("pop3: Server did not respond with +OK")
	}
	return
}

// User sends the username to the server.
func (c *Client) User(u string) (err error) {
	if _, err = c.Cmd("%s %s\r\n", USER, u); err != nil {
		return
	}
	return
}

// Pass sends the password to the server.
func (c *Client) Pass(p string) (err error) {
	if _, err = c.Cmd("%s %s\r\n", PASS, p); err != nil {
		return
	}
	return
}

// Quit sends the quit command to the server and closes the socket.
func (c *Client) Quit() (err error) {
	if err = c.Send("%s\r\n", QUIT); err != nil {
		return
	}
	return c.conn.Close()
}

// Auth sends the username and password to the server using the User and
// Pass methods. Noop is also called incase the server does not respond
// with invalid auth.
func (c *Client) Auth(u, p string) (err error) {
	if err = c.User(u); err != nil {
		return
	}

	if err = c.Pass(p); err != nil {
		return
	}

	// issue a dud command. Server might not respond with invalid auth
	// unless a cmd is issued
	return c.Noop()
}

// Stat retreives a listing for the current maildrop, consisting of the number
// of messages and the total size of the maildrop.
func (c *Client) Stat() (count, size int, err error) {
	line, err := c.Cmd("%s\r\n", STAT)
	if err != nil {
		return
	}

	// Number of messages in maildrop
	count, err = strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	if count == 0 {
		return
	}

	// Total size of messages in bytes
	size, err = strconv.Atoi(strings.Fields(line)[2])
	if err != nil {
		return
	}
	if size == 0 {
		return
	}
	return
}

// List returns the MessageList object which contains the message non unique
// id and its size.
func (c *Client) List(msg int) (list MessageList, err error) {
	line, err := c.Cmd("%s %s\r\n", LIST, msg)
	if err != nil {
		return
	}

	id, err := strconv.Atoi(strings.Fields(line)[0])
	if err != nil {
		return
	}

	size, err := strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	return MessageList{id, size}, nil
}

// ListAll returns a MessageList object which contains all messages in the
// maildrop.
func (c *Client) ListAll() (list []MessageList, err error) {
	if _, err = c.Cmd("%s\r\n", LIST); err != nil {
		return
	}

	lines, err := c.ReadLines()
	if err != nil {
		return
	}
	for _, v := range lines {
		id, err := strconv.Atoi(strings.Fields(v)[0])
		if err != nil {
			return nil, err
		}

		size, err := strconv.Atoi(strings.Fields(v)[1])
		if err != nil {
			return nil, err
		}
		list = append(list, MessageList{id, size})
	}
	return
}

// Retr downloads the given message and returns it as a mail.Message object.
func (c *Client) Retr(msg int) (m *mail.Message, err error) {
	if _, err = c.Cmd("%s %s\r\n", RETR, msg); err != nil {
		return
	}

	m, err = mail.ReadMessage(c.r)
	if err != nil {
		return
	}

	// mail.ReadMessage does not consume the message end dot in the buffer
	// so we must move the buffer along. Need to find a better way of
	// doing this.
	line, err := c.ReadLine()
	if err != nil {
		return
	}
	if line != "." {
		if err = c.r.UnreadByte(); err != nil {
			return
		}
	}
	return
}

// Dele will delete the given message from the maildrop.
// Changes will only take affect after the Quit command is issued.
func (c *Client) Dele(msg int) (err error) {
	if _, err = c.Cmd("%s %s\r\n", DELE, msg); err != nil {
		return
	}
	return
}

// Noop will do nothing however can prolong the end of a connection.
func (c *Client) Noop() (err error) {
	if _, err = c.Cmd("%s\r\n", NOOP); err != nil {
		return
	}
	return
}

// Rset will unmark any messages that have being marked for deletion in
// the current session.
func (c *Client) Rset() (err error) {
	if _, err = c.Cmd("%s\r\n", RSET); err != nil {
		return
	}
	return
}

// Top will return a varible number of lines for a given message as a
// mail.Message object.
func (c *Client) Top(msg int, n int) (m *mail.Message, err error) {
	if _, err = c.Cmd("%s %d %d\r\n", TOP, msg, n); err != nil {
		return
	}

	m, err = mail.ReadMessage(c.r)
	if err != nil {
		return
	}

	// mail.ReadMessage does not consume the message end dot in the buffer
	// so we must move the buffer along. Need to find a better way of
	// doing this.
	line, err := c.ReadLine()
	if err != nil {

		return
	}
	if line != "." {
		if err = c.r.UnreadByte(); err != nil {
			return
		}
	}
	return
}

// Uidl will return a MessageUidl object which contains the message non
// unique id and a unique id.
func (c *Client) Uidl(msg int) (list MessageUidl, err error) {
	line, err := c.Cmd("%s %s\r\n", UIDL, msg)
	if err != nil {
		return
	}

	id, err := strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	return MessageUidl{id, strings.Fields(line)[2]}, nil
}

// UidlAll will return a MessageUidl object which contains all messages in
// the maildrop.
func (c *Client) UidlAll() (list []MessageUidl, err error) {
	if _, err = c.Cmd("%s\r\n", UIDL); err != nil {
		return
	}

	lines, err := c.ReadLines()
	if err != nil {
		return
	}
	for _, v := range lines {
		id, err := strconv.Atoi(strings.Fields(v)[0])
		if err != nil {
			return nil, err
		}
		list = append(list, MessageUidl{id, strings.Fields(v)[1]})
	}
	return
}
<commit_msg>Fix wrong string fmt arg.<commit_after>// Package pop3 provides an implementation of the Post Office
// Protocol - Version 3.
package pop3

import (
	"bufio"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/mail"
	"strconv"
	"strings"
)

// Dial connects to the address on the named network.
func Dial(address string) (c *Client, err error) {
	conn, err := net.Dial("tcp", address)
	if err != nil {
		return
	}
	return NewClient(conn)
}

// DialTLS connects to the address on the named network using tls.
func DialTLS(address string) (c *Client, err error) {
	conn, err := tls.Dial("tcp", address, nil)
	if err != nil {
		return
	}
	return NewClient(conn)
}

// NewClient returns a new client object using an existing connection.
func NewClient(conn net.Conn) (c *Client, err error) {
	c = &Client{
		conn: conn,
		r:    bufio.NewReader(conn),
		w:    bufio.NewWriter(conn),
	}

	// Make sure we receive the server greeting
	line, err := c.ReadLine()
	if err != nil {
		return
	}
	if !IsOK(line) {
		return nil, errors.New("pop3: Server did not respond with +OK")
	}
	return
}

// ReadLine reads a single line from the buffer.
func (c *Client) ReadLine() (line string, err error) {
	b, _, err := c.r.ReadLine()
	if err == io.EOF {
		return
	}
	if err != nil {
		return
	}
	line = string(b)
	return
}

// ReadLines reads from the buffer until it hits the message end dot (".").
func (c *Client) ReadLines() (lines []string, err error) {
	for {
		line, err := c.ReadLine()
		if err != nil {
			return nil, err
		}

		// Look for a dot to indicate the end of a message
		// from the server.
		if line == "." {
			break
		}
		lines = append(lines, line)
	}
	return
}

// Send writes a command to the buffer and flushes it. Does not return any
// lines from the buffer.
func (c *Client) Send(format string, args ...interface{}) (err error) {
	if _, err = c.w.WriteString(fmt.Sprintf(format, args...)); err != nil {
		return
	}
	return c.w.Flush()
}

// Cmd sends a command to the server and returns a single line from the buffer.
func (c *Client) Cmd(format string,
	args ...interface{}) (line string, err error) {
	if err = c.Send(format, args...); err != nil {
		return
	}

	line, err = c.ReadLine()
	if err != nil {
		return
	}
	if !IsOK(line) {
		return "", errors.New("pop3: Server did not respond with +OK")
	}
	return
}

// User sends the username to the server.
func (c *Client) User(u string) (err error) {
	if _, err = c.Cmd("%s %s\r\n", USER, u); err != nil {
		return
	}
	return
}

// Pass sends the password to the server.
func (c *Client) Pass(p string) (err error) {
	if _, err = c.Cmd("%s %s\r\n", PASS, p); err != nil {
		return
	}
	return
}

// Quit sends the quit command to the server and closes the socket.
func (c *Client) Quit() (err error) {
	if err = c.Send("%s\r\n", QUIT); err != nil {
		return
	}
	return c.conn.Close()
}

// Auth sends the username and password to the server using the User and
// Pass methods. Noop is also called incase the server does not respond
// with invalid auth.
func (c *Client) Auth(u, p string) (err error) {
	if err = c.User(u); err != nil {
		return
	}

	if err = c.Pass(p); err != nil {
		return
	}

	// issue a dud command. Server might not respond with invalid auth
	// unless a cmd is issued
	return c.Noop()
}

// Stat retreives a listing for the current maildrop, consisting of the number
// of messages and the total size of the maildrop.
func (c *Client) Stat() (count, size int, err error) {
	line, err := c.Cmd("%s\r\n", STAT)
	if err != nil {
		return
	}

	// Number of messages in maildrop
	count, err = strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	if count == 0 {
		return
	}

	// Total size of messages in bytes
	size, err = strconv.Atoi(strings.Fields(line)[2])
	if err != nil {
		return
	}
	if size == 0 {
		return
	}
	return
}

// List returns the MessageList object which contains the message non unique
// id and its size.
func (c *Client) List(msg int) (list MessageList, err error) {
	line, err := c.Cmd("%s %d\r\n", LIST, msg)
	if err != nil {
		return
	}

	id, err := strconv.Atoi(strings.Fields(line)[0])
	if err != nil {
		return
	}

	size, err := strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	return MessageList{id, size}, nil
}

// ListAll returns a MessageList object which contains all messages in the
// maildrop.
func (c *Client) ListAll() (list []MessageList, err error) {
	if _, err = c.Cmd("%s\r\n", LIST); err != nil {
		return
	}

	lines, err := c.ReadLines()
	if err != nil {
		return
	}
	for _, v := range lines {
		id, err := strconv.Atoi(strings.Fields(v)[0])
		if err != nil {
			return nil, err
		}

		size, err := strconv.Atoi(strings.Fields(v)[1])
		if err != nil {
			return nil, err
		}
		list = append(list, MessageList{id, size})
	}
	return
}

// Retr downloads the given message and returns it as a mail.Message object.
func (c *Client) Retr(msg int) (m *mail.Message, err error) {
	if _, err = c.Cmd("%s %d\r\n", RETR, msg); err != nil {
		return
	}

	m, err = mail.ReadMessage(c.r)
	if err != nil {
		return
	}

	// mail.ReadMessage does not consume the message end dot in the buffer
	// so we must move the buffer along. Need to find a better way of
	// doing this.
	line, err := c.ReadLine()
	if err != nil {
		return
	}
	if line != "." {
		if err = c.r.UnreadByte(); err != nil {
			return
		}
	}
	return
}

// Dele will delete the given message from the maildrop.
// Changes will only take affect after the Quit command is issued.
func (c *Client) Dele(msg int) (err error) {
	if _, err = c.Cmd("%s %d\r\n", DELE, msg); err != nil {
		return
	}
	return
}

// Noop will do nothing however can prolong the end of a connection.
func (c *Client) Noop() (err error) {
	if _, err = c.Cmd("%s\r\n", NOOP); err != nil {
		return
	}
	return
}

// Rset will unmark any messages that have being marked for deletion in
// the current session.
func (c *Client) Rset() (err error) {
	if _, err = c.Cmd("%s\r\n", RSET); err != nil {
		return
	}
	return
}

// Top will return a varible number of lines for a given message as a
// mail.Message object.
func (c *Client) Top(msg int, n int) (m *mail.Message, err error) {
	if _, err = c.Cmd("%s %d %d\r\n", TOP, msg, n); err != nil {
		return
	}

	m, err = mail.ReadMessage(c.r)
	if err != nil {
		return
	}

	// mail.ReadMessage does not consume the message end dot in the buffer
	// so we must move the buffer along. Need to find a better way of
	// doing this.
	line, err := c.ReadLine()
	if err != nil {

		return
	}
	if line != "." {
		if err = c.r.UnreadByte(); err != nil {
			return
		}
	}
	return
}

// Uidl will return a MessageUidl object which contains the message non
// unique id and a unique id.
func (c *Client) Uidl(msg int) (list MessageUidl, err error) {
	line, err := c.Cmd("%s %d\r\n", UIDL, msg)
	if err != nil {
		return
	}

	id, err := strconv.Atoi(strings.Fields(line)[1])
	if err != nil {
		return
	}
	return MessageUidl{id, strings.Fields(line)[2]}, nil
}

// UidlAll will return a MessageUidl object which contains all messages in
// the maildrop.
func (c *Client) UidlAll() (list []MessageUidl, err error) {
	if _, err = c.Cmd("%s\r\n", UIDL); err != nil {
		return
	}

	lines, err := c.ReadLines()
	if err != nil {
		return
	}
	for _, v := range lines {
		id, err := strconv.Atoi(strings.Fields(v)[0])
		if err != nil {
			return nil, err
		}
		list = append(list, MessageUidl{id, strings.Fields(v)[1]})
	}
	return
}
<|endoftext|>