<commit_before>// Sync files and directories to and from local and remote object stores
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/spf13/pflag"

	"github.com/ncw/rclone/fs"
	// Active file systems
	_ "github.com/ncw/rclone/amazonclouddrive"
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/dropbox"
	_ "github.com/ncw/rclone/googlecloudstorage"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	// Flags
	cpuprofile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
	version       = pflag.BoolP("version", "V", false, "Print the version number")
	logFile       = pflag.StringP("log-file", "", "", "Log everything to this file")
	retries       = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
)

type Command struct {
	Name     string
	Help     string
	ArgsHelp string
	Run      func(fdst, fsrc fs.Fs) error
	MinArgs  int
	MaxArgs  int
	NoStats  bool
	Retry    bool
}

// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
	if len(args) < cmd.MinArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
		os.Exit(1)
	} else if len(args) > cmd.MaxArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
		os.Exit(1)
	}
}

var Commands = []Command{
	{
		Name:     "copy",
		ArgsHelp: "source:path dest:path",
		Help: `
        Copy the source to the destination.  Doesn't transfer
        unchanged files, testing by size and modification time or
        MD5SUM.  Doesn't delete files from the destination.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.CopyDir(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "sync",
		ArgsHelp: "source:path dest:path",
		Help: `
        Sync the source to the destination, changing the destination
        only.  Doesn't transfer unchanged files, testing by size and
        modification time or MD5SUM.  Destination is updated to match
        source, including deleting files if necessary.  Since this can
        cause data loss, test first with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Sync(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "move",
		ArgsHelp: "source:path dest:path",
		Help: `
        Moves the source to the destination.  This is equivalent to a
        copy followed by a purge, but may use server side operations
        to speed it up. Since this can cause data loss, test first
        with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.MoveDir(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "ls",
		ArgsHelp: "[remote:path]",
		Help: `
        List all the objects in the the path with size and path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.List(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsd",
		ArgsHelp: "[remote:path]",
		Help: `
        List all directories/containers/buckets in the the path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.ListDir(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsl",
		ArgsHelp: "[remote:path]",
		Help: `
        List all the objects in the the path with modification time,
        size and path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.ListLong(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "md5sum",
		ArgsHelp: "[remote:path]",
		Help: `
        Produces an md5sum file for all the objects in the path.  This
        is in the same format as the standard md5sum tool produces.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Md5sum(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "mkdir",
		ArgsHelp: "remote:path",
		Help: `
        Make the path if it doesn't already exist`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Mkdir(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "rmdir",
		ArgsHelp: "remote:path",
		Help: `
        Remove the path.  Note that you can't remove a path with
        objects in it, use purge for that.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Rmdir(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "purge",
		ArgsHelp: "remote:path",
		Help: `
        Remove the path and all of its contents.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Purge(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "check",
		ArgsHelp: "source:path dest:path",
		Help: `
        Checks the files in the source and destination match.  It
        compares sizes and MD5SUMs and prints a report of files which
        don't match.  It doesn't alter the source or destination.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Check(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name: "config",
		Help: `
        Enter an interactive configuration session.`,
		Run: func(fdst, fsrc fs.Fs) error {
			fs.EditConfig()
			return nil
		},
		NoStats: true,
	},
	{
		Name: "help",
		Help: `
        This help.`,
		NoStats: true,
	},
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directories to and from local and remote object stores - %s.

Syntax: [options] subcommand <parameters> <parameters...>

Subcommands:

`, fs.Version)
	for i := range Commands {
		cmd := &Commands[i]
		fmt.Fprintf(os.Stderr, "    %s %s\n", cmd.Name, cmd.ArgsHelp)
		fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
	}

	fmt.Fprintf(os.Stderr, "Options:\n")
	pflag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
It is only necessary to use a unique prefix of the subcommand, eg 'up'
for 'upload'.
`)
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// Parse the command line flags
func ParseFlags() {
	pflag.Usage = syntaxError
	pflag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	fs.LoadConfig()

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
}

// Parse the command from the command line
func ParseCommand() (*Command, []string) {
	args := pflag.Args()
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	cmd := strings.ToLower(args[0])
	args = args[1:]

	// Find the command doing a prefix match
	var command *Command
	for i := range Commands {
		trialCommand := &Commands[i]
		// exact command name found - use that
		if trialCommand.Name == cmd {
			command = trialCommand
			break
		} else if strings.HasPrefix(trialCommand.Name, cmd) {
			if command != nil {
				fs.Stats.Error()
				log.Fatalf("Not unique - matches multiple commands %q", cmd)
			}
			command = trialCommand
		}
	}
	if command == nil {
		fs.Stats.Error()
		log.Fatalf("Unknown command %q", cmd)
	}
	if command.Run == nil {
		syntaxError()
	}
	command.checkArgs(args)
	return command, args
}

// Create a Fs from a name
func NewFs(remote string) fs.Fs {
	f, err := fs.NewFs(remote)
	if err != nil {
		fs.Stats.Error()
		log.Fatalf("Failed to create file system for %q: %v", remote, err)
	}
	return f
}

// Print the stats every statsInterval
func StartStats() {
	if *statsInterval <= 0 {
		return
	}
	go func() {
		ch := time.Tick(*statsInterval)
		for {
			<-ch
			fs.Stats.Log()
		}
	}()
}

func main() {
	ParseFlags()
	if *version {
		fmt.Printf("rclone %s\n", fs.Version)
		os.Exit(0)
	}
	command, args := ParseCommand()

	// Log file output
	if *logFile != "" {
		f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
		if err != nil {
			log.Fatalf("Failed to open log file: %v", err)
		}
		f.Seek(0, os.SEEK_END)
		log.SetOutput(f)
		redirectStderr(f)
	}

	// Make source and destination fs
	var fdst, fsrc fs.Fs
	if len(args) >= 1 {
		fdst = NewFs(args[0])
	}
	if len(args) >= 2 {
		fsrc = fdst
		fdst = NewFs(args[1])
	}

	fs.CalculateModifyWindow(fdst, fsrc)

	if !command.NoStats {
		StartStats()
	}

	// Run the actual command
	if command.Run != nil {
		var err error
		for try := 1; try <= *retries; try++ {
			err = command.Run(fdst, fsrc)
			if !command.Retry || (err == nil && !fs.Stats.Errored()) {
				break
			}
			if err != nil {
				fs.Log(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, fs.Stats.GetErrors(), err)
			} else {
				fs.Log(nil, "Attempt %d/%d failed with %d errors", try, *retries, fs.Stats.GetErrors())
			}
			if try < *retries {
				fs.Stats.ResetErrors()
			}
		}
		if err != nil {
			log.Fatalf("Failed to %s: %v", command.Name, err)
		}
		if !command.NoStats && (!fs.Config.Quiet || fs.Stats.Errored() || *statsInterval > 0) {
			fmt.Fprintln(os.Stderr, fs.Stats)
		}
		if fs.Config.Verbose {
			fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
		}
		if fs.Stats.Errored() {
			os.Exit(1)
		}
		os.Exit(0)
	} else {
	}

}
<commit_msg>List multiple matched commands.<commit_after>// Sync files and directories to and from local and remote object stores
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/spf13/pflag"

	"github.com/ncw/rclone/fs"
	// Active file systems
	_ "github.com/ncw/rclone/amazonclouddrive"
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/dropbox"
	_ "github.com/ncw/rclone/googlecloudstorage"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	// Flags
	cpuprofile    = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
	statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
	version       = pflag.BoolP("version", "V", false, "Print the version number")
	logFile       = pflag.StringP("log-file", "", "", "Log everything to this file")
	retries       = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
)

type Command struct {
	Name     string
	Help     string
	ArgsHelp string
	Run      func(fdst, fsrc fs.Fs) error
	MinArgs  int
	MaxArgs  int
	NoStats  bool
	Retry    bool
}

// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
	if len(args) < cmd.MinArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
		os.Exit(1)
	} else if len(args) > cmd.MaxArgs {
		syntaxError()
		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
		os.Exit(1)
	}
}

var Commands = []Command{
	{
		Name:     "copy",
		ArgsHelp: "source:path dest:path",
		Help: `
        Copy the source to the destination.  Doesn't transfer
        unchanged files, testing by size and modification time or
        MD5SUM.  Doesn't delete files from the destination.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.CopyDir(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "sync",
		ArgsHelp: "source:path dest:path",
		Help: `
        Sync the source to the destination, changing the destination
        only.  Doesn't transfer unchanged files, testing by size and
        modification time or MD5SUM.  Destination is updated to match
        source, including deleting files if necessary.  Since this can
        cause data loss, test first with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Sync(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "move",
		ArgsHelp: "source:path dest:path",
		Help: `
        Moves the source to the destination.  This is equivalent to a
        copy followed by a purge, but may use server side operations
        to speed it up. Since this can cause data loss, test first
        with the --dry-run flag.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.MoveDir(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
		Retry:   true,
	},
	{
		Name:     "ls",
		ArgsHelp: "[remote:path]",
		Help: `
        List all the objects in the the path with size and path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.List(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsd",
		ArgsHelp: "[remote:path]",
		Help: `
        List all directories/containers/buckets in the the path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.ListDir(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "lsl",
		ArgsHelp: "[remote:path]",
		Help: `
        List all the objects in the the path with modification time,
        size and path.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.ListLong(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "md5sum",
		ArgsHelp: "[remote:path]",
		Help: `
        Produces an md5sum file for all the objects in the path.  This
        is in the same format as the standard md5sum tool produces.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Md5sum(fdst, os.Stdout)
		},
		MinArgs: 1,
		MaxArgs: 1,
	},
	{
		Name:     "mkdir",
		ArgsHelp: "remote:path",
		Help: `
        Make the path if it doesn't already exist`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Mkdir(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "rmdir",
		ArgsHelp: "remote:path",
		Help: `
        Remove the path.  Note that you can't remove a path with
        objects in it, use purge for that.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Rmdir(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "purge",
		ArgsHelp: "remote:path",
		Help: `
        Remove the path and all of its contents.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Purge(fdst)
		},
		MinArgs: 1,
		MaxArgs: 1,
		Retry:   true,
	},
	{
		Name:     "check",
		ArgsHelp: "source:path dest:path",
		Help: `
        Checks the files in the source and destination match.  It
        compares sizes and MD5SUMs and prints a report of files which
        don't match.  It doesn't alter the source or destination.`,
		Run: func(fdst, fsrc fs.Fs) error {
			return fs.Check(fdst, fsrc)
		},
		MinArgs: 2,
		MaxArgs: 2,
	},
	{
		Name: "config",
		Help: `
        Enter an interactive configuration session.`,
		Run: func(fdst, fsrc fs.Fs) error {
			fs.EditConfig()
			return nil
		},
		NoStats: true,
	},
	{
		Name: "help",
		Help: `
        This help.`,
		NoStats: true,
	},
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directories to and from local and remote object stores - %s.

Syntax: [options] subcommand <parameters> <parameters...>

Subcommands:

`, fs.Version)
	for i := range Commands {
		cmd := &Commands[i]
		fmt.Fprintf(os.Stderr, "    %s %s\n", cmd.Name, cmd.ArgsHelp)
		fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
	}

	fmt.Fprintf(os.Stderr, "Options:\n")
	pflag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
It is only necessary to use a unique prefix of the subcommand, eg 'mo'
for 'move'.
`)
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// Parse the command line flags
func ParseFlags() {
	pflag.Usage = syntaxError
	pflag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	fs.LoadConfig()

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			fs.Stats.Error()
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}
}

// Parse the command from the command line
func ParseCommand() (*Command, []string) {
	args := pflag.Args()
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	cmd := strings.ToLower(args[0])
	args = args[1:]

	// Find the command doing a prefix match
	var found = make([]*Command, 0, 1)
	var command *Command
	for i := range Commands {
		trialCommand := &Commands[i]
		// exact command name found - use that
		if trialCommand.Name == cmd {
			command = trialCommand
			break
		} else if strings.HasPrefix(trialCommand.Name, cmd) {
			found = append(found, trialCommand)
		}
	}
	if command == nil {
		switch len(found) {
		case 0:
			fs.Stats.Error()
			log.Fatalf("Unknown command %q", cmd)
		case 1:
			command = found[0]
		default:
			fs.Stats.Error()
			var names []string
			for _, cmd := range found {
				names = append(names, `"`+cmd.Name+`"`)
			}
			log.Fatalf("Not unique - matches multiple commands: %s", strings.Join(names, ", "))
		}
	}
	if command.Run == nil {
		syntaxError()
	}
	command.checkArgs(args)
	return command, args
}

// Create a Fs from a name
func NewFs(remote string) fs.Fs {
	f, err := fs.NewFs(remote)
	if err != nil {
		fs.Stats.Error()
		log.Fatalf("Failed to create file system for %q: %v", remote, err)
	}
	return f
}

// Print the stats every statsInterval
func StartStats() {
	if *statsInterval <= 0 {
		return
	}
	go func() {
		ch := time.Tick(*statsInterval)
		for {
			<-ch
			fs.Stats.Log()
		}
	}()
}

func main() {
	ParseFlags()
	if *version {
		fmt.Printf("rclone %s\n", fs.Version)
		os.Exit(0)
	}
	command, args := ParseCommand()

	// Log file output
	if *logFile != "" {
		f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
		if err != nil {
			log.Fatalf("Failed to open log file: %v", err)
		}
		f.Seek(0, os.SEEK_END)
		log.SetOutput(f)
		redirectStderr(f)
	}

	// Make source and destination fs
	var fdst, fsrc fs.Fs
	if len(args) >= 1 {
		fdst = NewFs(args[0])
	}
	if len(args) >= 2 {
		fsrc = fdst
		fdst = NewFs(args[1])
	}

	fs.CalculateModifyWindow(fdst, fsrc)

	if !command.NoStats {
		StartStats()
	}

	// Run the actual command
	if command.Run != nil {
		var err error
		for try := 1; try <= *retries; try++ {
			err = command.Run(fdst, fsrc)
			if !command.Retry || (err == nil && !fs.Stats.Errored()) {
				break
			}
			if err != nil {
				fs.Log(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, fs.Stats.GetErrors(), err)
			} else {
				fs.Log(nil, "Attempt %d/%d failed with %d errors", try, *retries, fs.Stats.GetErrors())
			}
			if try < *retries {
				fs.Stats.ResetErrors()
			}
		}
		if err != nil {
			log.Fatalf("Failed to %s: %v", command.Name, err)
		}
		if !command.NoStats && (!fs.Config.Quiet || fs.Stats.Errored() || *statsInterval > 0) {
			fmt.Fprintln(os.Stderr, fs.Stats)
		}
		if fs.Config.Verbose {
			fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
		}
		if fs.Stats.Errored() {
			os.Exit(1)
		}
		os.Exit(0)
	} else {
	}

}
<|endoftext|>


<commit_before>package torrent

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"

	"github.com/anacrolix/log"
	"github.com/anacrolix/missinggo/v2"
)

// Accesses Torrent data via a Client. Reads block until the data is available. Seeks and readahead
// also drive Client behaviour.
type Reader interface {
	io.ReadSeekCloser
	missinggo.ReadContexter
	// Configure the number of bytes ahead of a read that should also be prioritized in preparation
	// for further reads.
	SetReadahead(int64)
	// Don't wait for pieces to complete and be verified. Read calls return as soon as they can when
	// the underlying chunks become available.
	SetResponsive()
}

// Piece range by piece index, [begin, end).
type pieceRange struct {
	begin, end pieceIndex
}

type reader struct {
	t          *Torrent
	responsive bool
	// Adjust the read/seek window to handle Readers locked to File extents and the like.
	offset, length int64

	// Required when modifying pos and readahead, or reading them without opMu.
	mu  sync.Locker
	pos int64
	// Reads have been initiated since the last seek. This is used to prevent readahead occuring
	// after a seek or with a new reader at the starting position.
	reading   bool
	readahead int64
	// Function to dynamically calculate readahead. If nil, readahead is static.
	readaheadFunc func() int64
	// Position that reads have continued contiguously from.
	contiguousReadStartPos int64
	// The cached piece range this reader wants downloaded. The zero value corresponds to nothing.
	// We cache this so that changes can be detected, and bubbled up to the Torrent only as
	// required.
	pieces pieceRange
}

var _ io.ReadSeekCloser = (*reader)(nil)

func (r *reader) SetResponsive() {
	r.responsive = true
	r.t.cl.event.Broadcast()
}

// Disable responsive mode. TODO: Remove?
func (r *reader) SetNonResponsive() {
	r.responsive = false
	r.t.cl.event.Broadcast()
}

func (r *reader) SetReadahead(readahead int64) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.readahead = readahead
	r.readaheadFunc = nil
	r.posChanged()
}

// How many bytes are available to read. Max is the most we could require.
func (r *reader) available(off, max int64) (ret int64) {
	off += r.offset
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.responsive && !r.t.pieceComplete(pieceIndex(req.Index)) {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

// Calculates the pieces this reader wants downloaded, ignoring the cached value at r.pieces.
func (r *reader) piecesUncached() (ret pieceRange) {
	ra := r.readahead
	if r.readaheadFunc != nil {
		ra = r.readaheadFunc()
	}
	if ra < 1 {
		// Needs to be at least 1, because [x, x) means we don't want
		// anything.
		ra = 1
	}
	if !r.reading {
		ra = 0
	}
	if ra > r.length-r.pos {
		ra = r.length - r.pos
	}
	ret.begin, ret.end = r.t.byteRegionPieces(r.torrentOffset(r.pos), ra)
	return
}

func (r *reader) Read(b []byte) (n int, err error) {
	return r.ReadContext(context.Background(), b)
}

func (r *reader) ReadContext(ctx context.Context, b []byte) (n int, err error) {
	if len(b) > 0 {
		r.reading = true
		// TODO: Rework reader piece priorities so we don't have to push updates in to the Client
		// and take the lock here.
		r.mu.Lock()
		r.posChanged()
		r.mu.Unlock()
	}
	n, err = r.readOnceAt(ctx, b, r.pos)
	if n == 0 {
		if err == nil && len(b) > 0 {
			panic("expected error")
		} else {
			return
		}
	}

	r.mu.Lock()
	r.pos += int64(n)
	r.posChanged()
	r.mu.Unlock()
	if r.pos >= r.length {
		err = io.EOF
	} else if err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	return
}

var closedChan = make(chan struct{})

func init() {
	close(closedChan)
}

// Wait until some data should be available to read. Tickles the client if it isn't. Returns how
// much should be readable without blocking.
func (r *reader) waitAvailable(ctx context.Context, pos, wanted int64, wait bool) (avail int64, err error) {
	t := r.t
	for {
		r.t.cl.rLock()
		avail = r.available(pos, wanted)
		readerCond := t.piece(int((r.offset + pos) / t.info.PieceLength)).readerCond.Signaled()
		r.t.cl.rUnlock()
		if avail != 0 {
			return
		}
		var dontWait <-chan struct{}
		if !wait || wanted == 0 {
			dontWait = closedChan
		}
		select {
		case <-r.t.closed.Done():
			err = errors.New("torrent closed")
			return
		case <-ctx.Done():
			err = ctx.Err()
			return
		case <-r.t.dataDownloadDisallowed.On():
			err = errors.New("torrent data downloading disabled")
		case <-r.t.networkingEnabled.Off():
			err = errors.New("torrent networking disabled")
			return
		case <-dontWait:
			return
		case <-readerCond:
		}
	}
}

// Adds the reader's torrent offset to the reader object offset (for example the reader might be
// constrainted to a particular file within the torrent).
func (r *reader) torrentOffset(readerPos int64) int64 {
	return r.offset + readerPos
}

// Performs at most one successful read to torrent storage.
func (r *reader) readOnceAt(ctx context.Context, b []byte, pos int64) (n int, err error) {
	if pos >= r.length {
		err = io.EOF
		return
	}
	for {
		var avail int64
		avail, err = r.waitAvailable(ctx, pos, int64(len(b)), n == 0)
		if avail == 0 {
			return
		}
		firstPieceIndex := pieceIndex(r.torrentOffset(pos) / r.t.info.PieceLength)
		firstPieceOffset := r.torrentOffset(pos) % r.t.info.PieceLength
		b1 := missinggo.LimitLen(b, avail)
		n, err = r.t.readAt(b1, r.torrentOffset(pos))
		if n != 0 {
			err = nil
			return
		}
		r.t.cl.lock()
		// TODO: Just reset pieces in the readahead window. This might help
		// prevent thrashing with small caches and file and piece priorities.
		r.log(log.Fstr("error reading torrent %s piece %d offset %d, %d bytes: %v",
			r.t.infoHash.HexString(), firstPieceIndex, firstPieceOffset, len(b1), err))
		if !r.t.updatePieceCompletion(firstPieceIndex) {
			r.log(log.Fstr("piece %d completion unchanged", firstPieceIndex))
		}
		// Update the rest of the piece completions in the readahead window, without alerting to
		// changes (since only the first piece, the one above, could have generated the read error
		// we're currently handling).
		if r.pieces.begin != firstPieceIndex {
			panic(fmt.Sprint(r.pieces.begin, firstPieceIndex))
		}
		for index := r.pieces.begin + 1; index < r.pieces.end; index++ {
			r.t.updatePieceCompletion(index)
		}
		r.t.cl.unlock()
	}
}

// Hodor
func (r *reader) Close() error {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.t.deleteReader(r)
	return nil
}

func (r *reader) posChanged() {
	to := r.piecesUncached()
	from := r.pieces
	if to == from {
		return
	}
	r.pieces = to
	// log.Printf("reader pos changed %v->%v", from, to)
	r.t.readerPosChanged(from, to)
}

func (r *reader) Seek(off int64, whence int) (newPos int64, err error) {
	r.mu.Lock()
	defer r.mu.Unlock()
	newPos, err = func() (int64, error) {
		switch whence {
		case io.SeekStart:
			return off, err
		case io.SeekCurrent:
			return r.pos + off, nil
		case io.SeekEnd:
			return r.length + off, nil
		default:
			return r.pos, errors.New("bad whence")
		}
	}()
	if err != nil {
		return
	}
	if newPos == r.pos {
		return
	}
	r.reading = false
	r.pos = newPos
	r.contiguousReadStartPos = newPos

	r.posChanged()
	return
}

func (r *reader) log(m log.Msg) {
	r.t.logger.Log(m.Skip(1))
}

// Implementation inspired by https://news.ycombinator.com/item?id=27019613.
func (r *reader) defaultReadaheadFunc() int64 {
	return r.pos - r.contiguousReadStartPos
}
<commit_msg>Reorder fields to reduce padding in reader (#597)<commit_after>package torrent

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"

	"github.com/anacrolix/log"
	"github.com/anacrolix/missinggo/v2"
)

// Accesses Torrent data via a Client. Reads block until the data is available. Seeks and readahead
// also drive Client behaviour.
type Reader interface {
	io.ReadSeekCloser
	missinggo.ReadContexter
	// Configure the number of bytes ahead of a read that should also be prioritized in preparation
	// for further reads.
	SetReadahead(int64)
	// Don't wait for pieces to complete and be verified. Read calls return as soon as they can when
	// the underlying chunks become available.
	SetResponsive()
}

// Piece range by piece index, [begin, end).
type pieceRange struct {
	begin, end pieceIndex
}

type reader struct {
	t   *Torrent
	// Adjust the read/seek window to handle Readers locked to File extents and the like.
	offset, length int64

	// Function to dynamically calculate readahead. If nil, readahead is static.
	readaheadFunc func() int64

	// Required when modifying pos and readahead.
	mu  sync.Locker

	readahead, pos int64
	// Position that reads have continued contiguously from.
	contiguousReadStartPos int64
	// The cached piece range this reader wants downloaded. The zero value corresponds to nothing.
	// We cache this so that changes can be detected, and bubbled up to the Torrent only as
	// required.
	pieces pieceRange

	// Reads have been initiated since the last seek. This is used to prevent readaheads occurring
	// after a seek or with a new reader at the starting position.
	reading    bool
	responsive bool

}

var _ io.ReadSeekCloser = (*reader)(nil)

func (r *reader) SetResponsive() {
	r.responsive = true
	r.t.cl.event.Broadcast()
}

// Disable responsive mode. TODO: Remove?
func (r *reader) SetNonResponsive() {
	r.responsive = false
	r.t.cl.event.Broadcast()
}

func (r *reader) SetReadahead(readahead int64) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.readahead = readahead
	r.readaheadFunc = nil
	r.posChanged()
}

// How many bytes are available to read. Max is the most we could require.
func (r *reader) available(off, max int64) (ret int64) {
	off += r.offset
	for max > 0 {
		req, ok := r.t.offsetRequest(off)
		if !ok {
			break
		}
		if !r.responsive && !r.t.pieceComplete(pieceIndex(req.Index)) {
			break
		}
		if !r.t.haveChunk(req) {
			break
		}
		len1 := int64(req.Length) - (off - r.t.requestOffset(req))
		max -= len1
		ret += len1
		off += len1
	}
	// Ensure that ret hasn't exceeded our original max.
	if max < 0 {
		ret += max
	}
	return
}

// Calculates the pieces this reader wants downloaded, ignoring the cached value at r.pieces.
func (r *reader) piecesUncached() (ret pieceRange) {
	ra := r.readahead
	if r.readaheadFunc != nil {
		ra = r.readaheadFunc()
	}
	if ra < 1 {
		// Needs to be at least 1, because [x, x) means we don't want
		// anything.
		ra = 1
	}
	if !r.reading {
		ra = 0
	}
	if ra > r.length-r.pos {
		ra = r.length - r.pos
	}
	ret.begin, ret.end = r.t.byteRegionPieces(r.torrentOffset(r.pos), ra)
	return
}

func (r *reader) Read(b []byte) (n int, err error) {
	return r.ReadContext(context.Background(), b)
}

func (r *reader) ReadContext(ctx context.Context, b []byte) (n int, err error) {
	if len(b) > 0 {
		r.reading = true
		// TODO: Rework reader piece priorities so we don't have to push updates in to the Client
		// and take the lock here.
		r.mu.Lock()
		r.posChanged()
		r.mu.Unlock()
	}
	n, err = r.readOnceAt(ctx, b, r.pos)
	if n == 0 {
		if err == nil && len(b) > 0 {
			panic("expected error")
		} else {
			return
		}
	}

	r.mu.Lock()
	r.pos += int64(n)
	r.posChanged()
	r.mu.Unlock()
	if r.pos >= r.length {
		err = io.EOF
	} else if err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	return
}

var closedChan = make(chan struct{})

func init() {
	close(closedChan)
}

// Wait until some data should be available to read. Tickles the client if it isn't. Returns how
// much should be readable without blocking.
func (r *reader) waitAvailable(ctx context.Context, pos, wanted int64, wait bool) (avail int64, err error) {
	t := r.t
	for {
		r.t.cl.rLock()
		avail = r.available(pos, wanted)
		readerCond := t.piece(int((r.offset + pos) / t.info.PieceLength)).readerCond.Signaled()
		r.t.cl.rUnlock()
		if avail != 0 {
			return
		}
		var dontWait <-chan struct{}
		if !wait || wanted == 0 {
			dontWait = closedChan
		}
		select {
		case <-r.t.closed.Done():
			err = errors.New("torrent closed")
			return
		case <-ctx.Done():
			err = ctx.Err()
			return
		case <-r.t.dataDownloadDisallowed.On():
			err = errors.New("torrent data downloading disabled")
		case <-r.t.networkingEnabled.Off():
			err = errors.New("torrent networking disabled")
			return
		case <-dontWait:
			return
		case <-readerCond:
		}
	}
}

// Adds the reader's torrent offset to the reader object offset (for example the reader might be
// constrainted to a particular file within the torrent).
func (r *reader) torrentOffset(readerPos int64) int64 {
	return r.offset + readerPos
}

// Performs at most one successful read to torrent storage.
func (r *reader) readOnceAt(ctx context.Context, b []byte, pos int64) (n int, err error) {
	if pos >= r.length {
		err = io.EOF
		return
	}
	for {
		var avail int64
		avail, err = r.waitAvailable(ctx, pos, int64(len(b)), n == 0)
		if avail == 0 {
			return
		}
		firstPieceIndex := pieceIndex(r.torrentOffset(pos) / r.t.info.PieceLength)
		firstPieceOffset := r.torrentOffset(pos) % r.t.info.PieceLength
		b1 := missinggo.LimitLen(b, avail)
		n, err = r.t.readAt(b1, r.torrentOffset(pos))
		if n != 0 {
			err = nil
			return
		}
		r.t.cl.lock()
		// TODO: Just reset pieces in the readahead window. This might help
		// prevent thrashing with small caches and file and piece priorities.
		r.log(log.Fstr("error reading torrent %s piece %d offset %d, %d bytes: %v",
			r.t.infoHash.HexString(), firstPieceIndex, firstPieceOffset, len(b1), err))
		if !r.t.updatePieceCompletion(firstPieceIndex) {
			r.log(log.Fstr("piece %d completion unchanged", firstPieceIndex))
		}
		// Update the rest of the piece completions in the readahead window, without alerting to
		// changes (since only the first piece, the one above, could have generated the read error
		// we're currently handling).
		if r.pieces.begin != firstPieceIndex {
			panic(fmt.Sprint(r.pieces.begin, firstPieceIndex))
		}
		for index := r.pieces.begin + 1; index < r.pieces.end; index++ {
			r.t.updatePieceCompletion(index)
		}
		r.t.cl.unlock()
	}
}

// Hodor
func (r *reader) Close() error {
	r.t.cl.lock()
	defer r.t.cl.unlock()
	r.t.deleteReader(r)
	return nil
}

func (r *reader) posChanged() {
	to := r.piecesUncached()
	from := r.pieces
	if to == from {
		return
	}
	r.pieces = to
	// log.Printf("reader pos changed %v->%v", from, to)
	r.t.readerPosChanged(from, to)
}

func (r *reader) Seek(off int64, whence int) (newPos int64, err error) {
	r.mu.Lock()
	defer r.mu.Unlock()
	newPos, err = func() (int64, error) {
		switch whence {
		case io.SeekStart:
			return off, err
		case io.SeekCurrent:
			return r.pos + off, nil
		case io.SeekEnd:
			return r.length + off, nil
		default:
			return r.pos, errors.New("bad whence")
		}
	}()
	if err != nil {
		return
	}
	if newPos == r.pos {
		return
	}
	r.reading = false
	r.pos = newPos
	r.contiguousReadStartPos = newPos

	r.posChanged()
	return
}

func (r *reader) log(m log.Msg) {
	r.t.logger.Log(m.Skip(1))
}

// Implementation inspired by https://news.ycombinator.com/item?id=27019613.
func (r *reader) defaultReadaheadFunc() int64 {
	return r.pos - r.contiguousReadStartPos
}
<|endoftext|>


<commit_before>package csvutil
/*
*  File: reader.go
*  Author: Bryan Matsuo [bmatsuo@soe.ucsc.edu]
*  Created: Sat May 28 23:53:36 PDT 2011
*  Description: CSV reader library.
*/
import (
	"io"
	"bufio"
	"strings"
	"os"
)

//  A reader object for CSV data utilizing the bufio package.
type Reader struct {
	Sep int "Field separator character."
	Trim bool "Remove excess whitespace from field values."
	Cutset string "Set of characters to trim."
    LastRow Row "The last row read by the Reader."
	r io.Reader "Base reader object."
	br *bufio.Reader "For reading lines."
    bufSize int "Initial size of internal line buffers."
}

//  A simple row structure for rows read by a csvutil.Reader that
//  encapsulates any read error enountered along with any data read
//  prior to encountering an error.
type Row struct {
	Fields []string "CSV row field data"
	Error os.Error "Error encountered reading"
}

func mkrow(fields []string, err os.Error) Row {
	var r Row
	r.Fields = fields
	r.Error = err
	return r
}

//  Create a new reader object.
func NewReader(r io.Reader) *Reader {
	var csvr *Reader = new(Reader).Reset()
	csvr.r = r
	csvr.br = bufio.NewReader(r)
    csvr.bufSize = 80
	return csvr
}

//  Create a new reader with a buffer of a specified size.
func NewReaderSize(r io.Reader, size int) *Reader {
	var csvr *Reader = new(Reader).Reset()
	csvr.r = r
	var err os.Error
	csvr.br, err = bufio.NewReaderSize(r, size)
	if err != nil { panic(err) }
    csvr.bufSize = size
	return csvr
}

//  Read up to a new line and return a slice of string slices
func (csvr *Reader) ReadRow() Row {
	/* Read one row of the CSV and and return an array of the fields. */
    var(
        line, readLine, ln []byte
        err os.Error
        isPrefix bool
    )
	r := Row{Fields:nil, Error:nil}
    csvr.LastRow = r
    isPrefix = true
    for isPrefix {
        readLine, isPrefix, err = csvr.br.ReadLine()
	    r.Fields = nil
	    r.Error = err
	    if err == os.EOF { return r }
	    if err != nil { return r }
	    if isPrefix {
            readLen := len(readLine)
            if line == nil {
                line = make([]byte, 0, 2*readLen)
            }
            ln = make([]byte, readLen)
            copy(ln, readLine)
            line = append(line, ln...)
        } else {
            if line == nil {
                line = readLine
            }
        }
    }
	var fields []string
	fields = strings.FieldsFunc(
			string(line),
			func (c int) bool { return c == csvr.Sep } )
	if csvr.Trim {
		for i:=0 ; i<len(fields) ; i++ {
			fields[i] = strings.Trim(fields[i], csvr.Cutset)
		}
	}
	r.Fields = fields
    csvr.LastRow = r
	return r
}

//  Read rows into a preallocated buffer. Any error encountered is
//  returned. Returns the number of rows read in a single value return
//  context any errors encountered (including os.EOF).
func (csvr *Reader) ReadRows(rbuf [][]string) (int, os.Error) {
    var(
        err os.Error
        numRead int = 0
        n int = len(rbuf)
    )
    for i:=0 ; i<n ; i++ {
        r := csvr.ReadRow()
        numRead++
        if r.Error != nil {
            err = r.Error
            if r.Fields != nil {
                rbuf[i] = r.Fields
            }
            break
        }
        rbuf[i] = r.Fields
    }
    return numRead, err
}

//  Convenience methor to read at most n rows from csvr. Simple allocates
//  a row slice rs and calls csvr.ReadRows(rs). Returns the actual number
//  of rows read and any error that occurred (and halted reading).
func (csvr *Reader) ReadNRows(n int) (int, os.Error) {
    rows := make([][]string, n)
    return csvr.ReadRows(rows)
}

//  Reads any remaining rows of CSV data in the underlying io.Reader.
//  Uses resizing when a preallocated buffer of rows fills. Up to 16
//  rows can be read without any doubling occuring.
func (csvr *Reader) RemainingRows() (rows [][]string, err os.Error) {
    return csvr.RemainingRowsSize(16)
}

//  Like csvr.RemainingRows(), but allows specification of the initial
//  row buffer capacity.
func (csvr *Reader) RemainingRowsSize(size int) (rows [][]string, err os.Error) {
    err = nil
    var rbuf [][]string = make([][]string, 0, size)
    for r := range csvr.EachRow() {
        /*
        if cap(rbuf) == len(rbuf) {
            newbuf := make([][]string, len(rbuf), 2*len(rbuf))
            copy(rbuf,newbuf)
            rbuf = newbuf
        }
        */
        if r.Error != nil {
            err = r.Error
            if r.Fields != nil {
                rbuf = append(rbuf, r.Fields)
            }
            break
        }
        rbuf = append(rbuf, r.Fields)
    }
    return rbuf, err
}

//  A function with a concurrent routine for iterating through all
//  remaining rows of CSV data until EOF is encountered.
//      for r := range reader.EachRow() {
//          if r.Error != nil {
//              panic(r.Error)
//          }
//          var fields []string = r.Fields
//          // Process the desired entries of "fields".
//      }
//  This method utilizes a goroutine, and has the distict possibilty
//  of 'losing' at least one row when breaking of the of the iterating
//  loop. This can be managed by explicitly handling channel returned by
//  reader.EachRow() and closing/extracting the channel after the loop.
//      ch := reader.EachRow()
//      for r := range ch {
//          if r.Error != nil {
//              panic(r.Error)
//          }
//          // Process the desired entries of "fields".
//          if r.Fields[3] == "5" {
//              close(ch)
//              break
//          }
//      }
//      chrow, ok := <- ch
//      if ok {
//          // There was an extra row in the channel
//      }
//      panicrow := reader.LastRow
//      // Continue program execution.
//  However this method is simply recommended for use only when the
//  program needs to iteratively handle all remaining content of the
//  Reader.
func (csvr *Reader) EachRow() <-chan Row {
	/* Generator function for iterating through rows. */
    c := make(chan Row)
	var read_rows = func (c chan<- Row) {
        defer func() {
            if x:=recover(); x!=nil {
                /* Do nothing. */
            }
        } ()
		for true {
            csvr.LastRow = Row{Fields:nil, Error:nil}
			var r Row = csvr.ReadRow()
			if r.Fields == nil {
				if r.Error == os.EOF {
					break
				} else {
					panic(r.Error)
				}
			}
            csvr.LastRow = r
			c <- r
		}
		close(c)
	}
	go read_rows(c)
	return c
}

//  A function routine for setting all the configuration variables of a
//  csvutil.Reader in a single line.
func (csvr *Reader) Configure(sep int, trim bool, cutset string) *Reader {
	csvr.Sep = sep
	csvr.Trim = trim
	csvr.Cutset = cutset
	return csvr
}

//  Reset a Reader's configuration to the defaults. This is mostly meant
//  for internal use but is safe for general use.
func (csvr *Reader) Reset() *Reader {
	return csvr.Configure(DEFAULT_SEP, DEFAULT_TRIM, DEFAULT_CUTSET)
}

/* Comply with the reader interface. */
/*
func (csvr*Reader) Read(b []byte) (n int, err os.Error) {
	return csvr.r.Read(b)
}
*/
<commit_msg>Further cleaned ReadRow<commit_after>package csvutil
/*
*  File: reader.go
*  Author: Bryan Matsuo [bmatsuo@soe.ucsc.edu]
*  Created: Sat May 28 23:53:36 PDT 2011
*  Description: CSV reader library.
*/
import (
	"io"
	"bufio"
	"strings"
	"os"
)

//  A reader object for CSV data utilizing the bufio package.
type Reader struct {
	Sep int "Field separator character."
	Trim bool "Remove excess whitespace from field values."
	Cutset string "Set of characters to trim."
    LastRow Row "The last row read by the Reader."
	r io.Reader "Base reader object."
	br *bufio.Reader "For reading lines."
    bufSize int "Initial size of internal line buffers."
}

//  A simple row structure for rows read by a csvutil.Reader that
//  encapsulates any read error enountered along with any data read
//  prior to encountering an error.
type Row struct {
	Fields []string "CSV row field data"
	Error os.Error "Error encountered reading"
}

func mkrow(fields []string, err os.Error) Row {
	var r Row
	r.Fields = fields
	r.Error = err
	return r
}

//  Create a new reader object.
func NewReader(r io.Reader) *Reader {
	var csvr *Reader = new(Reader).Reset()
	csvr.r = r
	csvr.br = bufio.NewReader(r)
    csvr.bufSize = 80
	return csvr
}

//  Create a new reader with a buffer of a specified size.
func NewReaderSize(r io.Reader, size int) *Reader {
	var csvr *Reader = new(Reader).Reset()
	csvr.r = r
	var err os.Error
	csvr.br, err = bufio.NewReaderSize(r, size)
	if err != nil { panic(err) }
    csvr.bufSize = size
	return csvr
}

//  Read up to a new line and return a slice of string slices
func (csvr *Reader) ReadRow() Row {
	/* Read one row of the CSV and and return an array of the fields. */
    var(
        r Row
        line, readLine, ln []byte
        err os.Error
        isPrefix bool
    )
	r = Row{Fields:nil, Error:nil}
    csvr.LastRow = r
    isPrefix = true
    for isPrefix {
        readLine, isPrefix, err = csvr.br.ReadLine()
	    r.Fields = nil
	    r.Error = err
	    if err == os.EOF { return r }
	    if err != nil { return r }
	    if isPrefix {
            readLen := len(readLine)
            if line == nil {
                line = make([]byte, 0, 2*readLen)
            }
            ln = make([]byte, readLen)
            copy(ln, readLine)
            line = append(line, ln...)
        } else {
            // isPrefix is false here. The loop will break next iteration.
            if line == nil {
                line = readLine
            }
        }
    }
	r.Fields = strings.FieldsFunc(
			string(line),
			func (c int) bool { return c == csvr.Sep } )
	if csvr.Trim {
		for i:=0 ; i<len(r.Fields) ; i++ {
			r.Fields[i] = strings.Trim(r.Fields[i], csvr.Cutset)
		}
	}
    csvr.LastRow = r
	return r
}

//  Read rows into a preallocated buffer. Any error encountered is
//  returned. Returns the number of rows read in a single value return
//  context any errors encountered (including os.EOF).
func (csvr *Reader) ReadRows(rbuf [][]string) (int, os.Error) {
    var(
        err os.Error
        numRead int = 0
        n int = len(rbuf)
    )
    for i:=0 ; i<n ; i++ {
        r := csvr.ReadRow()
        numRead++
        if r.Error != nil {
            err = r.Error
            if r.Fields != nil {
                rbuf[i] = r.Fields
            }
            break
        }
        rbuf[i] = r.Fields
    }
    return numRead, err
}

//  Convenience methor to read at most n rows from csvr. Simple allocates
//  a row slice rs and calls csvr.ReadRows(rs). Returns the actual number
//  of rows read and any error that occurred (and halted reading).
func (csvr *Reader) ReadNRows(n int) (int, os.Error) {
    rows := make([][]string, n)
    return csvr.ReadRows(rows)
}

//  Reads any remaining rows of CSV data in the underlying io.Reader.
//  Uses resizing when a preallocated buffer of rows fills. Up to 16
//  rows can be read without any doubling occuring.
func (csvr *Reader) RemainingRows() (rows [][]string, err os.Error) {
    return csvr.RemainingRowsSize(16)
}

//  Like csvr.RemainingRows(), but allows specification of the initial
//  row buffer capacity.
func (csvr *Reader) RemainingRowsSize(size int) (rows [][]string, err os.Error) {
    err = nil
    var rbuf [][]string = make([][]string, 0, size)
    for r := range csvr.EachRow() {
        /*
        if cap(rbuf) == len(rbuf) {
            newbuf := make([][]string, len(rbuf), 2*len(rbuf))
            copy(rbuf,newbuf)
            rbuf = newbuf
        }
        */
        if r.Error != nil {
            err = r.Error
            if r.Fields != nil {
                rbuf = append(rbuf, r.Fields)
            }
            break
        }
        rbuf = append(rbuf, r.Fields)
    }
    return rbuf, err
}

//  A function with a concurrent routine for iterating through all
//  remaining rows of CSV data until EOF is encountered.
//      for r := range reader.EachRow() {
//          if r.Error != nil {
//              panic(r.Error)
//          }
//          var fields []string = r.Fields
//          // Process the desired entries of "fields".
//      }
//  This method utilizes a goroutine, and has the distict possibilty
//  of 'losing' at least one row when breaking of the of the iterating
//  loop. This can be managed by explicitly handling channel returned by
//  reader.EachRow() and closing/extracting the channel after the loop.
//      ch := reader.EachRow()
//      for r := range ch {
//          if r.Error != nil {
//              panic(r.Error)
//          }
//          // Process the desired entries of "fields".
//          if r.Fields[3] == "5" {
//              close(ch)
//              break
//          }
//      }
//      chrow, ok := <- ch
//      if ok {
//          // There was an extra row in the channel
//      }
//      panicrow := reader.LastRow
//      // Continue program execution.
//  However this method is simply recommended for use only when the
//  program needs to iteratively handle all remaining content of the
//  Reader.
func (csvr *Reader) EachRow() <-chan Row {
	/* Generator function for iterating through rows. */
    c := make(chan Row)
	var read_rows = func (c chan<- Row) {
        defer func() {
            if x:=recover(); x!=nil {
                /* Do nothing. */
            }
        } ()
		for true {
            csvr.LastRow = Row{Fields:nil, Error:nil}
			var r Row = csvr.ReadRow()
			if r.Fields == nil {
				if r.Error == os.EOF {
					break
				} else {
					panic(r.Error)
				}
			}
            csvr.LastRow = r
			c <- r
		}
		close(c)
	}
	go read_rows(c)
	return c
}

//  A function routine for setting all the configuration variables of a
//  csvutil.Reader in a single line.
func (csvr *Reader) Configure(sep int, trim bool, cutset string) *Reader {
	csvr.Sep = sep
	csvr.Trim = trim
	csvr.Cutset = cutset
	return csvr
}

//  Reset a Reader's configuration to the defaults. This is mostly meant
//  for internal use but is safe for general use.
func (csvr *Reader) Reset() *Reader {
	return csvr.Configure(DEFAULT_SEP, DEFAULT_TRIM, DEFAULT_CUTSET)
}

/* Comply with the reader interface. */
/*
func (csvr*Reader) Read(b []byte) (n int, err os.Error) {
	return csvr.r.Read(b)
}
*/
<|endoftext|>


<commit_before>// Copyright 2016 FullStory, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package smservice

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/fullstorydev/gosolr/smutil"
	"github.com/fullstorydev/gosolr/solrman/solrmanapi"
	"github.com/fullstorydev/gosolr/solrmonitor"
)

// Long call; run in a go routine. Run the given move operation and records the result when done.
func (s *SolrManService) runMoveOperation(move solrmanapi.OpRecord) {
	err := s.doRunMoveOperation(move)
	if err != nil {
		move.Error = fmt.Sprintf("failed MoveShard request: %s with err: %s", move.String(), err)
		move.FinishedMs = nowMillis()
		s.Logger.Errorf("failed MoveShard request: %s with err: %s", move.String(), err)
	} else {
		move.FinishedMs = nowMillis()
		s.Logger.Infof("completed MoveShard request: %s", move.String())
	}

	func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		delete(s.inProgressOps, move.Key())
	}()

	if err := s.Storage.DelInProgressOp(move); err != nil {
		s.Logger.Warningf("failed to DelInProgressOp completed move %s: %s", move.Key(), err)
	}
	if err := s.Storage.AddCompletedOp(move); err != nil {
		s.Logger.Warningf("failed to AddCompletedOp completed move %s: %s", move.Key(), err)
	}
}

// Run the given move operation; returns an error, or nil if it succeeds.
func (s *SolrManService) doRunMoveOperation(move solrmanapi.OpRecord) error {
	// Fail if we can't retrieve the initial state.
	coll, err := s.SolrMonitor.GetCollectionState(move.Collection)
	if err != nil {
		return err
	}
	if coll == nil {
		return smutil.Cherrf(err, "no such collection (maybe it disappeared?)")
	}

	var replicas map[string]solrmonitor.ReplicaState
	if shard, ok := coll.Shards[move.Shard]; !ok {
		// guard against no-such-shard
		return smutil.Cherrf(err, "no such shard %s in collection %s", move.Shard, move.Collection)
	} else {
		replicas = shard.Replicas
	}

	s.Audit.BeforeOp(move, *coll)
	lastZkVersion := coll.ZkNodeVersion
	success := false
	defer func() {
		var newCollState *solrmonitor.CollectionState
		for retry := 0; retry < 3; retry++ {
			var err error
			if newCollState, err = s.SolrMonitor.GetCollectionState(move.Collection); err != nil {
				s.Logger.Errorf("failed to retrieve collection state after op %s: %s", move.String(), err)
				return
			} else {
				if newCollState.ZkNodeVersion > lastZkVersion {
					break
				}
			}
			// Haven't seen an update yet, wait a little and try again.
			time.Sleep(5 * time.Second)
		}
		if success {
			s.Audit.SuccessOp(move, *newCollState)
		} else {
			s.Audit.FailedOp(move, *newCollState)
			s.disable()
		}
	}()

	// Add a replica if none exists.
	// TODO: scottb handle timeout separately from other failures, we should loop and retry?
	if replica := findReplica(replicas, move.DstNode, anyReplica); replica == "" {
		// No replica exists, need to add one.
		if err := s.solrClient.AddReplica(move.Collection, move.Shard, move.DstNode, ""); err != nil {
			return smutil.Cherrf(err, "failed to issue ADDREPLICA command")
		}
		s.Logger.Debugf("ADDREPLICA command issued successfully MoveShard request: %s", move.String())
	}

	// Wait on the replica to sync up and be "live"
	for {
		coll, err := s.SolrMonitor.GetCollectionState(move.Collection)
		if err != nil {
			// TODO: give up eventually after too many retries?
			s.Logger.Warningf("Error retrieving collection state for %s: %s", move.Collection, err)
		} else {
			if shard, ok := coll.Shards[move.Shard]; ok {
				// guard against no-such-shard
				replicas = shard.Replicas
			}
			if replica := findReplica(replicas, move.DstNode, activeReplica); replica != "" {
				// found a good replica!  sync complete
				s.Logger.Debugf("active replica %q found on %s - time to delete the original", replica, move.DstNode)
				s.Audit.SuccessOp(move, *coll) // should log state where both replicas exist
				lastZkVersion = coll.ZkNodeVersion
				break
			}
		}
		// consider something event-driven instead of polling
		time.Sleep(10 * time.Second)
	}

	// Now delete the original
	original := findReplica(replicas, move.SrcNode, activeReplica)
	if original == "" {
		return smutil.Errorf("no original found for shard %s of collection %q on node %s!?", move.Shard, move.Collection, move.SrcNode)
	}

	if err := s.solrClient.DeleteReplica(move.Collection, move.Shard, original, ""); err != nil {
		// If this fails because the replica doesn't exist, then we assume that some prior attempt finished the job
		// (probably the http request timed out, but the actual api action eventually succeeded within Solr).  So we
		// just treat this as a success.
		if solrErr := isNoSuchReplicaError(err, original); solrErr != nil {
			s.Logger.Debugf("assuming replica was previously deleted: %s", solrErr)
			success = true
			return nil
		} else {
			return smutil.Cherrf(err, "failed to issue DELETEREPLICA command")
		}
	}
	s.Logger.Debugf("DELETEREPLICA command issued successfully MoveShard request: %s", move.String())
	success = true
	return nil
}

func isNoSuchReplicaError(err error, replica string) *ErrorRsp {
	if err, ok := smutil.Root(err).(*ErrorRsp); ok {
		s := fmt.Sprintf("Invalid replica : %s", replica)
		if err.Code == http.StatusBadRequest && strings.Contains(err.Msg, s) {
			return err
		}
	}

	return nil
}
<commit_msg>solrman: delete original even if the original is inactive (#8814)<commit_after>// Copyright 2016 FullStory, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package smservice

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/fullstorydev/gosolr/smutil"
	"github.com/fullstorydev/gosolr/solrman/solrmanapi"
	"github.com/fullstorydev/gosolr/solrmonitor"
)

// Long call; run in a go routine. Run the given move operation and records the result when done.
func (s *SolrManService) runMoveOperation(move solrmanapi.OpRecord) {
	err := s.doRunMoveOperation(move)
	if err != nil {
		move.Error = fmt.Sprintf("failed MoveShard request: %s with err: %s", move.String(), err)
		move.FinishedMs = nowMillis()
		s.Logger.Errorf("failed MoveShard request: %s with err: %s", move.String(), err)
	} else {
		move.FinishedMs = nowMillis()
		s.Logger.Infof("completed MoveShard request: %s", move.String())
	}

	func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		delete(s.inProgressOps, move.Key())
	}()

	if err := s.Storage.DelInProgressOp(move); err != nil {
		s.Logger.Warningf("failed to DelInProgressOp completed move %s: %s", move.Key(), err)
	}
	if err := s.Storage.AddCompletedOp(move); err != nil {
		s.Logger.Warningf("failed to AddCompletedOp completed move %s: %s", move.Key(), err)
	}
}

// Run the given move operation; returns an error, or nil if it succeeds.
func (s *SolrManService) doRunMoveOperation(move solrmanapi.OpRecord) error {
	// Fail if we can't retrieve the initial state.
	coll, err := s.SolrMonitor.GetCollectionState(move.Collection)
	if err != nil {
		return err
	}
	if coll == nil {
		return smutil.Cherrf(err, "no such collection (maybe it disappeared?)")
	}

	var replicas map[string]solrmonitor.ReplicaState
	if shard, ok := coll.Shards[move.Shard]; !ok {
		// guard against no-such-shard
		return smutil.Cherrf(err, "no such shard %s in collection %s", move.Shard, move.Collection)
	} else {
		replicas = shard.Replicas
	}

	s.Audit.BeforeOp(move, *coll)
	lastZkVersion := coll.ZkNodeVersion
	success := false
	defer func() {
		var newCollState *solrmonitor.CollectionState
		for retry := 0; retry < 3; retry++ {
			var err error
			if newCollState, err = s.SolrMonitor.GetCollectionState(move.Collection); err != nil {
				s.Logger.Errorf("failed to retrieve collection state after op %s: %s", move.String(), err)
				return
			} else {
				if newCollState.ZkNodeVersion > lastZkVersion {
					break
				}
			}
			// Haven't seen an update yet, wait a little and try again.
			time.Sleep(5 * time.Second)
		}
		if success {
			s.Audit.SuccessOp(move, *newCollState)
		} else {
			s.Audit.FailedOp(move, *newCollState)
			s.disable()
		}
	}()

	// Add a replica if none exists.
	// TODO: scottb handle timeout separately from other failures, we should loop and retry?
	if replica := findReplica(replicas, move.DstNode, anyReplica); replica == "" {
		// No replica exists, need to add one.
		if err := s.solrClient.AddReplica(move.Collection, move.Shard, move.DstNode, ""); err != nil {
			return smutil.Cherrf(err, "failed to issue ADDREPLICA command")
		}
		s.Logger.Debugf("ADDREPLICA command issued successfully MoveShard request: %s", move.String())
	}

	// Wait on the replica to sync up and be "live"
	for {
		coll, err := s.SolrMonitor.GetCollectionState(move.Collection)
		if err != nil {
			// TODO: give up eventually after too many retries?
			s.Logger.Warningf("Error retrieving collection state for %s: %s", move.Collection, err)
		} else {
			if shard, ok := coll.Shards[move.Shard]; ok {
				// guard against no-such-shard
				replicas = shard.Replicas
			}
			if replica := findReplica(replicas, move.DstNode, activeReplica); replica != "" {
				// found a good replica!  sync complete
				s.Logger.Debugf("active replica %q found on %s - time to delete the original", replica, move.DstNode)
				s.Audit.SuccessOp(move, *coll) // should log state where both replicas exist
				lastZkVersion = coll.ZkNodeVersion
				break
			}
		}
		// consider something event-driven instead of polling
		time.Sleep(10 * time.Second)
	}

	// Now delete the original
	original := findReplica(replicas, move.SrcNode, anyReplica)
	if original == "" {
		return smutil.Errorf("no original found for shard %s of collection %q on node %s!?", move.Shard, move.Collection, move.SrcNode)
	}

	if err := s.solrClient.DeleteReplica(move.Collection, move.Shard, original, ""); err != nil {
		// If this fails because the replica doesn't exist, then we assume that some prior attempt finished the job
		// (probably the http request timed out, but the actual api action eventually succeeded within Solr).  So we
		// just treat this as a success.
		if solrErr := isNoSuchReplicaError(err, original); solrErr != nil {
			s.Logger.Debugf("assuming replica was previously deleted: %s", solrErr)
			success = true
			return nil
		} else {
			return smutil.Cherrf(err, "failed to issue DELETEREPLICA command")
		}
	}
	s.Logger.Debugf("DELETEREPLICA command issued successfully MoveShard request: %s", move.String())
	success = true
	return nil
}

func isNoSuchReplicaError(err error, replica string) *ErrorRsp {
	if err, ok := smutil.Root(err).(*ErrorRsp); ok {
		s := fmt.Sprintf("Invalid replica : %s", replica)
		if err.Code == http.StatusBadRequest && strings.Contains(err.Msg, s) {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package velocity

import (
	"fmt"
	"strings"
)

type DockerBuild struct {
	BaseStep   `yaml:",inline"`
	Dockerfile string   `json:"dockerfile" yaml:"dockerfile"`
	Context    string   `json:"context" yaml:"context"`
	Tags       []string `json:"tags" yaml:"tags"`
}

func NewDockerBuild() *DockerBuild {
	return &DockerBuild{
		Dockerfile: "",
		Context:    "",
		Tags:       []string{},
	}
}

func (dB DockerBuild) GetType() string {
	return "build"
}

func (dB DockerBuild) GetDescription() string {
	return dB.Description
}

func (dB DockerBuild) GetDetails() string {
	return fmt.Sprintf("dockerfile: %s, context: %s, tags: %s", dB.Dockerfile, dB.Context, dB.Tags)
}

func (dB *DockerBuild) Execute(emitter Emitter, params map[string]Parameter) error {
	emitter.Write([]byte(fmt.Sprintf("%s\n## %s\n\x1b[0m", infoANSI, dB.Description)))

	return buildContainer(
		dB.Context,
		dB.Dockerfile,
		dB.Tags,
		params,
		emitter,
	)
}

func (dB *DockerBuild) Validate(params map[string]Parameter) error {
	return nil
}

func (dB *DockerBuild) SetParams(params map[string]Parameter) error {
	for paramName, param := range params {
		dB.Context = strings.Replace(dB.Context, fmt.Sprintf("${%s}", paramName), param.Value, -1)
		dB.Dockerfile = strings.Replace(dB.Dockerfile, fmt.Sprintf("${%s}", paramName), param.Value, -1)

		tags := []string{}

		for _, t := range dB.Tags {
			tags = append(tags, strings.Replace(t, fmt.Sprintf("${%s}", paramName), param.Value, -1))
		}
		dB.Tags = tags
	}
	return nil
}
<commit_msg>[backend] Added success emit for docker build step<commit_after>package velocity

import (
	"fmt"
	"strings"
)

type DockerBuild struct {
	BaseStep   `yaml:",inline"`
	Dockerfile string   `json:"dockerfile" yaml:"dockerfile"`
	Context    string   `json:"context" yaml:"context"`
	Tags       []string `json:"tags" yaml:"tags"`
}

func NewDockerBuild() *DockerBuild {
	return &DockerBuild{
		Dockerfile: "",
		Context:    "",
		Tags:       []string{},
	}
}

func (dB DockerBuild) GetType() string {
	return "build"
}

func (dB DockerBuild) GetDescription() string {
	return dB.Description
}

func (dB DockerBuild) GetDetails() string {
	return fmt.Sprintf("dockerfile: %s, context: %s, tags: %s", dB.Dockerfile, dB.Context, dB.Tags)
}

func (dB *DockerBuild) Execute(emitter Emitter, params map[string]Parameter) error {
	emitter.Write([]byte(fmt.Sprintf("%s\n## %s\n\x1b[0m", infoANSI, dB.Description)))

	err := buildContainer(
		dB.Context,
		dB.Dockerfile,
		dB.Tags,
		params,
		emitter,
	)

	if err != nil {
		emitter.SetStatus("failed")
		emitter.Write([]byte(fmt.Sprintf("%s\n### FAILED: %s \x1b[0m", errorANSI, err)))
		return err
	}

	emitter.SetStatus("success")
	emitter.Write([]byte(fmt.Sprintf("%s\n### SUCCESS \x1b[0m", successANSI)))
	return nil
}

func (dB *DockerBuild) Validate(params map[string]Parameter) error {
	return nil
}

func (dB *DockerBuild) SetParams(params map[string]Parameter) error {
	for paramName, param := range params {
		dB.Context = strings.Replace(dB.Context, fmt.Sprintf("${%s}", paramName), param.Value, -1)
		dB.Dockerfile = strings.Replace(dB.Dockerfile, fmt.Sprintf("${%s}", paramName), param.Value, -1)

		tags := []string{}

		for _, t := range dB.Tags {
			tags = append(tags, strings.Replace(t, fmt.Sprintf("${%s}", paramName), param.Value, -1))
		}
		dB.Tags = tags
	}
	return nil
}
<|endoftext|>


<commit_before>package mmatcher

type Record struct {
	Id   string
	Atts []Atter
}

func (a *Record) Matches(b *Record, atts ...int) bool {
	if len(atts) <= 0 {
		atts = make([]int, len(a.Atts))
		for i := range atts {
			atts[i] = i
		}
	}
	matches := make([]bool, len(atts))
	for i, n := range atts {
		if n >= len(a.Atts) || i >= len(b.Atts) {
			return false
		}
		matches[i] = a.matchesAtt(b, NumericAtt{}, n)
	}
	for _, m := range matches {
		if !m {
			return false
		}
	}
	return true
}

func (a *Record) matchesAtt(b *Record, e Atter, i int) bool {
	if i < len(a.Atts) && i < len(b.Atts) {
		return a.Atts[i].Equal(b.Atts[i], e)
	}
	return false
}
<commit_msg>Rework record matching to use blank ranges if not given<commit_after>package mmatcher

type Record struct {
	Id   string
	Atts []Atter
}

func (a *Record) Matches(b *Record, positions ...int) bool {
	if len(positions) <= 0 {
		positions = make([]int, len(a.Atts))
		for i := range positions {
			positions[i] = i
		}
	}
	e := make([]Atter, len(a.Atts))
	return a.MatchesRanges(b, e, positions...)
}

func (a *Record) MatchesRanges(b *Record, e []Atter, positions ...int) bool {
	if len(a.Atts) != len(b.Atts) || len(e) != len(a.Atts) {
		return false
	}
	if len(positions) <= 0 {
		positions = make([]int, len(a.Atts))
		for i := range positions {
			positions[i] = i
		}
	}
	matches := make([]bool, len(positions))
	for i, n := range positions {
		if n >= len(a.Atts) || i >= len(b.Atts) {
			return false
		}
		matches[i] = a.matchesAtt(b, e[i], n)
	}
	for _, m := range matches {
		if !m {
			return false
		}
	}
	return true
}

func (a *Record) matchesAtt(b *Record, e Atter, i int) bool {
	if i < len(a.Atts) && i < len(b.Atts) {
		return a.Atts[i].Equal(b.Atts[i], e)
	}
	return false
}
<|endoftext|>


<commit_before>package router

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"net"
	"runtime"
	"time"

	mbus "github.com/cloudfoundry/go_cfmessagebus"
	vcap "github.com/cloudfoundry/gorouter/common"
	"github.com/cloudfoundry/gorouter/config"
	"github.com/cloudfoundry/gorouter/log"
	"github.com/cloudfoundry/gorouter/proxy"
	"github.com/cloudfoundry/gorouter/registry"
	"github.com/cloudfoundry/gorouter/route"
	"github.com/cloudfoundry/gorouter/util"
	"github.com/cloudfoundry/gorouter/varz"
)

type Router struct {
	config     *config.Config
	proxy      *proxy.Proxy
	mbusClient mbus.MessageBus
	registry   *registry.Registry
	varz       varz.Varz
	component  *vcap.VcapComponent
}

func NewRouter(c *config.Config) *Router {
	router := &Router{
		config: c,
	}

	// setup number of procs
	if router.config.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(router.config.GoMaxProcs)
	}

	router.establishMBus()

	router.registry = registry.NewRegistry(router.config, router.mbusClient)
	router.registry.StartPruningCycle()

	router.varz = varz.NewVarz(router.registry)
	router.proxy = proxy.NewProxy(router.config, router.registry, router.varz)

	var host string
	if router.config.Status.Port != 0 {
		host = fmt.Sprintf("%s:%d", router.config.Ip, router.config.Status.Port)
	}

	varz := &vcap.Varz{
		UniqueVarz: router.varz,
	}
	varz.LogCounts = log.Counter

	healthz := &vcap.Healthz{
		LockableObject: router.registry,
	}

	router.component = &vcap.VcapComponent{
		Type:        "Router",
		Index:       router.config.Index,
		Host:        host,
		Credentials: []string{router.config.Status.User, router.config.Status.Pass},
		Config:      router.config,
		Varz:        varz,
		Healthz:     healthz,
		InfoRoutes: map[string]json.Marshaler{
			"/routes": router.registry,
		},
	}

	vcap.StartComponent(router.component)

	return router
}

func (r *Router) RegisterComponent() {
	vcap.Register(r.component, r.mbusClient)
}

type registryMessage struct {
	Host string            `json:"host"`
	Port uint16            `json:"port"`
	Uris []route.Uri       `json:"uris"`
	Tags map[string]string `json:"tags"`
	App  string            `json:"app"`

	PrivateInstanceId string `json:"private_instance_id"`
}

func (r *Router) subscribeRegistry(subject string, successCallback func(*registryMessage)) {
	callback := func(payload []byte) {
		var msg registryMessage

		err := json.Unmarshal(payload, &msg)
		if err != nil {
			logMessage := fmt.Sprintf("%s: Error unmarshalling JSON (%d; %s): %s", subject, len(payload), payload, err)
			log.Warnd(map[string]interface{}{"payload": string(payload)}, logMessage)
		}

		logMessage := fmt.Sprintf("%s: Received message", subject)
		log.Debugd(map[string]interface{}{"message": msg}, logMessage)

		successCallback(&msg)
	}
	err := r.mbusClient.Subscribe(subject, callback)
	if err != nil {
		log.Errorf("Error subscribing to %s: %s", subject, err)
	}
}

func (r *Router) SubscribeRegister() {
	r.subscribeRegistry("router.register", func(registryMessage *registryMessage) {
		log.Infof("Got router.register: %v", registryMessage)

		for _, uri := range registryMessage.Uris {
			r.registry.Register(
				uri,
				makeRouteEndpoint(registryMessage),
			)
		}
	})
}

func (r *Router) SubscribeUnregister() {
	r.subscribeRegistry("router.unregister", func(registryMessage *registryMessage) {
		log.Infof("Got router.unregister: %v", registryMessage)

		for _, uri := range registryMessage.Uris {
			r.registry.Unregister(
				uri,
				makeRouteEndpoint(registryMessage),
			)
		}
	})
}

func (r *Router) HandleGreetings() {
	r.mbusClient.RespondToChannel("router.greet", func(_ []byte) []byte {
		response, _ := r.greetMessage()
		return response
	})
}

func (r *Router) flushApps(t time.Time) {
	x := r.registry.ActiveSince(t)

	y, err := json.Marshal(x)
	if err != nil {
		log.Warnf("flushApps: Error marshalling JSON: %s", err)
		return
	}

	b := bytes.Buffer{}
	w := zlib.NewWriter(&b)
	w.Write(y)
	w.Close()

	z := b.Bytes()

	log.Debugf("Active apps: %d, message size: %d", len(x), len(z))

	r.mbusClient.Publish("router.active_apps", z)
}

func (r *Router) ScheduleFlushApps() {
	if r.config.PublishActiveAppsInterval == 0 {
		return
	}

	go func() {
		t := time.NewTicker(r.config.PublishActiveAppsInterval)
		x := time.Now()

		for {
			select {
			case <-t.C:
				y := time.Now()
				r.flushApps(x)
				x = y
			}
		}
	}()
}

func (r *Router) SendStartMessage() {
	b, err := r.greetMessage()
	if err != nil {
		panic(err)
	}

	// Send start message once at start
	r.mbusClient.Publish("router.start", b)
}

func (r *Router) greetMessage() ([]byte, error) {
	host, err := vcap.LocalIP()
	if err != nil {
		return nil, err
	}

	d := vcap.RouterStart{
		vcap.GenerateUUID(),
		[]string{host},
		r.config.StartResponseDelayIntervalInSeconds,
	}

	return json.Marshal(d)
}

func (router *Router) Run() {
	var err error

	for {
		err = router.mbusClient.Connect()
		if err == nil {
			break
		}
		log.Errorf("Could not connect to NATS: %s", err)
		time.Sleep(500 * time.Millisecond)
	}

	router.RegisterComponent()

	// Subscribe register/unregister router
	router.SubscribeRegister()
	router.HandleGreetings()
	router.SubscribeUnregister()

	// Kickstart sending start messages
	router.SendStartMessage()

	// Send start again on reconnect
	router.mbusClient.OnConnect(func() {
		router.SendStartMessage()
	})

	// Schedule flushing active app's app_id
	router.ScheduleFlushApps()

	// Wait for one start message send interval, such that the router's registry
	// can be populated before serving requests.
	if router.config.StartResponseDelayInterval != 0 {
		log.Infof("Waiting %s before listening...", router.config.StartResponseDelayInterval)
		time.Sleep(router.config.StartResponseDelayInterval)
	}

	listen, err := net.Listen("tcp", fmt.Sprintf(":%d", router.config.Port))
	if err != nil {
		log.Fatalf("net.Listen: %s", err)
	}

	util.WritePidFile(router.config.Pidfile)

	log.Infof("Listening on %s", listen.Addr())

	server := proxy.Server{Handler: router.proxy}

	go func() {
		err := server.Serve(listen)
		if err != nil {
			log.Fatalf("proxy.Serve: %s", err)
		}
	}()
}

func (r *Router) establishMBus() {
	mbusClient, err := mbus.NewMessageBus("NATS")
	r.mbusClient = mbusClient
	if err != nil {
		panic("Could not connect to NATS")
	}

	host := r.config.Nats.Host
	user := r.config.Nats.User
	pass := r.config.Nats.Pass
	port := r.config.Nats.Port

	r.mbusClient.Configure(host, int(port), user, pass)
}

func makeRouteEndpoint(registryMessage *registryMessage) *route.Endpoint {
	return &route.Endpoint{
		Host: registryMessage.Host,
		Port: registryMessage.Port,

		ApplicationId: registryMessage.App,
		Tags:          registryMessage.Tags,

		PrivateInstanceId: registryMessage.PrivateInstanceId,
	}
}
<commit_msg>Reorganize methods in router<commit_after>package router

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"fmt"
	"net"
	"runtime"
	"time"

	mbus "github.com/cloudfoundry/go_cfmessagebus"
	vcap "github.com/cloudfoundry/gorouter/common"
	"github.com/cloudfoundry/gorouter/config"
	"github.com/cloudfoundry/gorouter/log"
	"github.com/cloudfoundry/gorouter/proxy"
	"github.com/cloudfoundry/gorouter/registry"
	"github.com/cloudfoundry/gorouter/route"
	"github.com/cloudfoundry/gorouter/util"
	"github.com/cloudfoundry/gorouter/varz"
)

type Router struct {
	config     *config.Config
	proxy      *proxy.Proxy
	mbusClient mbus.MessageBus
	registry   *registry.Registry
	varz       varz.Varz
	component  *vcap.VcapComponent
}

func NewRouter(c *config.Config) *Router {
	router := &Router{
		config: c,
	}

	// setup number of procs
	if router.config.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(router.config.GoMaxProcs)
	}

	router.establishMBus()

	router.registry = registry.NewRegistry(router.config, router.mbusClient)
	router.registry.StartPruningCycle()

	router.varz = varz.NewVarz(router.registry)
	router.proxy = proxy.NewProxy(router.config, router.registry, router.varz)

	var host string
	if router.config.Status.Port != 0 {
		host = fmt.Sprintf("%s:%d", router.config.Ip, router.config.Status.Port)
	}

	varz := &vcap.Varz{
		UniqueVarz: router.varz,
	}
	varz.LogCounts = log.Counter

	healthz := &vcap.Healthz{
		LockableObject: router.registry,
	}

	router.component = &vcap.VcapComponent{
		Type:        "Router",
		Index:       router.config.Index,
		Host:        host,
		Credentials: []string{router.config.Status.User, router.config.Status.Pass},
		Config:      router.config,
		Varz:        varz,
		Healthz:     healthz,
		InfoRoutes: map[string]json.Marshaler{
			"/routes": router.registry,
		},
	}

	vcap.StartComponent(router.component)

	return router
}

func (router *Router) Run() {
	var err error

	for {
		err = router.mbusClient.Connect()
		if err == nil {
			break
		}
		log.Errorf("Could not connect to NATS: %s", err)
		time.Sleep(500 * time.Millisecond)
	}

	router.RegisterComponent()

	// Subscribe register/unregister router
	router.SubscribeRegister()
	router.HandleGreetings()
	router.SubscribeUnregister()

	// Kickstart sending start messages
	router.SendStartMessage()

	// Send start again on reconnect
	router.mbusClient.OnConnect(func() {
		router.SendStartMessage()
	})

	// Schedule flushing active app's app_id
	router.ScheduleFlushApps()

	// Wait for one start message send interval, such that the router's registry
	// can be populated before serving requests.
	if router.config.StartResponseDelayInterval != 0 {
		log.Infof("Waiting %s before listening...", router.config.StartResponseDelayInterval)
		time.Sleep(router.config.StartResponseDelayInterval)
	}

	listen, err := net.Listen("tcp", fmt.Sprintf(":%d", router.config.Port))
	if err != nil {
		log.Fatalf("net.Listen: %s", err)
	}

	util.WritePidFile(router.config.Pidfile)

	log.Infof("Listening on %s", listen.Addr())

	server := proxy.Server{Handler: router.proxy}

	go func() {
		err := server.Serve(listen)
		if err != nil {
			log.Fatalf("proxy.Serve: %s", err)
		}
	}()
}

func (r *Router) RegisterComponent() {
	vcap.Register(r.component, r.mbusClient)
}

type registryMessage struct {
	Host string            `json:"host"`
	Port uint16            `json:"port"`
	Uris []route.Uri       `json:"uris"`
	Tags map[string]string `json:"tags"`
	App  string            `json:"app"`

	PrivateInstanceId string `json:"private_instance_id"`
}

func (r *Router) SubscribeRegister() {
	r.subscribeRegistry("router.register", func(registryMessage *registryMessage) {
		log.Infof("Got router.register: %v", registryMessage)

		for _, uri := range registryMessage.Uris {
			r.registry.Register(
				uri,
				makeRouteEndpoint(registryMessage),
			)
		}
	})
}

func (r *Router) SubscribeUnregister() {
	r.subscribeRegistry("router.unregister", func(registryMessage *registryMessage) {
		log.Infof("Got router.unregister: %v", registryMessage)

		for _, uri := range registryMessage.Uris {
			r.registry.Unregister(
				uri,
				makeRouteEndpoint(registryMessage),
			)
		}
	})
}

func (r *Router) HandleGreetings() {
	r.mbusClient.RespondToChannel("router.greet", func(_ []byte) []byte {
		response, _ := r.greetMessage()
		return response
	})
}

func (r *Router) SendStartMessage() {
	b, err := r.greetMessage()
	if err != nil {
		panic(err)
	}

	// Send start message once at start
	r.mbusClient.Publish("router.start", b)
}

func (r *Router) ScheduleFlushApps() {
	if r.config.PublishActiveAppsInterval == 0 {
		return
	}

	go func() {
		t := time.NewTicker(r.config.PublishActiveAppsInterval)
		x := time.Now()

		for {
			select {
			case <-t.C:
				y := time.Now()
				r.flushApps(x)
				x = y
			}
		}
	}()
}

func (r *Router) flushApps(t time.Time) {
	x := r.registry.ActiveSince(t)

	y, err := json.Marshal(x)
	if err != nil {
		log.Warnf("flushApps: Error marshalling JSON: %s", err)
		return
	}

	b := bytes.Buffer{}
	w := zlib.NewWriter(&b)
	w.Write(y)
	w.Close()

	z := b.Bytes()

	log.Debugf("Active apps: %d, message size: %d", len(x), len(z))

	r.mbusClient.Publish("router.active_apps", z)
}

func (r *Router) greetMessage() ([]byte, error) {
	host, err := vcap.LocalIP()
	if err != nil {
		return nil, err
	}

	d := vcap.RouterStart{
		vcap.GenerateUUID(),
		[]string{host},
		r.config.StartResponseDelayIntervalInSeconds,
	}

	return json.Marshal(d)
}

func (r *Router) subscribeRegistry(subject string, successCallback func(*registryMessage)) {
	callback := func(payload []byte) {
		var msg registryMessage

		err := json.Unmarshal(payload, &msg)
		if err != nil {
			logMessage := fmt.Sprintf("%s: Error unmarshalling JSON (%d; %s): %s", subject, len(payload), payload, err)
			log.Warnd(map[string]interface{}{"payload": string(payload)}, logMessage)
		}

		logMessage := fmt.Sprintf("%s: Received message", subject)
		log.Debugd(map[string]interface{}{"message": msg}, logMessage)

		successCallback(&msg)
	}
	err := r.mbusClient.Subscribe(subject, callback)
	if err != nil {
		log.Errorf("Error subscribing to %s: %s", subject, err)
	}
}

func (r *Router) establishMBus() {
	mbusClient, err := mbus.NewMessageBus("NATS")
	r.mbusClient = mbusClient
	if err != nil {
		panic("Could not connect to NATS")
	}

	host := r.config.Nats.Host
	user := r.config.Nats.User
	pass := r.config.Nats.Pass
	port := r.config.Nats.Port

	r.mbusClient.Configure(host, int(port), user, pass)
}

func makeRouteEndpoint(registryMessage *registryMessage) *route.Endpoint {
	return &route.Endpoint{
		Host: registryMessage.Host,
		Port: registryMessage.Port,

		ApplicationId: registryMessage.App,
		Tags:          registryMessage.Tags,

		PrivateInstanceId: registryMessage.PrivateInstanceId,
	}
}
<|endoftext|>