<commit_before>package mape

import (
	"debug/pe"
)

// ConvertOptionalHeader stores a given
// 32 bit OptionalHeader struct inside a 64 bit OptionalHeader
func ConvertOptionalHeader(file *pe.File) OptionalHeader {

	var opt OptionalHeader

	if file.Machine == 0x8664 {
		opt.Magic = file.OptionalHeader.(*pe.OptionalHeader64).Magic
		opt.MajorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorLinkerVersion
		opt.MinorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorLinkerVersion
		opt.SizeOfCode = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfCode
		opt.SizeOfInitializedData = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfInitializedData
		opt.SizeOfUninitializedData = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfUninitializedData
		opt.AddressOfEntryPoint = file.OptionalHeader.(*pe.OptionalHeader64).AddressOfEntryPoint
		opt.BaseOfCode = file.OptionalHeader.(*pe.OptionalHeader64).BaseOfCode
		opt.ImageBase = file.OptionalHeader.(*pe.OptionalHeader64).ImageBase
		opt.SectionAlignment = file.OptionalHeader.(*pe.OptionalHeader64).SectionAlignment
		opt.FileAlignment = file.OptionalHeader.(*pe.OptionalHeader64).FileAlignment
		opt.MajorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorOperatingSystemVersion
		opt.MinorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorOperatingSystemVersion
		opt.MajorImageVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorImageVersion
		opt.MinorImageVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorImageVersion
		opt.MajorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorSubsystemVersion
		opt.MinorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorSubsystemVersion
		opt.Win32VersionValue = file.OptionalHeader.(*pe.OptionalHeader64).Win32VersionValue
		opt.SizeOfImage = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfImage
		opt.SizeOfHeaders = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeaders
		opt.CheckSum = file.OptionalHeader.(*pe.OptionalHeader64).CheckSum
		opt.Subsystem = file.OptionalHeader.(*pe.OptionalHeader64).Subsystem
		opt.DllCharacteristics = file.OptionalHeader.(*pe.OptionalHeader64).DllCharacteristics
		opt.SizeOfStackReserve = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfStackReserve
		opt.SizeOfStackCommit = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfStackCommit
		opt.SizeOfHeapReserve = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeapReserve
		opt.SizeOfHeapCommit = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeapCommit
		opt.LoaderFlags = file.OptionalHeader.(*pe.OptionalHeader64).LoaderFlags
		opt.NumberOfRvaAndSizes = file.OptionalHeader.(*pe.OptionalHeader64).NumberOfRvaAndSizes

		for i, j := range file.OptionalHeader.(*pe.OptionalHeader32).DataDirectory {
			opt.DataDirectory[i].VirtualAddress = j.VirtualAddress
			opt.DataDirectory[i].Size = j.Size
		}

	} else if file.Machine == 0x14C {
		opt.Magic = file.OptionalHeader.(*pe.OptionalHeader32).Magic
		opt.MajorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorLinkerVersion
		opt.MinorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorLinkerVersion
		opt.SizeOfCode = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfCode
		opt.SizeOfInitializedData = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfInitializedData
		opt.SizeOfUninitializedData = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfUninitializedData
		opt.AddressOfEntryPoint = file.OptionalHeader.(*pe.OptionalHeader32).AddressOfEntryPoint
		opt.BaseOfCode = file.OptionalHeader.(*pe.OptionalHeader32).BaseOfCode
		opt.ImageBase = uint64(file.OptionalHeader.(*pe.OptionalHeader32).ImageBase)
		opt.SectionAlignment = file.OptionalHeader.(*pe.OptionalHeader32).SectionAlignment
		opt.FileAlignment = file.OptionalHeader.(*pe.OptionalHeader32).FileAlignment
		opt.MajorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorOperatingSystemVersion
		opt.MinorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorOperatingSystemVersion
		opt.MajorImageVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorImageVersion
		opt.MinorImageVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorImageVersion
		opt.MajorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorSubsystemVersion
		opt.MinorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorSubsystemVersion
		opt.Win32VersionValue = file.OptionalHeader.(*pe.OptionalHeader32).Win32VersionValue
		opt.SizeOfImage = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfImage
		opt.SizeOfHeaders = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeaders
		opt.CheckSum = file.OptionalHeader.(*pe.OptionalHeader32).CheckSum
		opt.Subsystem = file.OptionalHeader.(*pe.OptionalHeader32).Subsystem
		opt.DllCharacteristics = file.OptionalHeader.(*pe.OptionalHeader32).DllCharacteristics
		opt.SizeOfStackReserve = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfStackReserve)
		opt.SizeOfStackCommit = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfStackCommit)
		opt.SizeOfHeapReserve = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeapReserve)
		opt.SizeOfHeapCommit = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeapCommit)
		opt.LoaderFlags = file.OptionalHeader.(*pe.OptionalHeader32).LoaderFlags
		opt.NumberOfRvaAndSizes = file.OptionalHeader.(*pe.OptionalHeader32).NumberOfRvaAndSizes

		for i, j := range file.OptionalHeader.(*pe.OptionalHeader64).DataDirectory {
			opt.DataDirectory[i].VirtualAddress = j.VirtualAddress
			opt.DataDirectory[i].Size = j.Size
		}
	}

	return opt

}
<commit_msg>Bug fix for scanning data directories<commit_after>package mape

import (
	"debug/pe"
)

// ConvertOptionalHeader stores a given
// 32 bit OptionalHeader struct inside a 64 bit OptionalHeader
func ConvertOptionalHeader(file *pe.File) OptionalHeader {

	var opt OptionalHeader

	if file.Machine == 0x8664 {
		opt.Magic = file.OptionalHeader.(*pe.OptionalHeader64).Magic
		opt.MajorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorLinkerVersion
		opt.MinorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorLinkerVersion
		opt.SizeOfCode = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfCode
		opt.SizeOfInitializedData = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfInitializedData
		opt.SizeOfUninitializedData = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfUninitializedData
		opt.AddressOfEntryPoint = file.OptionalHeader.(*pe.OptionalHeader64).AddressOfEntryPoint
		opt.BaseOfCode = file.OptionalHeader.(*pe.OptionalHeader64).BaseOfCode
		opt.ImageBase = file.OptionalHeader.(*pe.OptionalHeader64).ImageBase
		opt.SectionAlignment = file.OptionalHeader.(*pe.OptionalHeader64).SectionAlignment
		opt.FileAlignment = file.OptionalHeader.(*pe.OptionalHeader64).FileAlignment
		opt.MajorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorOperatingSystemVersion
		opt.MinorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorOperatingSystemVersion
		opt.MajorImageVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorImageVersion
		opt.MinorImageVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorImageVersion
		opt.MajorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MajorSubsystemVersion
		opt.MinorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader64).MinorSubsystemVersion
		opt.Win32VersionValue = file.OptionalHeader.(*pe.OptionalHeader64).Win32VersionValue
		opt.SizeOfImage = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfImage
		opt.SizeOfHeaders = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeaders
		opt.CheckSum = file.OptionalHeader.(*pe.OptionalHeader64).CheckSum
		opt.Subsystem = file.OptionalHeader.(*pe.OptionalHeader64).Subsystem
		opt.DllCharacteristics = file.OptionalHeader.(*pe.OptionalHeader64).DllCharacteristics
		opt.SizeOfStackReserve = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfStackReserve
		opt.SizeOfStackCommit = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfStackCommit
		opt.SizeOfHeapReserve = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeapReserve
		opt.SizeOfHeapCommit = file.OptionalHeader.(*pe.OptionalHeader64).SizeOfHeapCommit
		opt.LoaderFlags = file.OptionalHeader.(*pe.OptionalHeader64).LoaderFlags
		opt.NumberOfRvaAndSizes = file.OptionalHeader.(*pe.OptionalHeader64).NumberOfRvaAndSizes

		for i, j := range file.OptionalHeader.(*pe.OptionalHeader64).DataDirectory {
			opt.DataDirectory[i].VirtualAddress = j.VirtualAddress
			opt.DataDirectory[i].Size = j.Size
		}

	} else if file.Machine == 0x14C {
		opt.Magic = file.OptionalHeader.(*pe.OptionalHeader32).Magic
		opt.MajorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorLinkerVersion
		opt.MinorLinkerVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorLinkerVersion
		opt.SizeOfCode = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfCode
		opt.SizeOfInitializedData = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfInitializedData
		opt.SizeOfUninitializedData = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfUninitializedData
		opt.AddressOfEntryPoint = file.OptionalHeader.(*pe.OptionalHeader32).AddressOfEntryPoint
		opt.BaseOfCode = file.OptionalHeader.(*pe.OptionalHeader32).BaseOfCode
		opt.ImageBase = uint64(file.OptionalHeader.(*pe.OptionalHeader32).ImageBase)
		opt.SectionAlignment = file.OptionalHeader.(*pe.OptionalHeader32).SectionAlignment
		opt.FileAlignment = file.OptionalHeader.(*pe.OptionalHeader32).FileAlignment
		opt.MajorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorOperatingSystemVersion
		opt.MinorOperatingSystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorOperatingSystemVersion
		opt.MajorImageVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorImageVersion
		opt.MinorImageVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorImageVersion
		opt.MajorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MajorSubsystemVersion
		opt.MinorSubsystemVersion = file.OptionalHeader.(*pe.OptionalHeader32).MinorSubsystemVersion
		opt.Win32VersionValue = file.OptionalHeader.(*pe.OptionalHeader32).Win32VersionValue
		opt.SizeOfImage = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfImage
		opt.SizeOfHeaders = file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeaders
		opt.CheckSum = file.OptionalHeader.(*pe.OptionalHeader32).CheckSum
		opt.Subsystem = file.OptionalHeader.(*pe.OptionalHeader32).Subsystem
		opt.DllCharacteristics = file.OptionalHeader.(*pe.OptionalHeader32).DllCharacteristics
		opt.SizeOfStackReserve = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfStackReserve)
		opt.SizeOfStackCommit = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfStackCommit)
		opt.SizeOfHeapReserve = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeapReserve)
		opt.SizeOfHeapCommit = uint64(file.OptionalHeader.(*pe.OptionalHeader32).SizeOfHeapCommit)
		opt.LoaderFlags = file.OptionalHeader.(*pe.OptionalHeader32).LoaderFlags
		opt.NumberOfRvaAndSizes = file.OptionalHeader.(*pe.OptionalHeader32).NumberOfRvaAndSizes

		for i, j := range file.OptionalHeader.(*pe.OptionalHeader32).DataDirectory {
			opt.DataDirectory[i].VirtualAddress = j.VirtualAddress
			opt.DataDirectory[i].Size = j.Size
		}
	}

	return opt

}
<|endoftext|>


<commit_before>//Copyright 2012, Daniel Morsing
//For licensing information, See the LICENSE file

package main

import (
	"bytes"
	"errors"
	_ "expvar"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"net/textproto"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/DanielMorsing/gonzbee/nntp"
	"github.com/DanielMorsing/gonzbee/nzb"
	"github.com/DanielMorsing/gonzbee/par2"
	"github.com/DanielMorsing/gonzbee/yenc"
)

var (
	rm       = flag.Bool("rm", false, "Remove the nzb file after downloading")
	saveDir  = flag.String("d", "", "Save to this directory")
	par      = flag.Bool("par", false, "only download par2 files")
	profAddr = flag.String("prof", "", "address to open profiling server on")
)

var extStrip = regexp.MustCompile(`\.nzb$`)

var existErr = errors.New("file exists")

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "No NZB files given")
		os.Exit(1)
	}

	if *profAddr != "" {
		laddr, err := net.Listen("tcp", *profAddr)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		go func() {
			log.Fatalln(http.Serve(laddr, nil))
		}()
	}

	for _, path := range flag.Args() {
		file, err := os.Open(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		nzb, err := nzb.Parse(file)
		file.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		err = downloadNzb(nzb, extStrip.ReplaceAllString(path, ""))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		if *rm {
			err = os.Remove(path)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	filewg.Wait()
}

// download all the files contained in an nzb,
func downloadNzb(nzbFile *nzb.Nzb, dir string) error {
	if *saveDir != "" {
		dir = *saveDir
	}
	err := os.Mkdir(dir, os.ModePerm)
	// if the directory already exist, assume that it's an old download that was canceled
	// and restarted.
	if err != nil && !os.IsExist(err) {
		return err
	}
	parfiles := filterPars(nzbFile)
	// first download the parfiles
	for file, _ := range parfiles {
		err = downloadFile(dir, file)
		if err == existErr {
			continue
		} else if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	if *par {
		for _, pfiles := range parfiles {
			for _, f := range pfiles {
				err := downloadFile(dir, f.file)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}
		}
		return nil
	}
	// download the rest of the files.
	for _, file := range nzbFile.File {
		err = downloadFile(dir, file)
		if err == existErr {
			continue
		} else if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	// create a list of files downloaded
	var paths []string
	for _, file := range nzbFile.File {
		filename := file.Subject.Filename()
		path := filepath.Join(dir, filename)
		paths = append(paths, path)
	}
	filewg.Wait()
	for fp, set := range parfiles {
		var n int
		paths, n, err = verifyPar(fp, dir, paths)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		files := selectPars(set, n)

		for _, file := range files {
			err = downloadFile(dir, file)
			if err == existErr {
				continue
			} else if err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	return nil
}

func verifyPar(fp *nzb.File, dir string, paths []string) ([]string, int, error) {
	filename := fp.Subject.Filename()
	path := filepath.Join(dir, filename)
	f, err := os.Open(path)
	if err != nil {
		return paths, 0, err
	}
	defer f.Close()
	fset := par2.NewFileset(f)
	if !fset.CanVerify() {
		return paths, 0, nil
	}
	pathSet := make(map[string]bool)
	for _, s := range paths {
		pathSet[s] = true
	}
	matches, blockNeeded := fset.Verify(paths)
	for _, fm := range matches {
		if pathSet[fm.Path] {
			delete(pathSet, fm.Path)
		}
	}
	retPaths := make([]string, 0, len(pathSet))
	for s := range pathSet {
		retPaths = append(retPaths, s)
	}
	return retPaths, blockNeeded, nil
}

// download a single file contained in an nzb.
func downloadFile(dir string, nzbfile *nzb.File) error {
	file, err := newFile(dir, nzbfile)
	if err != nil {
		return err
	}
	for _, f := range nzbfile.Segments {
		c, err := getConn()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		go decodeMsg(c, file, nzbfile.Groups, f.MsgId)
	}
	return nil
}

// decodes an nntp message and writes it to a section of the file.
func decodeMsg(c *nntp.Conn, f *file, groups []string, MsgId string) {
	var err error
	defer f.Done()
	rc, err := c.GetMessage(MsgId)
	if err != nil {
		fmt.Fprintln(os.Stderr, "nntp error getting", MsgId, ":", err)
		if _, ok := err.(*textproto.Error); ok {
			putConn(c)
		} else {
			putBroken(c)
		}
		return
	}
	putConn(c)

	yread, err := yenc.NewPart(bytes.NewBuffer(rc))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	wr := f.WriterAt(yread.Begin)
	_, err = io.Copy(wr, yread)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// waitgroup that keeps track if there are any files being downloaded.
var filewg sync.WaitGroup

type file struct {
	name      string
	path      string
	file      *os.File
	partsLeft int
	mu        sync.Mutex
}

func newFile(dirname string, nzbfile *nzb.File) (*file, error) {
	filename := nzbfile.Subject.Filename()
	if filename == "" {
		return nil, errors.New("bad subject")
	}

	path := filepath.Join(dirname, filename)
	if _, err := os.Stat(path); err == nil {
		return nil, existErr
	}

	temppath := path + ".gonztemp"
	f, err := os.Create(temppath)
	if err != nil {
		return nil, err
	}

	ret := &file{
		name:      filename,
		path:      path,
		partsLeft: len(nzbfile.Segments),
		file:      f,
	}
	filewg.Add(1)
	return ret, nil
}

func (f *file) WriterAt(offset int64) io.Writer {
	return &fileWriter{
		f:      f.file,
		offset: offset,
	}
}

func (f *file) Done() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.partsLeft--
	if f.partsLeft != 0 {
		return
	}
	fmt.Printf("Done downloading file %q\n", f.name)
	os.Rename(f.file.Name(), f.path)
	f.file.Close()
	filewg.Done()

}

// filewriter allows for multiple goroutines to write concurrently to
// non-overlapping sections of a file
type fileWriter struct {
	f      *os.File
	offset int64
}

func (f *fileWriter) Write(b []byte) (int, error) {
	n, err := f.f.WriteAt(b, f.offset)
	f.offset += int64(n)
	return n, err
}
<commit_msg>move files if they match a par2 match<commit_after>//Copyright 2012, Daniel Morsing
//For licensing information, See the LICENSE file

package main

import (
	"bytes"
	"errors"
	_ "expvar"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"net/textproto"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/DanielMorsing/gonzbee/nntp"
	"github.com/DanielMorsing/gonzbee/nzb"
	"github.com/DanielMorsing/gonzbee/par2"
	"github.com/DanielMorsing/gonzbee/yenc"
)

var (
	rm       = flag.Bool("rm", false, "Remove the nzb file after downloading")
	saveDir  = flag.String("d", "", "Save to this directory")
	par      = flag.Bool("par", false, "only download par2 files")
	profAddr = flag.String("prof", "", "address to open profiling server on")
)

var extStrip = regexp.MustCompile(`\.nzb$`)

var existErr = errors.New("file exists")

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "No NZB files given")
		os.Exit(1)
	}

	if *profAddr != "" {
		laddr, err := net.Listen("tcp", *profAddr)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		go func() {
			log.Fatalln(http.Serve(laddr, nil))
		}()
	}

	for _, path := range flag.Args() {
		file, err := os.Open(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		nzb, err := nzb.Parse(file)
		file.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		err = downloadNzb(nzb, extStrip.ReplaceAllString(path, ""))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		if *rm {
			err = os.Remove(path)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	filewg.Wait()
}

// download all the files contained in an nzb,
func downloadNzb(nzbFile *nzb.Nzb, dir string) error {
	if *saveDir != "" {
		dir = *saveDir
	}
	err := os.Mkdir(dir, os.ModePerm)
	// if the directory already exist, assume that it's an old download that was canceled
	// and restarted.
	if err != nil && !os.IsExist(err) {
		return err
	}
	parfiles := filterPars(nzbFile)
	// first download the parfiles
	for file, _ := range parfiles {
		err = downloadFile(dir, file)
		if err == existErr {
			continue
		} else if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	if *par {
		for _, pfiles := range parfiles {
			for _, f := range pfiles {
				err := downloadFile(dir, f.file)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}
		}
		return nil
	}
	// download the rest of the files.
	for _, file := range nzbFile.File {
		err = downloadFile(dir, file)
		if err == existErr {
			continue
		} else if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	// create a list of files downloaded
	var paths []string
	for _, file := range nzbFile.File {
		filename := file.Subject.Filename()
		path := filepath.Join(dir, filename)
		paths = append(paths, path)
	}
	filewg.Wait()
	for fp, set := range parfiles {
		var n int
		paths, n, err = verifyPar(fp, dir, paths)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		files := selectPars(set, n)

		for _, file := range files {
			err = downloadFile(dir, file)
			if err == existErr {
				continue
			} else if err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	return nil
}

func verifyPar(fp *nzb.File, dir string, paths []string) ([]string, int, error) {
	filename := fp.Subject.Filename()
	path := filepath.Join(dir, filename)
	f, err := os.Open(path)
	if err != nil {
		return paths, 0, err
	}
	defer f.Close()
	fset := par2.NewFileset(f)
	if !fset.CanVerify() {
		return paths, 0, nil
	}
	pathSet := make(map[string]bool)
	for _, s := range paths {
		pathSet[s] = true
	}
	matches, blockNeeded := fset.Verify(paths)
	for _, fm := range matches {
		if pathSet[fm.Path] {
			delete(pathSet, fm.Path)
			par2path := filepath.Join(dir, fm.File.Name)
			if par2path != fm.Path {
				err := os.Rename(fm.Path, par2path)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}
		}
	}
	retPaths := make([]string, 0, len(pathSet))
	for s := range pathSet {
		retPaths = append(retPaths, s)
	}
	return retPaths, blockNeeded, nil
}

// download a single file contained in an nzb.
func downloadFile(dir string, nzbfile *nzb.File) error {
	file, err := newFile(dir, nzbfile)
	if err != nil {
		return err
	}
	for _, f := range nzbfile.Segments {
		c, err := getConn()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		go decodeMsg(c, file, nzbfile.Groups, f.MsgId)
	}
	return nil
}

// decodes an nntp message and writes it to a section of the file.
func decodeMsg(c *nntp.Conn, f *file, groups []string, MsgId string) {
	var err error
	defer f.Done()
	rc, err := c.GetMessage(MsgId)
	if err != nil {
		fmt.Fprintln(os.Stderr, "nntp error getting", MsgId, ":", err)
		if _, ok := err.(*textproto.Error); ok {
			putConn(c)
		} else {
			putBroken(c)
		}
		return
	}
	putConn(c)

	yread, err := yenc.NewPart(bytes.NewBuffer(rc))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	wr := f.WriterAt(yread.Begin)
	_, err = io.Copy(wr, yread)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// waitgroup that keeps track if there are any files being downloaded.
var filewg sync.WaitGroup

type file struct {
	name      string
	path      string
	file      *os.File
	partsLeft int
	mu        sync.Mutex
}

func newFile(dirname string, nzbfile *nzb.File) (*file, error) {
	filename := nzbfile.Subject.Filename()
	if filename == "" {
		return nil, errors.New("bad subject")
	}

	path := filepath.Join(dirname, filename)
	if _, err := os.Stat(path); err == nil {
		return nil, existErr
	}

	temppath := path + ".gonztemp"
	f, err := os.Create(temppath)
	if err != nil {
		return nil, err
	}

	ret := &file{
		name:      filename,
		path:      path,
		partsLeft: len(nzbfile.Segments),
		file:      f,
	}
	filewg.Add(1)
	return ret, nil
}

func (f *file) WriterAt(offset int64) io.Writer {
	return &fileWriter{
		f:      f.file,
		offset: offset,
	}
}

func (f *file) Done() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.partsLeft--
	if f.partsLeft != 0 {
		return
	}
	fmt.Printf("Done downloading file %q\n", f.name)
	os.Rename(f.file.Name(), f.path)
	f.file.Close()
	filewg.Done()

}

// filewriter allows for multiple goroutines to write concurrently to
// non-overlapping sections of a file
type fileWriter struct {
	f      *os.File
	offset int64
}

func (f *fileWriter) Write(b []byte) (int, error) {
	n, err := f.f.WriteAt(b, f.offset)
	f.offset += int64(n)
	return n, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package relaxng

import (
	"encoding/xml"
	"fmt"
)

func ParseGrammar(buf []byte) (*Grammar, error) {
	g := &Grammar{}
	err := xml.Unmarshal(buf, g)
	return g, err
}

func (g *Grammar) String() string {
	data, err := xml.MarshalIndent(g, "", "\t")
	if err != nil {
		panic(err)
	}
	return string(data)
}

/*
The simplified RelaxNG Grammar as specified in
http://relaxng.org/spec-20011203.html
	grammar	  		::=  <grammar> <start> top </start> define* </grammar>
	define	  		::=  <define name="NCName"> <element> nameClass top </element> </define>
	top	  			::=  <notAllowed/>
					| Pattern
	Pattern	  		::=  <empty/>
					| Pattern
	Pattern	  		::=  <text/>
					| <data type="NCName" datatypeLibrary="anyURI"> param* [exceptPattern] </data>
					| <value datatypeLibrary="anyURI" type="NCName" ns="string"> string </value>
					| <list> Pattern </list>
					| <attribute> nameClass Pattern </attribute>
					| <ref name="NCName"/>
					| <oneOrMore> Pattern </oneOrMore>
					| <choice> Pattern Pattern </choice>
					| <group> Pattern Pattern </group>
					| <interleave> Pattern Pattern </interleave>
	param	  		::=  <param name="NCName"> string </param>
	exceptPattern	::=  <except> Pattern </except>
	nameClass	  	::=  <anyName> [exceptNameClass] </anyName>
					| <nsName ns="string"> [exceptNameClass] </nsName>
					| <name ns="string"> NCName </name>
					| <choice> nameClass nameClass </choice>
	exceptNameClass	::= <except> nameClass </except>
*/
type Grammar struct {
	Start  *NameOrPattern `xml:"start"`
	Define []Define       `xml:"define"`
}

type Define struct {
	Name    string         `xml:"name,attr"`
	Element *NameOrPattern `xml:"element"`
}

type NameOrPattern struct {
	NotAllowed *Empty         `xml:"notAllowed"`
	Empty      *Empty         `xml:"empty"`
	Text       *Empty         `xml:"text"`
	Data       *Data          `xml:"data"`
	Value      *Value         `xml:"value"`
	List       *NameOrPattern `xml:"list"`
	Attribute  *NameOrPattern `xml:"attribute"`
	Ref        *Ref           `xml:"ref"`
	OneOrMore  *NameOrPattern `xml:"oneOrMore"`
	Choice     *Pair          `xml:"choice"`
	Group      *Pair          `xml:"group"`
	Interleave *Pair          `xml:"interleave"`

	AnyName *AnyNameClass  `xml:"anyName"`
	NsName  *NsNameClass   `xml:"nsName"`
	Name    *NameNameClass `xml:"name"`
}

// func (this *NameOrPattern) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// 	name := start.Name.Local
// 	_, isNameOrPattern := nameOrPattern[name]
// 	if isNameOrPattern {
// 		return this.unmarshalXML(d, start)
// 	}
// 	var s xml.StartElement
// 	var ok bool
// 	for !ok {
// 		t, err := d.Token()
// 		if err != nil {
// 			return err
// 		}
// 		s, ok = t.(xml.StartElement)
// 	}
// 	if err := this.unmarshalXML(d, s); err != nil {
// 		return err
// 	}
// 	return d.Skip()
// }

var nameOrPattern = map[string]struct{}{
	"notAllowed": struct{}{},
	"empty":      struct{}{},
	"text":       struct{}{},
	"data":       struct{}{},
	"value":      struct{}{},
	"list":       struct{}{},
	"attribute":  struct{}{},
	"ref":        struct{}{},
	"oneOrMore":  struct{}{},
	"choice":     struct{}{},
	"group":      struct{}{},
	"interleave": struct{}{},
	"anyName":    struct{}{},
	"nsName":     struct{}{},
	"name":       struct{}{},
}

func (this *NameOrPattern) unmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	switch start.Name.Local {
	case "notAllowed":
		this.NotAllowed = &Empty{}
		return d.DecodeElement(this.NotAllowed, &start)
	case "empty":
		this.Empty = &Empty{}
		return d.DecodeElement(this.Empty, &start)
	case "text":
		this.Text = &Empty{}
		return d.DecodeElement(this.Text, &start)
	case "data":
		this.Data = &Data{}
		return d.DecodeElement(this.Data, &start)
	case "value":
		this.Value = &Value{}
		return d.DecodeElement(this.Value, &start)
	case "list":
		this.List = &NameOrPattern{}
		return d.DecodeElement(this.List, &start)
	case "attribute":
		this.Attribute = &NameOrPattern{}
		return d.DecodeElement(this.Attribute, &start)
	case "ref":
		this.Ref = &Ref{}
		return d.DecodeElement(this.Ref, &start)
	case "oneOrMore":
		this.OneOrMore = &NameOrPattern{}
		return d.DecodeElement(this.OneOrMore, &start)
	case "choice":
		this.Choice = &Pair{}
		return d.DecodeElement(this.Choice, &start)
	case "group":
		this.Group = &Pair{}
		return d.DecodeElement(this.Group, &start)
	case "interleave":
		this.Interleave = &Pair{}
		return d.DecodeElement(this.Interleave, &start)
	case "anyName":
		this.AnyName = &AnyNameClass{}
		return d.DecodeElement(this.AnyName, &start)
	case "nsName":
		this.NsName = &NsNameClass{}
		return d.DecodeElement(this.NsName, &start)
	case "name":
		this.Name = &NameNameClass{}
		return d.DecodeElement(this.Name, &start)
	}
	return fmt.Errorf("unknown pattern " + start.Name.Local)
}

type Empty struct{}

type Data struct {
	Type            string         `xml:"type,attr"`
	DatatypeLibrary string         `xml:"datatypeLibrary,attr"`
	Param           []Param        `xml:"param"`
	Except          *NameOrPattern `xml:"except"`
}

// func (this *Data) unmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// 	switch start.Name.Local {
// 	case "param":
// 		p := &Param{}
// 		if err := d.DecodeElement(p, &start); err != nil {
// 			return err
// 		}
// 		this.Param = append(this.Param, *p)
// 	case "except":
// 		this.Except = &NameOrPattern{}
// 		return d.DecodeElement(this.Except, &start)
// 	}
// 	panic("unknown data " + start.Name.Local)
// }

// func (this *Data) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// 	name := start.Name.Local
// 	if name != "data" {
// 		panic("wtf")
// 	}
// 	for _, a := range start.Attr {
// 		if a.Name.Local == "type" {
// 			this.Type = a.Value
// 		} else if a.Name.Local == "datatypeLibrary" {
// 			this.DatatypeLibrary = a.Value
// 		}
// 	}
// 	for {
// 		t, err := d.Token()
// 		if err != nil {
// 			if err == io.EOF {
// 				return nil
// 			}
// 			return err
// 		}
// 		s, ok := t.(xml.StartElement)
// 		if !ok {
// 			continue
// 		}
// 		if err := this.unmarshalXML(d, s); err != nil {
// 			return err
// 		}
// 	}
// 	panic("unreachable")
// }

type Value struct {
	DatatypeLibrary string `xml:"datatypeLibrary,attr"`
	Type            string `xml:"type,attr"`
	Ns              string `xml:"ns,attr"`
	Text            string `xml:",chardata"`
}

type Ref struct {
	Name string `xml:"name,attr"`
}

type Pair struct {
	Left  *NameOrPattern
	Right *NameOrPattern
}

func skipToStart(d *xml.Decoder) (*xml.StartElement, error) {
	for {
		t, err := d.Token()
		if err != nil {
			return nil, err
		}
		s, ok := t.(xml.StartElement)
		if ok {
			return &s, nil
		}
	}
}

func (this *Pair) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	fmt.Printf("unmarshaling pair %s\n", start.Name.Local)
	s, err := skipToStart(d)
	if err != nil {
		return err
	}
	fmt.Printf("\t %s\n", s.Name.Local)
	this.Left = &NameOrPattern{}
	if err := this.Left.unmarshalXML(d, *s); err != nil {
		return err
	}
	s, err = skipToStart(d)
	if err != nil {
		return err
	}
	fmt.Printf("\t %s\n", s.Name.Local)
	this.Right = &NameOrPattern{}
	if err := this.Right.unmarshalXML(d, *s); err != nil {
		return err
	}
	return d.Skip()
}

type Param struct {
	Name string `xml:",attr"`
	Text string `xml:",chardata"`
}

type AnyNameClass struct {
	Except *ExceptNameClass `xml:"except"`
}

type NsNameClass struct {
	Ns     string           `xml:"ns,attr"`
	Except *ExceptNameClass `xml:"except"`
}

type NameNameClass struct {
	Ns   string `xml:"ns,attr"`
	Text string `xml:",chardata"`
}

type ExceptNameClass struct {
	Text string `xml:",chardata"`
}
<commit_msg>better marshaling<commit_after>// Copyright 2015 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package relaxng

import (
	"encoding/xml"
	"fmt"
	"reflect"
)

func ParseGrammar(buf []byte) (*Grammar, error) {
	g := &Grammar{}
	err := xml.Unmarshal(buf, g)
	return g, err
}

func (g *Grammar) String() string {
	data, err := xml.MarshalIndent(g, "", "\t")
	if err != nil {
		panic(err)
	}
	return string(data)
}

/*
The simplified RelaxNG Grammar as specified in
http://relaxng.org/spec-20011203.html
	grammar	  		::=  <grammar> <start> top </start> define* </grammar>
	define	  		::=  <define name="NCName"> <element> nameClass top </element> </define>
	top	  			::=  <notAllowed/>
					| Pattern
	Pattern	  		::=  <empty/>
					| Pattern
	Pattern	  		::=  <text/>
					| <data type="NCName" datatypeLibrary="anyURI"> param* [exceptPattern] </data>
					| <value datatypeLibrary="anyURI" type="NCName" ns="string"> string </value>
					| <list> Pattern </list>
					| <attribute> nameClass Pattern </attribute>
					| <ref name="NCName"/>
					| <oneOrMore> Pattern </oneOrMore>
					| <choice> Pattern Pattern </choice>
					| <group> Pattern Pattern </group>
					| <interleave> Pattern Pattern </interleave>
	param	  		::=  <param name="NCName"> string </param>
	exceptPattern	::=  <except> Pattern </except>
	nameClass	  	::=  <anyName> [exceptNameClass] </anyName>
					| <nsName ns="string"> [exceptNameClass] </nsName>
					| <name ns="string"> NCName </name>
					| <choice> nameClass nameClass </choice>
	exceptNameClass	::= <except> nameClass </except>
*/
type Grammar struct {
	Start  *NameOrPattern `xml:"start"`
	Define []Define       `xml:"define"`
}

type Define struct {
	Name    string         `xml:"name,attr"`
	Element *NameOrPattern `xml:"element"`
}

type NameOrPattern struct {
	NotAllowed *NotAllowed `xml:"notAllowed"`
	Empty      *Empty      `xml:"empty"`
	Text       *Text       `xml:"text"`
	Data       *Data       `xml:"data"`
	Value      *Value      `xml:"value"`
	List       *List       `xml:"list"`
	Attribute  *Attribute  `xml:"attribute"`
	Ref        *Ref        `xml:"ref"`
	OneOrMore  *OneOrMore  `xml:"oneOrMore"`
	Choice     *Pair       `xml:"choice"`
	Group      *Pair       `xml:"group"`
	Interleave *Pair       `xml:"interleave"`

	AnyName *AnyNameClass  `xml:"anyName"`
	NsName  *NsNameClass   `xml:"nsName"`
	Name    *NameNameClass `xml:"name"`
}

func (this *NameOrPattern) unmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	t := reflect.TypeOf(this).Elem()
	numFields := t.NumField()
	v := reflect.ValueOf(this).Elem()
	for i := 0; i < numFields; i++ {
		f := t.Field(i)
		xmlTag := f.Tag.Get("xml")
		if xmlTag == start.Name.Local {
			n := reflect.New(f.Type)
			err := d.DecodeElement(n.Interface(), &start)
			if err != nil {
				return err
			}
			v.Field(i).Set(n.Elem())
			return nil
		}
	}
	return fmt.Errorf("unknown pattern " + start.Name.Local)
}

func (this *NameOrPattern) marshalXML(e *xml.Encoder, start xml.StartElement) error {
	v := reflect.ValueOf(this).Elem()
	numFields := v.NumField()
	for i := 0; i < numFields; i++ {
		if !v.Field(i).IsNil() {
			return e.Encode(v.Field(i).Interface())
		}
	}
	return fmt.Errorf("unset pattern")
}

type NotAllowed struct {
	XMLName xml.Name `xml:"notAllowed"`
}

type Empty struct {
	XMLName xml.Name `xml:"empty"`
}

type Text struct {
	XMLName xml.Name `xml:"text"`
}

type Data struct {
	XMLName         xml.Name       `xml:"data"`
	Type            string         `xml:"type,attr"`
	DatatypeLibrary string         `xml:"datatypeLibrary,attr"`
	Param           []Param        `xml:"param"`
	Except          *NameOrPattern `xml:"except"`
}

type Value struct {
	XMLName         xml.Name `xml:"value"`
	DatatypeLibrary string   `xml:"datatypeLibrary,attr"`
	Type            string   `xml:"type,attr"`
	Ns              string   `xml:"ns,attr"`
	Text            string   `xml:",chardata"`
}

type List struct {
	XMLName xml.Name `xml:"list"`
	*NameOrPattern
}

type Attribute struct {
	XMLName xml.Name `xml:"attribute"`
	*NameOrPattern
}

type OneOrMore struct {
	XMLName xml.Name `xml:"oneOrMore"`
	*NameOrPattern
}

type Ref struct {
	XMLName xml.Name `xml:"ref"`
	Name    string   `xml:"name,attr"`
}

type Pair struct {
	Left  *NameOrPattern
	Right *NameOrPattern
}

func skipToStart(d *xml.Decoder) (*xml.StartElement, error) {
	for {
		t, err := d.Token()
		if err != nil {
			return nil, err
		}
		s, ok := t.(xml.StartElement)
		if ok {
			return &s, nil
		}
	}
}

func (this *Pair) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	fmt.Printf("unmarshaling pair %s\n", start.Name.Local)
	s, err := skipToStart(d)
	if err != nil {
		return err
	}
	fmt.Printf("\t %s\n", s.Name.Local)
	this.Left = &NameOrPattern{}
	if err := this.Left.unmarshalXML(d, *s); err != nil {
		return err
	}
	s, err = skipToStart(d)
	if err != nil {
		return err
	}
	fmt.Printf("\t %s\n", s.Name.Local)
	this.Right = &NameOrPattern{}
	if err := this.Right.unmarshalXML(d, *s); err != nil {
		return err
	}
	return d.Skip()
}

func (this *Pair) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	if err := e.EncodeToken(start); err != nil {
		return err
	}
	fmt.Printf("marshaling pair\n")
	if err := this.Left.marshalXML(e, start); err != nil {
		return err
	}
	if err := this.Right.marshalXML(e, start); err != nil {
		return err
	}
	if err := e.EncodeToken(start.End()); err != nil {
		return err
	}
	return nil
}

type Param struct {
	Name string `xml:",attr"`
	Text string `xml:",chardata"`
}

type AnyNameClass struct {
	XMLName xml.Name         `xml:"anyName"`
	Except  *ExceptNameClass `xml:"except"`
}

type NsNameClass struct {
	XMLName xml.Name         `xml:"nsName"`
	Ns      string           `xml:"ns,attr"`
	Except  *ExceptNameClass `xml:"except"`
}

type NameNameClass struct {
	XMLName xml.Name `xml:"name"`
	Ns      string   `xml:"ns,attr"`
	Text    string   `xml:",chardata"`
}

type ExceptNameClass struct {
	Text string `xml:",chardata"`
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// 一个简单的Go语言热编译工具。
//
// gobuild会实时监控指定目录下的文件变化(重命名，删除，创建，添加)，
// 一旦触发，就会调用`go build`编译Go源文件并执行。
//
// 命令行语法:
//  gobuild [options] [dependents]
//
// options:可指定以下选项:
//  -h    显示当前帮助信息；
//  -v    显示gobuild和go程序的版本信息；
//  -o    执行编译后的可执行文件名；
//  -r    是否搜索子目录，默认为true；
//  -ext  需要监视的扩展名，默认值为"go"，区分大小写，会去掉每个扩展名的首尾空格。
//        若需要监视所有类型文件，请使用*，传递空值代表不监视任何文件；
//  -main 指定需要编译的文件，默认为""。
//
// dependents:指定其它依赖的目录，只能出现在命令的尾部。
package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/issue9/term/colors"
)

// 当前程序的版本号
const version = "0.2.7.150609"

const usage = `gobuild 用于热编译Go程序。

gobuild会实时监控指定目录下的文件变化(重命名，删除，创建，添加)，
一旦触发，就会重新编译指定的Go源文件。

命令行语法:
 gobuild [options] [dependents]

 options:
  -h    显示当前帮助信息；
  -v    显示gobuild和go程序的版本信息；
  -o    执行编译后的可执行文件名；
  -r    是否搜索子目录，默认为true；
  -ext  需要监视的扩展名，默认值为"go"，区分大小写，会去掉每个扩展名的首尾空格。
        若需要监视所有类型文件，请使用*，传递空值代表不监视任何文件；
  -main 指定需要编译的文件，默认为""。

 dependents:
  指定其它依赖的目录，只能出现在命令的尾部。

常见用法:

 gobuild
   监视当前目录，若有变动，则重新编译当前目录下的*.go文件；

 gobuild -main=main.go
   监视当前目录，若有变动，则重新编译当前目录下的main.go文件；

 gobuild -main="main.go" dir1 dir2
   监视当前目录及dir1和dir2，若有变动，则重新编译当前目录下的main.go文件；
`

func main() {
	// 检测基本环境是否满足
	if gopath := os.Getenv("GOPATH"); len(gopath) == 0 {
		log(erro, "未设置环境变量GOPATH")
		return
	}

	// 初始化flag
	var showHelp, showVersion, recursive bool
	var mainFiles, outputName, extString string

	flag.BoolVar(&showHelp, "h", false, "显示帮助信息")
	flag.BoolVar(&showVersion, "v", false, "显示版本号")
	flag.BoolVar(&recursive, "r", true, "是否查找子目录")
	flag.StringVar(&outputName, "o", "", "指定输出名称")
	flag.StringVar(&extString, "ext", "go", "指定监视的文件扩展名")
	flag.StringVar(&mainFiles, "main", "", "指定需要编译的文件")
	flag.Usage = func() {
		fmt.Println(usage)
	}

	flag.Parse()

	if showHelp {
		flag.Usage()
		return
	}

	if showVersion {
		colors.Print(colors.Stdout, colors.Green, colors.Default, "gobuild: ")
		colors.Println(colors.Stdout, colors.Default, colors.Default, version)
		colors.Print(colors.Stdout, colors.Green, colors.Default, "Go: ")
		goVersion := runtime.Version() + " " + runtime.GOOS + "/" + runtime.GOARCH
		colors.Println(colors.Stdout, colors.Default, colors.Default, goVersion)
		return
	}

	// 初始化builder实例想着的内容。

	wd, err := os.Getwd()
	if err != nil {
		log(erro, "获取当前工作目录时，发生以下错误:", err)
		return
	}

	// 初始化goCmd的参数
	args := []string{"build", "-o", outputName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	b := &builder{
		exts:      getExts(extString),
		appCmd:    getAppCmd(outputName, wd),
		goCmdArgs: args,
	}

	b.watch(recursivePath(recursive, append(flag.Args(), wd)))
	go b.build()

	done := make(chan bool)
	<-done
}

// 根据recursive值确定是否递归查找paths每个目录下的子目录。
func recursivePath(recursive bool, paths []string) []string {
	if !recursive {
		return paths
	}

	ret := []string{}

	walk := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			log(erro, "在遍历监视目录时，发生以下错误:", err)
		}

		if fi.IsDir() {
			ret = append(ret, path)
		}
		return nil
	}

	for _, path := range paths {
		if err := filepath.Walk(path, walk); err != nil {
			log(erro, "在遍历监视目录时，发生以下错误:", err)
		}
	}

	return ret
}

// 将extString分解成数组，并清理掉无用的内容，比如空字符串
func getExts(extString string) []string {
	exts := strings.Split(extString, ",")
	ret := make([]string, 0, len(exts))
	for _, ext := range exts {
		ext = strings.TrimSpace(ext)

		if len(ext) == 0 {
			continue
		}
		if ext[0] != '.' {
			ext = "." + ext
		}
		ret = append(ret, ext)
	}

	switch {
	case len(ret) == 0: // 允许不监视任意文件，便输出一信息来警告
		log(warn, "将ext设置为空值，意味着不监视任何文件的改变！")
	case len(ret) > 0:
		log(info, "系统将监视以下类型的文件:", ret)
	}

	return ret
}

func getAppCmd(outputName, wd string) *exec.Cmd {
	if len(outputName) == 0 {
		outputName = filepath.Base(wd)
	}
	if runtime.GOOS == "windows" && !strings.HasSuffix(outputName, ".exe") {
		outputName += ".exe"
	}
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = wd + string(filepath.Separator) + outputName
	}

	log(info, "输出文件为:", outputName)

	appCmd := exec.Command(outputName)
	appCmd.Stderr = os.Stderr
	appCmd.Stdout = os.Stdout
	return appCmd
}
<commit_msg>不再监视隐藏目录，比如.git等，但不影响隐藏文件<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// 一个简单的Go语言热编译工具。
//
// gobuild会实时监控指定目录下的文件变化(重命名，删除，创建，添加)，
// 一旦触发，就会调用`go build`编译Go源文件并执行。
//
// 命令行语法:
//  gobuild [options] [dependents]
//
// options:可指定以下选项:
//  -h    显示当前帮助信息；
//  -v    显示gobuild和go程序的版本信息；
//  -o    执行编译后的可执行文件名；
//  -r    是否搜索子目录，默认为true；
//  -ext  需要监视的扩展名，默认值为"go"，区分大小写，会去掉每个扩展名的首尾空格。
//        若需要监视所有类型文件，请使用*，传递空值代表不监视任何文件；
//  -main 指定需要编译的文件，默认为""。
//
// dependents:指定其它依赖的目录，只能出现在命令的尾部。
package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/issue9/term/colors"
)

// 当前程序的版本号
const version = "0.2.7.150609"

const usage = `gobuild 用于热编译Go程序。

gobuild会实时监控指定目录下的文件变化(重命名，删除，创建，添加)，
一旦触发，就会重新编译指定的Go源文件。

命令行语法:
 gobuild [options] [dependents]

 options:
  -h    显示当前帮助信息；
  -v    显示gobuild和go程序的版本信息；
  -o    执行编译后的可执行文件名；
  -r    是否搜索子目录，默认为true；
  -ext  需要监视的扩展名，默认值为"go"，区分大小写，会去掉每个扩展名的首尾空格。
        若需要监视所有类型文件，请使用*，传递空值代表不监视任何文件；
  -main 指定需要编译的文件，默认为""。

 dependents:
  指定其它依赖的目录，只能出现在命令的尾部。

常见用法:

 gobuild
   监视当前目录，若有变动，则重新编译当前目录下的*.go文件；

 gobuild -main=main.go
   监视当前目录，若有变动，则重新编译当前目录下的main.go文件；

 gobuild -main="main.go" dir1 dir2
   监视当前目录及dir1和dir2，若有变动，则重新编译当前目录下的main.go文件；
`

func main() {
	// 检测基本环境是否满足
	if gopath := os.Getenv("GOPATH"); len(gopath) == 0 {
		log(erro, "未设置环境变量GOPATH")
		return
	}

	// 初始化flag
	var showHelp, showVersion, recursive bool
	var mainFiles, outputName, extString string

	flag.BoolVar(&showHelp, "h", false, "显示帮助信息")
	flag.BoolVar(&showVersion, "v", false, "显示版本号")
	flag.BoolVar(&recursive, "r", true, "是否查找子目录")
	flag.StringVar(&outputName, "o", "", "指定输出名称")
	flag.StringVar(&extString, "ext", "go", "指定监视的文件扩展名")
	flag.StringVar(&mainFiles, "main", "", "指定需要编译的文件")
	flag.Usage = func() {
		fmt.Println(usage)
	}

	flag.Parse()

	if showHelp {
		flag.Usage()
		return
	}

	if showVersion {
		colors.Print(colors.Stdout, colors.Green, colors.Default, "gobuild: ")
		colors.Println(colors.Stdout, colors.Default, colors.Default, version)
		colors.Print(colors.Stdout, colors.Green, colors.Default, "Go: ")
		goVersion := runtime.Version() + " " + runtime.GOOS + "/" + runtime.GOARCH
		colors.Println(colors.Stdout, colors.Default, colors.Default, goVersion)
		return
	}

	// 初始化builder实例想着的内容。

	wd, err := os.Getwd()
	if err != nil {
		log(erro, "获取当前工作目录时，发生以下错误:", err)
		return
	}

	// 初始化goCmd的参数
	args := []string{"build", "-o", outputName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	b := &builder{
		exts:      getExts(extString),
		appCmd:    getAppCmd(outputName, wd),
		goCmdArgs: args,
	}

	b.watch(recursivePath(recursive, append(flag.Args(), wd)))
	go b.build()

	done := make(chan bool)
	<-done
}

// 根据recursive值确定是否递归查找paths每个目录下的子目录。
func recursivePath(recursive bool, paths []string) []string {
	if !recursive {
		return paths
	}

	ret := []string{}

	walk := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			log(erro, "在遍历监视目录时，发生以下错误:", err)
		}

		//(BUG):不能监视隐藏目录下的文件
		if fi.IsDir() && strings.Index(path, "/.") < 0 {
			ret = append(ret, path)
		}
		return nil
	}

	for _, path := range paths {
		if err := filepath.Walk(path, walk); err != nil {
			log(erro, "在遍历监视目录时，发生以下错误:", err)
		}
	}

	return ret
}

// 将extString分解成数组，并清理掉无用的内容，比如空字符串
func getExts(extString string) []string {
	exts := strings.Split(extString, ",")
	ret := make([]string, 0, len(exts))
	for _, ext := range exts {
		ext = strings.TrimSpace(ext)

		if len(ext) == 0 {
			continue
		}
		if ext[0] != '.' {
			ext = "." + ext
		}
		ret = append(ret, ext)
	}

	switch {
	case len(ret) == 0: // 允许不监视任意文件，便输出一信息来警告
		log(warn, "将ext设置为空值，意味着不监视任何文件的改变！")
	case len(ret) > 0:
		log(info, "系统将监视以下类型的文件:", ret)
	}

	return ret
}

func getAppCmd(outputName, wd string) *exec.Cmd {
	if len(outputName) == 0 {
		outputName = filepath.Base(wd)
	}
	if runtime.GOOS == "windows" && !strings.HasSuffix(outputName, ".exe") {
		outputName += ".exe"
	}
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = wd + string(filepath.Separator) + outputName
	}

	log(info, "输出文件为:", outputName)

	appCmd := exec.Command(outputName)
	appCmd.Stderr = os.Stderr
	appCmd.Stdout = os.Stdout
	return appCmd
}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package main

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/openfaas/faas/gateway/handlers"

	"github.com/openfaas/faas/gateway/metrics"
	"github.com/openfaas/faas/gateway/plugin"
	"github.com/openfaas/faas/gateway/types"
	natsHandler "github.com/openfaas/nats-queue-worker/handler"
)

func main() {

	osEnv := types.OsEnv{}
	readConfig := types.ReadConfig{}
	config := readConfig.Read(osEnv)

	log.Printf("HTTP Read Timeout: %s", config.ReadTimeout)
	log.Printf("HTTP Write Timeout: %s", config.WriteTimeout)

	if !config.UseExternalProvider() {
		log.Fatalln("As of this version of OpenFaaS, you must use external provider even for Docker Swarm.")
	}

	log.Printf("Binding to external function provider: %s", config.FunctionsProviderURL)

	metricsOptions := metrics.BuildMetricsOptions()
	metrics.RegisterMetrics(metricsOptions)

	var faasHandlers types.HandlerSet

	servicePollInterval := time.Second * 5

	reverseProxy := types.NewHTTPClientReverseProxy(config.FunctionsProviderURL, config.UpstreamTimeout)

	loggingNotifier := handlers.LoggingNotifier{}
	prometheusNotifier := handlers.PrometheusFunctionNotifier{
		Metrics: &metricsOptions,
	}
	functionNotifiers := []handlers.HTTPNotifier{loggingNotifier, prometheusNotifier}
	forwardingNotifiers := []handlers.HTTPNotifier{loggingNotifier, prometheusNotifier}

	urlResolver := handlers.SingleHostBaseURLResolver{BaseURL: config.FunctionsProviderURL.String()}
	var functionURLResolver handlers.BaseURLResolver

	if config.DirectFunctions {
		functionURLResolver = handlers.FunctionAsHostBaseURLResolver{FunctionSuffix: config.DirectFunctionsSuffix}
	} else {
		functionURLResolver = urlResolver
	}

	faasHandlers.Proxy = handlers.MakeForwardingProxyHandler(reverseProxy, functionNotifiers, functionURLResolver)

	faasHandlers.RoutelessProxy = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.ListFunctions = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.DeployFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.DeleteFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.UpdateFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	queryFunction := handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)

	alertHandler := plugin.NewExternalServiceQuery(*config.FunctionsProviderURL)
	faasHandlers.Alert = handlers.MakeAlertHandler(alertHandler)

	metrics.AttachExternalWatcher(*config.FunctionsProviderURL, metricsOptions, "func", servicePollInterval)

	if config.UseNATS() {
		log.Println("Async enabled: Using NATS Streaming.")
		natsQueue, queueErr := natsHandler.CreateNatsQueue(*config.NATSAddress, *config.NATSPort)
		if queueErr != nil {
			log.Fatalln(queueErr)
		}

		faasHandlers.QueuedProxy = handlers.MakeQueuedProxy(metricsOptions, true, natsQueue)
		faasHandlers.AsyncReport = handlers.MakeAsyncReport(metricsOptions)
	}

	prometheusQuery := metrics.NewPrometheusQuery(config.PrometheusHost, config.PrometheusPort, &http.Client{})
	listFunctions := metrics.AddMetricsHandler(faasHandlers.ListFunctions, prometheusQuery)
	faasHandlers.Proxy = handlers.MakeCallIDMiddleware(faasHandlers.Proxy)
	r := mux.NewRouter()

	// r.StrictSlash(false)	// This didn't work, so register routes twice.
	r.HandleFunc("/function/{name:[-a-zA-Z_0-9]+}", faasHandlers.Proxy)
	r.HandleFunc("/function/{name:[-a-zA-Z_0-9]+}/", faasHandlers.Proxy)

	r.HandleFunc("/system/alert", faasHandlers.Alert)

	r.HandleFunc("/system/function/{name:[-a-zA-Z_0-9]+}", queryFunction).Methods(http.MethodGet)
	r.HandleFunc("/system/functions", listFunctions).Methods(http.MethodGet)
	r.HandleFunc("/system/functions", faasHandlers.DeployFunction).Methods(http.MethodPost)
	r.HandleFunc("/system/functions", faasHandlers.DeleteFunction).Methods(http.MethodDelete)
	r.HandleFunc("/system/functions", faasHandlers.UpdateFunction).Methods(http.MethodPut)

	if faasHandlers.QueuedProxy != nil {
		r.HandleFunc("/async-function/{name:[-a-zA-Z_0-9]+}/", faasHandlers.QueuedProxy).Methods(http.MethodPost)
		r.HandleFunc("/async-function/{name:[-a-zA-Z_0-9]+}", faasHandlers.QueuedProxy).Methods(http.MethodPost)

		r.HandleFunc("/system/async-report", faasHandlers.AsyncReport)
	}

	fs := http.FileServer(http.Dir("./assets/"))

	// This URL allows access from the UI to the OpenFaaS store
	allowedCORSHost := "raw.githubusercontent.com"
	fsCORS := handlers.DecorateWithCORS(fs, allowedCORSHost)

	r.PathPrefix("/ui/").Handler(http.StripPrefix("/ui", fsCORS)).Methods(http.MethodGet)

	r.HandleFunc("/", faasHandlers.RoutelessProxy).Methods(http.MethodPost)

	metricsHandler := metrics.PrometheusHandler()
	r.Handle("/metrics", metricsHandler)
	r.Handle("/", http.RedirectHandler("/ui/", http.StatusMovedPermanently)).Methods(http.MethodGet)

	tcpPort := 8080

	s := &http.Server{
		Addr:           fmt.Sprintf(":%d", tcpPort),
		ReadTimeout:    config.ReadTimeout,
		WriteTimeout:   config.WriteTimeout,
		MaxHeaderBytes: http.DefaultMaxHeaderBytes, // 1MB - can be overridden by setting Server.MaxHeaderBytes.
		Handler:        r,
	}

	log.Fatal(s.ListenAndServe())
}
<commit_msg>Remove Prometheus tracking for forwarded requests<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package main

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/openfaas/faas/gateway/handlers"

	"github.com/openfaas/faas/gateway/metrics"
	"github.com/openfaas/faas/gateway/plugin"
	"github.com/openfaas/faas/gateway/types"
	natsHandler "github.com/openfaas/nats-queue-worker/handler"
)

func main() {

	osEnv := types.OsEnv{}
	readConfig := types.ReadConfig{}
	config := readConfig.Read(osEnv)

	log.Printf("HTTP Read Timeout: %s", config.ReadTimeout)
	log.Printf("HTTP Write Timeout: %s", config.WriteTimeout)

	if !config.UseExternalProvider() {
		log.Fatalln("As of this version of OpenFaaS, you must use external provider even for Docker Swarm.")
	}

	log.Printf("Binding to external function provider: %s", config.FunctionsProviderURL)

	metricsOptions := metrics.BuildMetricsOptions()
	metrics.RegisterMetrics(metricsOptions)

	var faasHandlers types.HandlerSet

	servicePollInterval := time.Second * 5

	reverseProxy := types.NewHTTPClientReverseProxy(config.FunctionsProviderURL, config.UpstreamTimeout)

	loggingNotifier := handlers.LoggingNotifier{}
	prometheusNotifier := handlers.PrometheusFunctionNotifier{
		Metrics: &metricsOptions,
	}
	functionNotifiers := []handlers.HTTPNotifier{loggingNotifier, prometheusNotifier}
	forwardingNotifiers := []handlers.HTTPNotifier{loggingNotifier}

	urlResolver := handlers.SingleHostBaseURLResolver{BaseURL: config.FunctionsProviderURL.String()}
	var functionURLResolver handlers.BaseURLResolver

	if config.DirectFunctions {
		functionURLResolver = handlers.FunctionAsHostBaseURLResolver{FunctionSuffix: config.DirectFunctionsSuffix}
	} else {
		functionURLResolver = urlResolver
	}

	faasHandlers.Proxy = handlers.MakeForwardingProxyHandler(reverseProxy, functionNotifiers, functionURLResolver)

	faasHandlers.RoutelessProxy = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.ListFunctions = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.DeployFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.DeleteFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	faasHandlers.UpdateFunction = handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)
	queryFunction := handlers.MakeForwardingProxyHandler(reverseProxy, forwardingNotifiers, urlResolver)

	alertHandler := plugin.NewExternalServiceQuery(*config.FunctionsProviderURL)
	faasHandlers.Alert = handlers.MakeAlertHandler(alertHandler)

	metrics.AttachExternalWatcher(*config.FunctionsProviderURL, metricsOptions, "func", servicePollInterval)

	if config.UseNATS() {
		log.Println("Async enabled: Using NATS Streaming.")
		natsQueue, queueErr := natsHandler.CreateNatsQueue(*config.NATSAddress, *config.NATSPort)
		if queueErr != nil {
			log.Fatalln(queueErr)
		}

		faasHandlers.QueuedProxy = handlers.MakeQueuedProxy(metricsOptions, true, natsQueue)
		faasHandlers.AsyncReport = handlers.MakeAsyncReport(metricsOptions)
	}

	prometheusQuery := metrics.NewPrometheusQuery(config.PrometheusHost, config.PrometheusPort, &http.Client{})
	listFunctions := metrics.AddMetricsHandler(faasHandlers.ListFunctions, prometheusQuery)
	faasHandlers.Proxy = handlers.MakeCallIDMiddleware(faasHandlers.Proxy)
	r := mux.NewRouter()

	// r.StrictSlash(false)	// This didn't work, so register routes twice.
	r.HandleFunc("/function/{name:[-a-zA-Z_0-9]+}", faasHandlers.Proxy)
	r.HandleFunc("/function/{name:[-a-zA-Z_0-9]+}/", faasHandlers.Proxy)

	r.HandleFunc("/system/alert", faasHandlers.Alert)

	r.HandleFunc("/system/function/{name:[-a-zA-Z_0-9]+}", queryFunction).Methods(http.MethodGet)
	r.HandleFunc("/system/functions", listFunctions).Methods(http.MethodGet)
	r.HandleFunc("/system/functions", faasHandlers.DeployFunction).Methods(http.MethodPost)
	r.HandleFunc("/system/functions", faasHandlers.DeleteFunction).Methods(http.MethodDelete)
	r.HandleFunc("/system/functions", faasHandlers.UpdateFunction).Methods(http.MethodPut)

	if faasHandlers.QueuedProxy != nil {
		r.HandleFunc("/async-function/{name:[-a-zA-Z_0-9]+}/", faasHandlers.QueuedProxy).Methods(http.MethodPost)
		r.HandleFunc("/async-function/{name:[-a-zA-Z_0-9]+}", faasHandlers.QueuedProxy).Methods(http.MethodPost)

		r.HandleFunc("/system/async-report", faasHandlers.AsyncReport)
	}

	fs := http.FileServer(http.Dir("./assets/"))

	// This URL allows access from the UI to the OpenFaaS store
	allowedCORSHost := "raw.githubusercontent.com"
	fsCORS := handlers.DecorateWithCORS(fs, allowedCORSHost)

	r.PathPrefix("/ui/").Handler(http.StripPrefix("/ui", fsCORS)).Methods(http.MethodGet)

	r.HandleFunc("/", faasHandlers.RoutelessProxy).Methods(http.MethodPost)

	metricsHandler := metrics.PrometheusHandler()
	r.Handle("/metrics", metricsHandler)
	r.Handle("/", http.RedirectHandler("/ui/", http.StatusMovedPermanently)).Methods(http.MethodGet)

	tcpPort := 8080

	s := &http.Server{
		Addr:           fmt.Sprintf(":%d", tcpPort),
		ReadTimeout:    config.ReadTimeout,
		WriteTimeout:   config.WriteTimeout,
		MaxHeaderBytes: http.DefaultMaxHeaderBytes, // 1MB - can be overridden by setting Server.MaxHeaderBytes.
		Handler:        r,
	}

	log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before>// This is the Goophry main application.
// It parses all commandline flags, creates instances of all necessary packages
// and triggers the creation of all worker go-routines.
package main

import (
	"./goo/basepath"
	"./goo/config"
	"./goo/output"
	"./goo/stats"
	"./goo/taskqueue"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
)

const GoophryVersion = "1.0.0-rc.1"

var (
	configFile  string
	verbose     bool
	logfile     string
	showVersion bool
)

func init() {
	flag.StringVar(&configFile, "c", "", "path to config file")
	flag.StringVar(&logfile, "l", "", "path to logfile")
	flag.BoolVar(&verbose, "v", false, "enable verbose/debugging output")
	flag.BoolVar(&showVersion, "V", false, "show version")
}

func main() {
	flag.Parse()

	if showVersion == true {
		fmt.Printf("Goophry version %s\n", GoophryVersion)
		os.Exit(0)
	}

	base, err := basepath.NewBasepath()
	if err != nil {
		log.Fatal("basepath: ", err)
	}

	// When no configuration file was passed as a flag, use the default location.
	if configFile == "" {
		configFile = base.GetPathWith("./goophry.config.json")
	}

	conf, err := config.NewConfig(configFile)
	if err != nil {
		log.Fatal("config: ", err)
	}

	out := output.NewOutput()
	out.SetDebug(verbose)
	out.SetNotifyCmd(conf.ErrorCmd)

	// When no logfile was passed as a flag but it was set in the configuration,
	// use that one instead.
	if logfile == "" && conf.Logfile != "" {
		logfile = base.GetPathWith(conf.Logfile)
	}

	if logfile != "" {
		err = out.SetLogfile(logfile)

		if err != nil {
			log.Fatal("out.SetLogfile(): ", err)
		}
	}

	sta := stats.NewStats()
	sta.SetVersion(GoophryVersion)

	tq := taskqueue.NewTaskqueue()
	tq.SetConfig(conf)
	tq.SetOutput(out)
	tq.SetStats(&sta)

	for _, ct := range conf.Tasks {
		ct.Script = base.GetPathWith(ct.Script)
		tq.CreateWorkers(ct)

		sta.InitTask(ct.Type)
	}

	// If the StatsInterface was set, start the HTTP-server for it.
	if conf.StatsInterface != "" {
		if conf.StatsPattern == "" {
			conf.StatsPattern = "/"
		}

		go func() {
			var err error

			if conf.StatsTLSCertFile != "" {
				conf.StatsTLSCertFile = base.GetPathWith(conf.StatsTLSCertFile)
				conf.StatsTLSKeyFile = base.GetPathWith(conf.StatsTLSKeyFile)

				out.Debug("Serving stats on https://" + conf.StatsInterface + conf.StatsPattern)
				err = sta.ServeHttps(conf.StatsInterface, conf.StatsPattern, conf.StatsTLSCertFile, conf.StatsTLSKeyFile)
			} else {
				out.Debug("Serving stats on http://" + conf.StatsInterface + conf.StatsPattern)
				err = sta.ServeHttp(conf.StatsInterface, conf.StatsPattern)
			}

			if err != nil {
				out.NotifyError(fmt.Sprintf("stats.ServeHttp(): %s", err))
			}
		}()
	}

	// Start another go-routine to initiate the graceful shutdown of all taskqueue-workers,
	// when the application shall be terminated.
	cc := make(chan os.Signal)
	signal.Notify(cc, os.Interrupt, os.Kill, syscall.SIGTERM)
	go func() {
		<-cc
		tq.Stop()
	}()

	tq.Wait()
}
<commit_msg>version bump<commit_after>// This is the Goophry main application.
// It parses all commandline flags, creates instances of all necessary packages
// and triggers the creation of all worker go-routines.
package main

import (
	"./goo/basepath"
	"./goo/config"
	"./goo/output"
	"./goo/stats"
	"./goo/taskqueue"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"
)

const GoophryVersion = "1.0.1"

var (
	configFile  string
	verbose     bool
	logfile     string
	showVersion bool
)

func init() {
	flag.StringVar(&configFile, "c", "", "path to config file")
	flag.StringVar(&logfile, "l", "", "path to logfile")
	flag.BoolVar(&verbose, "v", false, "enable verbose/debugging output")
	flag.BoolVar(&showVersion, "V", false, "show version")
}

func main() {
	flag.Parse()

	if showVersion == true {
		fmt.Printf("Goophry version %s\n", GoophryVersion)
		os.Exit(0)
	}

	base, err := basepath.NewBasepath()
	if err != nil {
		log.Fatal("basepath: ", err)
	}

	// When no configuration file was passed as a flag, use the default location.
	if configFile == "" {
		configFile = base.GetPathWith("./goophry.config.json")
	}

	conf, err := config.NewConfig(configFile)
	if err != nil {
		log.Fatal("config: ", err)
	}

	out := output.NewOutput()
	out.SetDebug(verbose)
	out.SetNotifyCmd(conf.ErrorCmd)

	// When no logfile was passed as a flag but it was set in the configuration,
	// use that one instead.
	if logfile == "" && conf.Logfile != "" {
		logfile = base.GetPathWith(conf.Logfile)
	}

	if logfile != "" {
		err = out.SetLogfile(logfile)

		if err != nil {
			log.Fatal("out.SetLogfile(): ", err)
		}
	}

	sta := stats.NewStats()
	sta.SetVersion(GoophryVersion)

	tq := taskqueue.NewTaskqueue()
	tq.SetConfig(conf)
	tq.SetOutput(out)
	tq.SetStats(&sta)

	for _, ct := range conf.Tasks {
		ct.Script = base.GetPathWith(ct.Script)
		tq.CreateWorkers(ct)

		sta.InitTask(ct.Type)
	}

	// If the StatsInterface was set, start the HTTP-server for it.
	if conf.StatsInterface != "" {
		if conf.StatsPattern == "" {
			conf.StatsPattern = "/"
		}

		go func() {
			var err error

			if conf.StatsTLSCertFile != "" {
				conf.StatsTLSCertFile = base.GetPathWith(conf.StatsTLSCertFile)
				conf.StatsTLSKeyFile = base.GetPathWith(conf.StatsTLSKeyFile)

				out.Debug("Serving stats on https://" + conf.StatsInterface + conf.StatsPattern)
				err = sta.ServeHttps(conf.StatsInterface, conf.StatsPattern, conf.StatsTLSCertFile, conf.StatsTLSKeyFile)
			} else {
				out.Debug("Serving stats on http://" + conf.StatsInterface + conf.StatsPattern)
				err = sta.ServeHttp(conf.StatsInterface, conf.StatsPattern)
			}

			if err != nil {
				out.NotifyError(fmt.Sprintf("stats.ServeHttp(): %s", err))
			}
		}()
	}

	// Start another go-routine to initiate the graceful shutdown of all taskqueue-workers,
	// when the application shall be terminated.
	cc := make(chan os.Signal)
	signal.Notify(cc, os.Interrupt, os.Kill, syscall.SIGTERM)
	go func() {
		<-cc
		tq.Stop()
	}()

	tq.Wait()
}
<|endoftext|>


<commit_before>package main

import "log"

func handleMatch(whData webhookData) {
	matchID := whData.MatchID
	dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
	if dbMatchData.MatchID != matchID {
		addMatch(matchID)
	}

	if dbMatchData.IsFull != true {
		matchData := getMatchData(matchID)
		matchText := makeMatchText(matchData)
		log.Println("match data is not full")
		if whData.Origin == "scanner" && dbMatchData.IsShort != true {
			log.Println("data from scanner and it's not posted yet")
			makeMatchImage(matchData, false)
			sendMatchToVk(matchID, matchText, false)
		} else if whData.Origin == "" {
			log.Println("data from parser")
			makeMatchImage(matchData, true)
			if dbMatchData.IsShort == true {
				log.Println("data was posted, now editing")
				editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			} else {
				log.Println("data was never posted, now posting")
				sendMatchToVk(matchID, matchText, true)
			}
		}
	}

}
<commit_msg>Fix logging<commit_after>package main

import (
	"log"
)

func handleMatch(whData webhookData) {
	matchID := whData.MatchID
	log.Printf("Received match %d from webhook", matchID)
	dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
	if dbMatchData.MatchID != matchID {
		addMatch(matchID)
	}

	if dbMatchData.IsFull != true {
		log.Println("Full match data wasn't posted")
		matchData := getMatchData(matchID)
		matchText := makeMatchText(matchData)
		if whData.Origin == "scanner" && dbMatchData.IsShort != true {
			log.Println("Data from scanner and it's not posted yet")
			makeMatchImage(matchData, false)
			sendMatchToVk(matchID, matchText, false)
		} else if whData.Origin == "" {
			log.Println("Data from parser")
			makeMatchImage(matchData, true)
			if dbMatchData.IsShort == true {
				log.Println("Data was posted, now editing")
				editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			} else {
				log.Println("Data never posted, now posting")
				sendMatchToVk(matchID, matchText, true)
			}
		}
	}

}
<|endoftext|>


<commit_before>package geom

import (
	"testing"

	"github.com/omniscale/imposm3/element"
	"github.com/omniscale/imposm3/geom/geos"
)

func TestLineString(t *testing.T) {
	nodes := make([]element.Node, 2)
	nodes[0] = element.Node{Lat: 0, Long: 0}
	nodes[1] = element.Node{Lat: 0, Long: 10}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := LineString(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	if geom.Length() != 10.0 {
		t.Fatal(geom.Length)
	}
}

func TestPolygon(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 0},
	}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := Polygon(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	if geom.Area() != 50.0 {
		t.Fatal(geom.Area())
	}
}

func TestPolygonNotClosed(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
	}
	g := geos.NewGeos()
	defer g.Finish()
	_, err := Polygon(g, nodes)
	if err == nil {
		t.Fatal("no error")
	}
}

func TestPolygonIntersection(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 0},
		element.Node{Lat: 0, Long: 0},
	}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := Polygon(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	result := g.Intersection(geom, g.FromWkt("LINESTRING(-10 5, 20 5)"))

	if !g.Equals(result, g.FromWkt("LINESTRING(0 5, 10 5)")) {
		t.Fatal(g.AsWkt(result))
	}
}

func BenchmarkLineString(b *testing.B) {
	size := 16
	nodes := make([]element.Node, size)
	for i := 0; i < size; i++ {
		nodes[i] = element.Node{Lat: 0, Long: float64(i)}
	}
	g := geos.NewGeos()
	defer g.Finish()

	for i := 0; i < b.N; i++ {
		LineString(g, nodes)
	}
}

func TestUnduplicateNodes(t *testing.T) {
	var nodes []element.Node

	nodes = []element.Node{
		element.Node{Lat: 0, Long: 0},
	}
	if res := unduplicateNodes(nodes); len(res) != 1 {
		t.Fatal(res)
	}
	nodes = []element.Node{
		element.Node{Lat: 47.0, Long: 80.0},
		element.Node{Lat: 47.0, Long: 80.0},
	}
	if res := unduplicateNodes(nodes); len(res) != 1 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
	}
	if res := unduplicateNodes(nodes); len(res) != 2 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 0, Long: 10},
	}
	if res := unduplicateNodes(nodes); len(res) != 4 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 10, Long: -10},
		element.Node{Lat: 10, Long: 0},
		element.Node{Lat: 0, Long: 0},
	}
	if res := unduplicateNodes(nodes); len(res) != 5 {
		t.Fatal(res)
	}

}
<commit_msg>fix test error output<commit_after>package geom

import (
	"testing"

	"github.com/omniscale/imposm3/element"
	"github.com/omniscale/imposm3/geom/geos"
)

func TestLineString(t *testing.T) {
	nodes := make([]element.Node, 2)
	nodes[0] = element.Node{Lat: 0, Long: 0}
	nodes[1] = element.Node{Lat: 0, Long: 10}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := LineString(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	if geom.Length() != 10.0 {
		t.Fatal(geom.Length())
	}
}

func TestPolygon(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 0},
	}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := Polygon(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	if geom.Area() != 50.0 {
		t.Fatal(geom.Area())
	}
}

func TestPolygonNotClosed(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
	}
	g := geos.NewGeos()
	defer g.Finish()
	_, err := Polygon(g, nodes)
	if err == nil {
		t.Fatal("no error")
	}
}

func TestPolygonIntersection(t *testing.T) {
	nodes := []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 0},
		element.Node{Lat: 0, Long: 0},
	}
	g := geos.NewGeos()
	defer g.Finish()
	geom, err := Polygon(g, nodes)
	if err != nil {
		t.Fatal(err)
	}

	result := g.Intersection(geom, g.FromWkt("LINESTRING(-10 5, 20 5)"))

	if !g.Equals(result, g.FromWkt("LINESTRING(0 5, 10 5)")) {
		t.Fatal(g.AsWkt(result))
	}
}

func BenchmarkLineString(b *testing.B) {
	size := 16
	nodes := make([]element.Node, size)
	for i := 0; i < size; i++ {
		nodes[i] = element.Node{Lat: 0, Long: float64(i)}
	}
	g := geos.NewGeos()
	defer g.Finish()

	for i := 0; i < b.N; i++ {
		LineString(g, nodes)
	}
}

func TestUnduplicateNodes(t *testing.T) {
	var nodes []element.Node

	nodes = []element.Node{
		element.Node{Lat: 0, Long: 0},
	}
	if res := unduplicateNodes(nodes); len(res) != 1 {
		t.Fatal(res)
	}
	nodes = []element.Node{
		element.Node{Lat: 47.0, Long: 80.0},
		element.Node{Lat: 47.0, Long: 80.0},
	}
	if res := unduplicateNodes(nodes); len(res) != 1 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
	}
	if res := unduplicateNodes(nodes); len(res) != 2 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 10, Long: 10},
		element.Node{Lat: 0, Long: 10},
		element.Node{Lat: 0, Long: 10},
	}
	if res := unduplicateNodes(nodes); len(res) != 4 {
		t.Fatal(res)
	}

	nodes = []element.Node{
		element.Node{Lat: 0, Long: 0},
		element.Node{Lat: 0, Long: -10},
		element.Node{Lat: 10, Long: -10},
		element.Node{Lat: 10, Long: 0},
		element.Node{Lat: 0, Long: 0},
	}
	if res := unduplicateNodes(nodes); len(res) != 5 {
		t.Fatal(res)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"
)

func handleGetFullMatchData(matchID int64, startTime time.Time) {
	currentTime := time.Now()
	hasSend := false
	retries := 0
	for currentTime.Sub(startTime) < time.Hour*24 && !hasSend {
		time.Sleep(5 * time.Minute)
		retries++
		log.Printf("Retrying getting full match %d data", matchID)
		matchData := getMatchData(matchID)
		if len(matchData.RadiantGoldAdv) != 0 && len(matchData.PicksBans) != 0 {
			log.Printf("Found full match %d data", matchID)
			matchText := makeMatchText(matchData)
			makeMatchImage(matchData, true)
			dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
			editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			hasSend = true
		}
		if retries%12 == 0 {
			forceScan(matchID)
		}
		currentTime = time.Now()
	}
}

func handleMatch(whData oDotaMatchData) {
	matchID := whData.MatchID
	log.Printf("Received match %d from webhook", matchID)
	dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
	if dbMatchData.MatchID != matchID {
		addMatch(matchID)
	}
	if dbMatchData.IsFull != true {
		log.Println("Full match data wasn't posted")
		matchData := whData
		matchText := makeMatchText(matchData)
		if whData.Origin == "scanner" {
			if dbMatchData.IsShort != true {
				log.Println("Data from scanner and it's not posted yet")
				makeMatchImage(matchData, false)
				sendMatchToVk(matchID, matchText, false)
				startTime := time.Now()
				go handleGetFullMatchData(matchID, startTime)
			} else {
				log.Println("Data from scanner and it was posted")
			}
		} else if whData.Origin == "" {
			log.Println("Data from parser")
			matchData := getMatchData(matchID)
			makeMatchImage(matchData, true)
			if dbMatchData.IsShort == true {
				log.Println("Data was posted, now editing")
				editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			} else {
				log.Println("Data never posted, now posting")
				sendMatchToVk(matchID, matchText, true)
			}
		}
	} else {
		log.Println("Full data was posted nothing to do here")
	}

}
<commit_msg>Add retry number to check if everything is OK<commit_after>package main

import (
	"log"
	"time"
)

func handleGetFullMatchData(matchID int64, startTime time.Time) {
	currentTime := time.Now()
	hasSend := false
	retries := 0
	for currentTime.Sub(startTime) < time.Hour*24 && !hasSend {
		time.Sleep(5 * time.Minute)
		retries++
		log.Printf("Retrying getting full match %d data (retry %d)", matchID, retries)
		matchData := getMatchData(matchID)
		if len(matchData.RadiantGoldAdv) != 0 && len(matchData.PicksBans) != 0 {
			log.Printf("Found full match %d data", matchID)
			matchText := makeMatchText(matchData)
			makeMatchImage(matchData, true)
			dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
			editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			hasSend = true
		}
		if retries%12 == 0 {
			forceScan(matchID)
		}
		currentTime = time.Now()
	}
}

func handleMatch(whData oDotaMatchData) {
	matchID := whData.MatchID
	log.Printf("Received match %d from webhook", matchID)
	dbMatchData, _ := dao.get(dbMatch{MatchID: matchID})
	if dbMatchData.MatchID != matchID {
		addMatch(matchID)
	}
	if dbMatchData.IsFull != true {
		log.Println("Full match data wasn't posted")
		matchData := whData
		matchText := makeMatchText(matchData)
		if whData.Origin == "scanner" {
			if dbMatchData.IsShort != true {
				log.Println("Data from scanner and it's not posted yet")
				makeMatchImage(matchData, false)
				sendMatchToVk(matchID, matchText, false)
				startTime := time.Now()
				go handleGetFullMatchData(matchID, startTime)
			} else {
				log.Println("Data from scanner and it was posted")
			}
		} else if whData.Origin == "" {
			log.Println("Data from parser")
			matchData := getMatchData(matchID)
			makeMatchImage(matchData, true)
			if dbMatchData.IsShort == true {
				log.Println("Data was posted, now editing")
				editMatchAtVk(matchID, dbMatchData.PostID, matchText)
			} else {
				log.Println("Data never posted, now posting")
				sendMatchToVk(matchID, matchText, true)
			}
		}
	} else {
		log.Println("Full data was posted nothing to do here")
	}

}
<|endoftext|>


<commit_before>package mapreduce

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"sync"
	"testing"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"

	"github.com/pachyderm/pfs/lib/btrfs"
)

func check(err error, t *testing.T) {
	if err != nil {
		t.Fatal(err)
	}
}

// TestS3 checks that the underlying S3 library is correct.
func TestS3(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping S3 integration test")
	}
	auth, err := aws.EnvAuth()
	log.Printf("auth: %#v", auth)
	if err != nil {
		log.Print(err)
		return
	}
	client := s3.New(auth, aws.USWest)
	bucket := client.Bucket("pachyderm-data")
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			inFile, err := bucket.GetReader(fmt.Sprintf("chess/file%09d", i))
			if err != nil {
				log.Print(err)
				return
			}
			io.Copy(ioutil.Discard, inFile)
			inFile.Close()
		}(i)
	}
}

// Public types: Job
// Public functions: Materialize, Map, Reduce, PrepJob, WaitJob
// Could be made private: Map, Reduce, WaitJob
// Test: make two calls to Materialize and see what happens when they overlap
// Test: spin up / spin down / start containers. check its ip address, send it a command, spin it down.
// Test: positive test of 1000 jobs (Map/Reduce combinations)

func TestMapJob(t *testing.T) {
	inRepoName := "repo_TestMapJob_input"
	outRepoName := "repo_TestMapJob_output"
	check(btrfs.Init(inRepoName), t)
	check(btrfs.Init(outRepoName), t)

	// Create a file to map over:
	f, err := btrfs.Create(fmt.Sprintf("%s/master/foo", inRepoName))
	if err != nil {
		t.Fatal(err)
	}
	f.WriteString("hello world\n")
	f.Close()

	// Commit it:
	check(btrfs.Commit(inRepoName, "commit1", "master"), t)

	// Set up the job:
	j := Job{
		Type: "map",
		// Input: "", <- how is this used in the beginning?
		Image: "jdoliner/hello-world",
		Cmd:   []string{"wc -l"},
	}
	matInfo := materializeInfo{inRepoName, outRepoName, "master", "commit2"}

	// the following shard and mod combination will always be true,
	// because for all n, (n % 1) == 0:
	shard := uint64(0)
	mod := uint64(1)
	Map(j, "TestMapJob", matInfo, shard, mod)

	// Check that the output file exists:
	wantFilename := fmt.Sprintf("%s/commit2/foo", outRepoName)
	exists, err := btrfs.FileExists(wantFilename)
	check(err, t)
	if !exists {
		t.Fatalf("expected output file to exist: %s", wantFilename)
	}

	// Check that the output file contains the expected output:
	wantContents := []byte("1\n") // i.e. the number of lines reported by `wc -l`
	gotContents, err := ioutil.ReadFile(wantFilename)
	check(err, t)
	if !bytes.Equal(wantContents, gotContents) {
		t.Fatalf("expected %s but got %s", string(wantContents), string(gotContents))
	}
}

// Test: different input directory structures with Reduce (when do you use multiple reduce calls?)
// Test: Degenerate topologies (jobs that depend on each other (deadlock expected, use a timeout), jobs that are orphaned)
// Test: Degenerate jobs themselves (e.g. nonterminating, bad exit status)
// Test: WaitJob test that it returns when a Materialize call finishes.
// Test: WaitJob test that it never returns when a pipeline never finishes.

// TODO(rw): reify errors into their own types
<commit_msg>WIP making a test work. perhaps.<commit_after>package mapreduce

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"sync"
	"testing"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"

	"github.com/pachyderm/pfs/lib/btrfs"
)

func check(err error, t *testing.T) {
	if err != nil {
		t.Fatal(err)
	}
}

// TestS3 checks that the underlying S3 library is correct.
func TestS3(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping S3 integration test")
	}
	auth, err := aws.EnvAuth()
	log.Printf("auth: %#v", auth)
	if err != nil {
		log.Print(err)
		return
	}
	client := s3.New(auth, aws.USWest)
	bucket := client.Bucket("pachyderm-data")
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			inFile, err := bucket.GetReader(fmt.Sprintf("chess/file%09d", i))
			if err != nil {
				log.Print(err)
				return
			}
			io.Copy(ioutil.Discard, inFile)
			inFile.Close()
		}(i)
	}
}

// Public types: Job
// Public functions: Materialize, Map, Reduce, PrepJob, WaitJob
// Could be made private: Map, Reduce, WaitJob
// Test: make two calls to Materialize and see what happens when they overlap
// Test: spin up / spin down / start containers. check its ip address, send it a command, spin it down.
// Test: positive test of 1000 jobs (Map/Reduce combinations)

func TestMapJob(t *testing.T) {
	inRepoName := "repo_TestMapJob_input"
	outRepoName := "repo_TestMapJob_output"
	check(btrfs.Init(inRepoName), t)
	check(btrfs.Init(outRepoName), t)

	// Create a file to map over:
	f, err := btrfs.CreateAll(fmt.Sprintf("%s/master/my_first_job/foo", inRepoName))
	if err != nil {
		t.Fatal(err)
	}
	f.WriteString("hello world\n")
	f.Close()

	// Commit it:
	check(btrfs.Commit(inRepoName, "commit1", "master"), t)

	// Set up the job:
	j := Job{
		Type: "map",
		// Input: "", <- how is this used in the beginning?
		Input: "my_first_job",
		Image: "jdoliner/hello-world",
		//Cmd:   []string{"wc -l"},
		Cmd: []string{"/go/bin/hello-world-mr"},
	}
	matInfo := materializeInfo{inRepoName, outRepoName, "master", "commit2"}

	// the following shard and mod combination will always be true,
	// because for all n, (n % 1) == 0:
	shard := uint64(0)
	mod := uint64(1)
	Map(j, "TestMapJob", matInfo, shard, mod)

	// Check that the output file exists:
	wantFilename := fmt.Sprintf("%s/commit2/my_first_job/foo", outRepoName)
	exists, err := btrfs.FileExists(wantFilename)
	check(err, t)
	if !exists {
		t.Fatalf("expected output file to exist: %s", wantFilename)
	}

	// Check that the output file contains the expected output:
	wantContents := []byte("1\n") // i.e. the number of lines reported by `wc -l`
	gotContents, err := ioutil.ReadFile(wantFilename)
	check(err, t)
	if !bytes.Equal(wantContents, gotContents) {
		t.Fatalf("expected %s but got %s", string(wantContents), string(gotContents))
	}
}

// Test: different input directory structures with Reduce (when do you use multiple reduce calls?)
// Test: Degenerate topologies (jobs that depend on each other (deadlock expected, use a timeout), jobs that are orphaned)
// Test: Degenerate jobs themselves (e.g. nonterminating, bad exit status)
// Test: WaitJob test that it returns when a Materialize call finishes.
// Test: WaitJob test that it never returns when a pipeline never finishes.

// TODO(rw): reify errors into their own types
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/importer"
	gotypes "go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *gotypes.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	err = fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<commit_msg>fix return count from Fprintf.. dozy morning<commit_after>package main

import (
	"flag"
	"fmt"
	"go/importer"
	gotypes "go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *gotypes.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	_, err = fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package codebase

import (
    "fmt"
    "log"
    "time"
)

const (
    CHANGE_STATUS     = "status"
    CHANGE_MILESTONE  = "milestone"
    CHANGE_CATEGORY   = "category"
    CHANGE_PRIORITY   = "priority"
    CHANGE_NEW_TICKET = "new ticket"

    TYPE_NEW_TICKET = "ticketing_ticket"
)

type Event struct {
    Id        int    `xml:"id"`
    Title     string `xml:"title"`
    Date      string `xml:"timestamp"`
    Type      string `xml:"type"`
    HtmlTitle string `xml:"html-title"`
    HtmlText  string `xml:"html-text"`
    UserID    int    `xml:"user-id"`
    UserEmail string `xml:"actor-email"`
    ProjectID int    `xml:"project-id"`
    Deleted   bool   `xml:"deleted"`
    Avatar    string `xml:"avatar-url"`

    Raw struct {
        TicketID         int          `xml:"number"`
        Subject          string       `xml:"subject"`
        Content          string       `xml:"content"`
        Changes          EventChanges `xml:"changes"`
        ProjectPermalink string       `xml:"project-permalink"`
    }   `xml:"raw-properties"`
}

type EventChanges struct {
    Status    []string `xml:"status-id>status-id"`
    Assignee  []string `xml:"assignee-id>assignee-id"`
    Subject   []string `xml:"subject>subject"`
    Priority  []string `xml:"priority-id>priority-id"`
    Milestone []string `xml:"milestone-id>milestone-id"`
    Category  []string `xml:"category-id>category-id"`
}

type eventQueryOptions struct {
    baseQueryOptions
    Raw   bool      `url:"raw,omitempty"`
    Since time.Time `url:"since,omitempty"`
}

type Descriptor interface {
    MapChange(field, before, after string) (description string)
}

func (c *CodeBaseAPI) Activities(since time.Time, user User, descriptor Descriptor) (events []Event) {
    type eventArray struct {
        Events []Event `xml:"event"`
    }

    queryOpts := eventQueryOptions{
        Raw:   true,
        Since: since,
    }
    queryOpts.Page = 1

    for {
        proxy := eventArray{}
        if err := c.fetchFromCodebase("activity", &proxy, queryOpts); err != nil {
            log.Fatalln("Could not fetch activities:", err)
        }

        if len(proxy.Events) == 0 {
            break
        }

        for _, event := range proxy.Events {
            if user.UserID != 0 && event.UserID != user.UserID {
                continue
            }

            if event.HasChanges(descriptor) == false {
                continue
            }

            events = append(events, event)
        }
        queryOpts.Page++
    }

    return
}

func (e *Event) HasChanges(descriptor Descriptor) bool {
    return e.Changes(descriptor) != ""
}

func (e *Event) TicketUrl(company string) string {
    return fmt.Sprintf("https://%s.codebasehq.com/projects/%s/tickets/%v", company, e.Raw.ProjectPermalink, e.Raw.TicketID)
}

func (e *Event) Day() time.Weekday {
    date, _ := time.Parse("2006-01-02 15:04:05 MST", e.Date)
    return date.Weekday()
}

func (e *Event) Changes(descriptor Descriptor) string {
    changes := ""

    type changeToMap struct {
        changeType string
        from       string
        to         string
    }

    chagnesToMap := make([]changeToMap, 0)

    if len(e.Raw.Changes.Status) == 2 {
        change := changeToMap{CHANGE_STATUS, e.Raw.Changes.Status[0], e.Raw.Changes.Status[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Milestone) == 2 {
        change := changeToMap{CHANGE_MILESTONE, e.Raw.Changes.Milestone[0], e.Raw.Changes.Milestone[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Category) == 2 {
        change := changeToMap{CHANGE_CATEGORY, e.Raw.Changes.Category[0], e.Raw.Changes.Category[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Priority) == 2 {
        change := changeToMap{CHANGE_PRIORITY, e.Raw.Changes.Priority[0], e.Raw.Changes.Priority[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if e.Type == TYPE_NEW_TICKET {
        change := changeToMap{changeType: CHANGE_NEW_TICKET}
        chagnesToMap = append(chagnesToMap, change)
    }

    for _, change := range chagnesToMap {
        changeDescription := fmt.Sprintf("%s -> %s", change.from, change.to)

        if descriptor != nil {
            changeDescription = descriptor.MapChange(change.changeType, change.from, change.to)
        }

        if changeDescription != "" {
            changes += changeDescription
        }
    }

    return changes
}
<commit_msg>Concatenate multiple change descriptions by comma<commit_after>package codebase

import (
    "fmt"
    "log"
    "strings"
    "time"
)

const (
    CHANGE_STATUS     = "status"
    CHANGE_MILESTONE  = "milestone"
    CHANGE_CATEGORY   = "category"
    CHANGE_PRIORITY   = "priority"
    CHANGE_NEW_TICKET = "new ticket"

    TYPE_NEW_TICKET = "ticketing_ticket"
)

type Event struct {
    Id        int    `xml:"id"`
    Title     string `xml:"title"`
    Date      string `xml:"timestamp"`
    Type      string `xml:"type"`
    HtmlTitle string `xml:"html-title"`
    HtmlText  string `xml:"html-text"`
    UserID    int    `xml:"user-id"`
    UserEmail string `xml:"actor-email"`
    ProjectID int    `xml:"project-id"`
    Deleted   bool   `xml:"deleted"`
    Avatar    string `xml:"avatar-url"`

    Raw struct {
        TicketID         int          `xml:"number"`
        Subject          string       `xml:"subject"`
        Content          string       `xml:"content"`
        Changes          EventChanges `xml:"changes"`
        ProjectPermalink string       `xml:"project-permalink"`
    }   `xml:"raw-properties"`
}

type EventChanges struct {
    Status    []string `xml:"status-id>status-id"`
    Assignee  []string `xml:"assignee-id>assignee-id"`
    Subject   []string `xml:"subject>subject"`
    Priority  []string `xml:"priority-id>priority-id"`
    Milestone []string `xml:"milestone-id>milestone-id"`
    Category  []string `xml:"category-id>category-id"`
}

type eventQueryOptions struct {
    baseQueryOptions
    Raw   bool      `url:"raw,omitempty"`
    Since time.Time `url:"since,omitempty"`
}

type Descriptor interface {
    MapChange(field, before, after string) (description string)
}

func (c *CodeBaseAPI) Activities(since time.Time, user User, descriptor Descriptor) (events []Event) {
    type eventArray struct {
        Events []Event `xml:"event"`
    }

    queryOpts := eventQueryOptions{
        Raw:   true,
        Since: since,
    }
    queryOpts.Page = 1

    for {
        proxy := eventArray{}
        if err := c.fetchFromCodebase("activity", &proxy, queryOpts); err != nil {
            log.Fatalln("Could not fetch activities:", err)
        }

        if len(proxy.Events) == 0 {
            break
        }

        for _, event := range proxy.Events {
            if user.UserID != 0 && event.UserID != user.UserID {
                continue
            }

            if event.HasChanges(descriptor) == false {
                continue
            }

            events = append(events, event)
        }
        queryOpts.Page++
    }

    return
}

func (e *Event) HasChanges(descriptor Descriptor) bool {
    return e.Changes(descriptor) != ""
}

func (e *Event) TicketUrl(company string) string {
    return fmt.Sprintf("https://%s.codebasehq.com/projects/%s/tickets/%v", company, e.Raw.ProjectPermalink, e.Raw.TicketID)
}

func (e *Event) Day() time.Weekday {
    date, _ := time.Parse("2006-01-02 15:04:05 MST", e.Date)
    return date.Weekday()
}

func (e *Event) Changes(descriptor Descriptor) string {
    changes := make([]string, 0)

    type changeToMap struct {
        changeType string
        from       string
        to         string
    }

    chagnesToMap := make([]changeToMap, 0)

    if len(e.Raw.Changes.Status) == 2 {
        change := changeToMap{CHANGE_STATUS, e.Raw.Changes.Status[0], e.Raw.Changes.Status[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Milestone) == 2 {
        change := changeToMap{CHANGE_MILESTONE, e.Raw.Changes.Milestone[0], e.Raw.Changes.Milestone[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Category) == 2 {
        change := changeToMap{CHANGE_CATEGORY, e.Raw.Changes.Category[0], e.Raw.Changes.Category[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if len(e.Raw.Changes.Priority) == 2 {
        change := changeToMap{CHANGE_PRIORITY, e.Raw.Changes.Priority[0], e.Raw.Changes.Priority[1]}
        chagnesToMap = append(chagnesToMap, change)
    }

    if e.Type == TYPE_NEW_TICKET {
        change := changeToMap{changeType: CHANGE_NEW_TICKET}
        chagnesToMap = append(chagnesToMap, change)
    }

    for _, change := range chagnesToMap {
        changeDescription := fmt.Sprintf("%s -> %s", change.from, change.to)

        if descriptor != nil {
            changeDescription = descriptor.MapChange(change.changeType, change.from, change.to)
        }

        if changeDescription != "" {
            changes = append(changes, changeDescription)
        }
    }

    return strings.Join(changes, ", ")
}
<|endoftext|>