<commit_before>// Quick and dirty uploader and serial terminal for LPC8xx chips.
// -jcw, 2015-02-02

package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/chimera/rs232"
	"golang.org/x/crypto/ssh/terminal"
)

var (
	termFlag   = flag.Bool("t", false, "launch as serial terminal after upload")
	waitFlag   = flag.Bool("w", false, "wait for connection to the boot loader")
	offsetFlag = flag.Int("o", 0, "upload offset (must be a multiple of 1024)")
	idleFlag   = flag.Int("i", 0, "exit terminal after N idle seconds (0: off)")
	netFlag    = flag.Bool("n", false, "use telnet protocol for RTS & DTR")
)

func main() {
	log.SetFlags(0) // no timestamps

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: lpc8xx ?options? tty ?binfile?")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(2)
	}

	ttyName := flag.Arg(0)
	binFile := ""
	if flag.NArg() > 1 {
		binFile = flag.Arg(1)
	}

	conn := connect(ttyName)

	id := conn.Identify()
	fmt.Printf("found: %X\n", id)

	conn.SendAndWait("N", "0")
	buf := bytes.NewBuffer([]byte{})

	for i := 0; i < 4; i++ {
		b, err := strconv.ParseUint(<-conn.lines, 10, 32)
		Check(err)
		binary.Write(buf, binary.LittleEndian, uint32(b))
	}
	fmt.Printf("hwuid: %X\n", buf.Bytes())

	if binFile != "" {
		data, err := ioutil.ReadFile(binFile)
		Check(err)

		fmt.Print("flash: 00000 ")
		for n := range conn.Program(*offsetFlag, data) {
			fmt.Printf("\b\b\b\b\b\b%05d ", n*64)
		}
		fmt.Println("done")
	}

	conn.serial.SetDTR(true) // pulse DTR to reset
	conn.serial.SetDTR(false)

	if *termFlag {
		fmt.Println("entering terminal mode, press <ESC> to quit:\n")
		conn.Terminal()
		fmt.Println()
	}
}

func connect(port string) *connection {
	var dev serialLink

	if _, err := os.Stat(port); os.IsNotExist(err) {
		sock, err := net.Dial("tcp", port)
		Check(err)
		dev = &telnet{sock}
	} else {
		opt := rs232.Options{BitRate: 115200, DataBits: 8, StopBits: 1}
		dev, err = rs232.Open(port, opt)
		Check(err)
	}

	conn := &connection{dev, make(chan string)}
	go func() {
		scanner := bufio.NewScanner(dev)
		for scanner.Scan() {
			conn.lines <- scanner.Text()
		}
		close(conn.lines)
	}()
	return conn
}

type serialLink interface {
	io.ReadWriter
	SetDTR(level bool) error
	SetRTS(level bool) error
}

type telnet struct {
	net.Conn
}

func (c *telnet) SetDTR(level bool) error {
	if *netFlag {
		// TODO proper byte sequence
		_, err := net.Conn(c).Write([]byte("<DTR>"))
		return err
	}
	return nil
}

func (c *telnet) SetRTS(level bool) error {
	if *netFlag {
		// TODO proper byte sequence
		_, err := net.Conn(c).Write([]byte("<RTS>"))
		return err
	}
	return nil
}

func (c *telnet) Read(p []byte) (n int, err error) {
	// TODO read, unescape, and ignore in-band data
	return net.Conn(c).Read(p)
}

func (c *telnet) Write(b []byte) (int, error) {
	escaped := bytes.Replace(b, []byte{0xFF}, []byte{0xFF, 0xFF}, -1)
	_, err := net.Conn(c).Write(escaped)
	return len(b), err
}

type connection struct {
	serial serialLink
	lines  chan string
}

func (c *connection) ReadReply() string {
	select {
	case reply := <-c.lines:
		return reply
	case <-time.After(250 * time.Millisecond):
		return ""
	}
}

func (c *connection) SendAndWait(cmd string, expect string) {
	c.serial.Write([]byte(cmd + "\r\n"))
	var reply string
	for i := 0; i < 4; i++ {
		reply = c.ReadReply()
		if reply == "" {
			log.Fatal("no response, timeout")
		}
		if reply == expect {
			return
		}
	}
	log.Fatal(reply)
}

func (c *connection) Identify() int {
	c.serial.SetRTS(true) // keep RTS on for ISP mode

	for {
		c.serial.SetDTR(true) // pulse DTR to reset
		for c.ReadReply() != "" {
			// flush
		}
		c.serial.SetDTR(false)

		c.serial.Write([]byte("?\r\n"))
		if c.ReadReply() == "Synchronized" || !*waitFlag {
			break
		}
	}

	c.serial.SetRTS(false)

	c.SendAndWait("Synchronized", "OK")
	c.SendAndWait("12000", "OK")
	c.SendAndWait("A 0", "0")

	c.SendAndWait("J", "0")
	id, err := strconv.Atoi(c.ReadReply())
	Check(err)

	return id
}

func (c *connection) Program(startAddress int, data []byte) chan int {
	const sectorSize = 1024
	const pageSize = 64

	for len(data)%pageSize != 0 {
		data = append(data, 0xFF)
	}
	firstPage := startAddress / pageSize
	lastPage := firstPage + len(data)/pageSize - 1

	for len(data)%sectorSize != 0 {
		data = append(data, 0xFF)
	}
	firstSector := startAddress / sectorSize
	lastSector := firstSector + len(data)/sectorSize - 1

	if startAddress == 0 {
		fixChecksum(data)
	}

	c.SendAndWait("U 23130", "0") // unlock for programming

	// erase entire range, as 1024-byte sectors
	c.SendAndWait(fmt.Sprint("P ", firstSector, lastSector), "0") // prepare
	c.SendAndWait(fmt.Sprint("E ", firstSector, lastSector), "0") // erase

	r := make(chan int)
	go func() {
		defer close(r)
		// program in 64-byte pages (sectors won't fit in the LPC810's RAM)
		for page := firstPage; page <= lastPage; page++ {
			const RAM_ADDR = 0x10000300
			// write one page of data to RAM
			offset := (page - firstPage) * pageSize
			c.SendAndWait(fmt.Sprint("W ", RAM_ADDR, pageSize), "0")
			c.serial.Write(data[offset : offset+pageSize])
			// prepare and copy the data to flash memory
			sector := (page * pageSize) / sectorSize
			c.SendAndWait(fmt.Sprint("P ", sector, sector), "0")
			destAddr := page * pageSize
			c.SendAndWait(fmt.Sprint("C ", destAddr, RAM_ADDR, pageSize), "0")
			r <- (page - firstPage) + 1
		}
	}()
	return r
}

// fix the checksum to mark code as valid (see UM10398, p.416)
func fixChecksum(data []byte) {
	buf := bytes.NewReader(data)
	values := [7]uint32{}
	err := binary.Read(buf, binary.LittleEndian, values[:])
	Check(err)
	var sum uint32
	for _, v := range values {
		sum -= v
	}
	data[28] = byte(sum)
	data[29] = byte(sum >> 8)
	data[30] = byte(sum >> 16)
	data[31] = byte(sum >> 24)
}

func (c *connection) Terminal() {
	timeout := time.Duration(*idleFlag) * time.Second
	idleTimer := time.NewTimer(timeout)

	// FIXME still in line mode, so only complete lines will be shown
	//  look into bufio.Reader for a possible solution
	go func() {
		for s := range c.lines {
			idleTimer.Reset(timeout)
			fmt.Println(s)
		}
	}()

	// put stdin in raw mode
	oldState, err := terminal.MakeRaw(0)
	Check(err)
	defer terminal.Restore(0, oldState)

	// cleanup when program is terminated via a signal
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM)
	go func() {
		sigMsg := <-sigChan
		terminal.Restore(0, oldState)
		log.Fatal(sigMsg)
	}()

	// cleanup when idle timer fires, and exit cleanly
	if *idleFlag > 0 {
		go func() {
			<-idleTimer.C
			terminal.Restore(0, oldState)
			fmt.Println("\nidle timeout")
			os.Exit(0)
		}()
	}

	// copy key presses to the serial port
	for {
		var b [1]byte
		n, _ := os.Stdin.Read(b[:])
		idleTimer.Reset(timeout)
		if n < 1 || b[0] == 0x1B { // ESC
			break
		}
		c.serial.Write(b[:n])
	}

}

func Check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<commit_msg>only escape writes if telnet mode is enabled<commit_after>// Quick and dirty uploader and serial terminal for LPC8xx chips.
// -jcw, 2015-02-02

package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/chimera/rs232"
	"golang.org/x/crypto/ssh/terminal"
)

var (
	termFlag   = flag.Bool("t", false, "launch as serial terminal after upload")
	waitFlag   = flag.Bool("w", false, "wait for connection to the boot loader")
	offsetFlag = flag.Int("o", 0, "upload offset (must be a multiple of 1024)")
	idleFlag   = flag.Int("i", 0, "exit terminal after N idle seconds (0: off)")
	netFlag    = flag.Bool("n", false, "use telnet protocol for RTS & DTR")
)

func main() {
	log.SetFlags(0) // no timestamps

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: lpc8xx ?options? tty ?binfile?")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(2)
	}

	ttyName := flag.Arg(0)
	binFile := ""
	if flag.NArg() > 1 {
		binFile = flag.Arg(1)
	}

	conn := connect(ttyName)

	id := conn.Identify()
	fmt.Printf("found: %X\n", id)

	conn.SendAndWait("N", "0")
	buf := bytes.NewBuffer([]byte{})

	for i := 0; i < 4; i++ {
		b, err := strconv.ParseUint(<-conn.lines, 10, 32)
		Check(err)
		binary.Write(buf, binary.LittleEndian, uint32(b))
	}
	fmt.Printf("hwuid: %X\n", buf.Bytes())

	if binFile != "" {
		data, err := ioutil.ReadFile(binFile)
		Check(err)

		fmt.Print("flash: 00000 ")
		for n := range conn.Program(*offsetFlag, data) {
			fmt.Printf("\b\b\b\b\b\b%05d ", n*64)
		}
		fmt.Println("done")
	}

	conn.serial.SetDTR(true) // pulse DTR to reset
	conn.serial.SetDTR(false)

	if *termFlag {
		fmt.Println("entering terminal mode, press <ESC> to quit:\n")
		conn.Terminal()
		fmt.Println()
	}
}

func connect(port string) *connection {
	var dev serialLink

	if _, err := os.Stat(port); os.IsNotExist(err) {
		sock, err := net.Dial("tcp", port)
		Check(err)
		dev = &telnet{sock}
	} else {
		opt := rs232.Options{BitRate: 115200, DataBits: 8, StopBits: 1}
		dev, err = rs232.Open(port, opt)
		Check(err)
	}

	conn := &connection{dev, make(chan string)}
	go func() {
		scanner := bufio.NewScanner(dev)
		for scanner.Scan() {
			conn.lines <- scanner.Text()
		}
		close(conn.lines)
	}()
	return conn
}

type serialLink interface {
	io.ReadWriter
	SetDTR(level bool) error
	SetRTS(level bool) error
}

type telnet struct {
	net.Conn
}

func (c *telnet) SetDTR(level bool) error {
	if *netFlag {
		// TODO proper byte sequence
		_, err := net.Conn(c).Write([]byte("<DTR>"))
		return err
	}
	return nil
}

func (c *telnet) SetRTS(level bool) error {
	if *netFlag {
		// TODO proper byte sequence
		_, err := net.Conn(c).Write([]byte("<RTS>"))
		return err
	}
	return nil
}

func (c *telnet) Read(p []byte) (n int, err error) {
	// TODO read, unescape, and ignore in-band data
	return net.Conn(c).Read(p)
}

func (c *telnet) Write(b []byte) (int, error) {
	if *netFlag {
		b = bytes.Replace(b, []byte{0xFF}, []byte{0xFF, 0xFF}, -1)
	}
	_, err := net.Conn(c).Write(b)
	return len(b), err
}

type connection struct {
	serial serialLink
	lines  chan string
}

func (c *connection) ReadReply() string {
	select {
	case reply := <-c.lines:
		return reply
	case <-time.After(250 * time.Millisecond):
		return ""
	}
}

func (c *connection) SendAndWait(cmd string, expect string) {
	c.serial.Write([]byte(cmd + "\r\n"))
	var reply string
	for i := 0; i < 4; i++ {
		reply = c.ReadReply()
		if reply == "" {
			log.Fatal("no response, timeout")
		}
		if reply == expect {
			return
		}
	}
	log.Fatal(reply)
}

func (c *connection) Identify() int {
	c.serial.SetRTS(true) // keep RTS on for ISP mode

	for {
		c.serial.SetDTR(true) // pulse DTR to reset
		for c.ReadReply() != "" {
			// flush
		}
		c.serial.SetDTR(false)

		c.serial.Write([]byte("?\r\n"))
		if c.ReadReply() == "Synchronized" || !*waitFlag {
			break
		}
	}

	c.serial.SetRTS(false)

	c.SendAndWait("Synchronized", "OK")
	c.SendAndWait("12000", "OK")
	c.SendAndWait("A 0", "0")

	c.SendAndWait("J", "0")
	id, err := strconv.Atoi(c.ReadReply())
	Check(err)

	return id
}

func (c *connection) Program(startAddress int, data []byte) chan int {
	const sectorSize = 1024
	const pageSize = 64

	for len(data)%pageSize != 0 {
		data = append(data, 0xFF)
	}
	firstPage := startAddress / pageSize
	lastPage := firstPage + len(data)/pageSize - 1

	for len(data)%sectorSize != 0 {
		data = append(data, 0xFF)
	}
	firstSector := startAddress / sectorSize
	lastSector := firstSector + len(data)/sectorSize - 1

	if startAddress == 0 {
		fixChecksum(data)
	}

	c.SendAndWait("U 23130", "0") // unlock for programming

	// erase entire range, as 1024-byte sectors
	c.SendAndWait(fmt.Sprint("P ", firstSector, lastSector), "0") // prepare
	c.SendAndWait(fmt.Sprint("E ", firstSector, lastSector), "0") // erase

	r := make(chan int)
	go func() {
		defer close(r)
		// program in 64-byte pages (sectors won't fit in the LPC810's RAM)
		for page := firstPage; page <= lastPage; page++ {
			const RAM_ADDR = 0x10000300
			// write one page of data to RAM
			offset := (page - firstPage) * pageSize
			c.SendAndWait(fmt.Sprint("W ", RAM_ADDR, pageSize), "0")
			c.serial.Write(data[offset : offset+pageSize])
			// prepare and copy the data to flash memory
			sector := (page * pageSize) / sectorSize
			c.SendAndWait(fmt.Sprint("P ", sector, sector), "0")
			destAddr := page * pageSize
			c.SendAndWait(fmt.Sprint("C ", destAddr, RAM_ADDR, pageSize), "0")
			r <- (page - firstPage) + 1
		}
	}()
	return r
}

// fix the checksum to mark code as valid (see UM10398, p.416)
func fixChecksum(data []byte) {
	buf := bytes.NewReader(data)
	values := [7]uint32{}
	err := binary.Read(buf, binary.LittleEndian, values[:])
	Check(err)
	var sum uint32
	for _, v := range values {
		sum -= v
	}
	data[28] = byte(sum)
	data[29] = byte(sum >> 8)
	data[30] = byte(sum >> 16)
	data[31] = byte(sum >> 24)
}

func (c *connection) Terminal() {
	timeout := time.Duration(*idleFlag) * time.Second
	idleTimer := time.NewTimer(timeout)

	// FIXME still in line mode, so only complete lines will be shown
	//  look into bufio.Reader for a possible solution
	go func() {
		for s := range c.lines {
			idleTimer.Reset(timeout)
			fmt.Println(s)
		}
	}()

	// put stdin in raw mode
	oldState, err := terminal.MakeRaw(0)
	Check(err)
	defer terminal.Restore(0, oldState)

	// cleanup when program is terminated via a signal
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM)
	go func() {
		sigMsg := <-sigChan
		terminal.Restore(0, oldState)
		log.Fatal(sigMsg)
	}()

	// cleanup when idle timer fires, and exit cleanly
	if *idleFlag > 0 {
		go func() {
			<-idleTimer.C
			terminal.Restore(0, oldState)
			fmt.Println("\nidle timeout")
			os.Exit(0)
		}()
	}

	// copy key presses to the serial port
	for {
		var b [1]byte
		n, _ := os.Stdin.Read(b[:])
		idleTimer.Reset(timeout)
		if n < 1 || b[0] == 0x1B { // ESC
			break
		}
		c.serial.Write(b[:n])
	}

}

func Check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<|endoftext|>


<commit_before>package webdriver_test

import (
	"fmt"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"

	"github.com/johanbrandhorst/protobuf/test/shared"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	if os.Getenv("GOPHERJS_SERVER_ADDR") != "" {
		if os.Getenv("CHROMEDRIVER_ADDR") != "" {
			browserTest("Google Chrome", os.Getenv("GOPHERJS_SERVER_ADDR"), func(opts ...agouti.Option) (*agouti.Page, error) {
				return agouti.NewPage(fmt.Sprintf("http://%s", os.Getenv("CHROMEDRIVER_ADDR")),
					agouti.Desired(agouti.Capabilities{
						"loggingPrefs": map[string]string{
							"browser": "INFO",
						},
					}))
			})
		}

		if os.Getenv("SELENIUM_ADDR") != "" {
			browserTest("Mozilla Firefox", os.Getenv("GOPHERJS_SERVER_ADDR"), func(opts ...agouti.Option) (*agouti.Page, error) {
				return agouti.NewPage(fmt.Sprintf("http://%s/wd/hub", os.Getenv("SELENIUM_ADDR")),
					agouti.Desired(agouti.Capabilities{
						"loggingPrefs": map[string]string{
							"browser": "INFO",
						},
						"acceptInsecureCerts": true,
					}),
					agouti.Browser("firefox"),
				)
			})
		}
	} else {
		browserTest("ChromeDriver", "localhost"+shared.GopherJSServer, chromeDriver.NewPage)
	}
})

type pageFunc func(...agouti.Option) (*agouti.Page, error)

func browserTest(browserName, address string, newPage pageFunc) {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = newPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	Context(fmt.Sprintf("when testing %s", browserName), func() {
		It("should not find any errors", func() {
			By("Loading the test page", func() {
				Expect(page.Navigate("https://" + address)).NotTo(HaveOccurred())
			})

			By("Finding the number of failures", func() {
				Eventually(page.FirstByClass("failed"), 2).Should(BeFound())
				Eventually(page.FindByID("qunit-testresult").FindByClass("failed"), 2).Should(BeFound())
				numFailures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
				Expect(err).NotTo(HaveOccurred())
				if numFailures == "0" {
					return
				}

				logs, err := page.ReadAllLogs("browser")
				Expect(err).NotTo(HaveOccurred())
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")
				for _, log := range logs {
					fmt.Fprintf(GinkgoWriter, "[%s][%s]\t%s\n", log.Time.Format("15:04:05.000"), log.Level, log.Message)
				}
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")

				// We have at least one failure - lets compile an error message
				Eventually(page.AllByXPath("//li[contains(@id, 'qunit-test-output') and @class='fail']")).Should(BeFound())
				failures := page.AllByXPath("//li[contains(@id, 'qunit-test-output') and @class='fail']")
				elements, err := failures.Elements()
				Expect(err).NotTo(HaveOccurred())
				var errMsgs []string
				for _, element := range elements {
					// Get module name
					msg, err := element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".module-name",
					})
					Expect(err).NotTo(HaveOccurred())
					modName, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())
					// Get test name
					msg, err = element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".test-name",
					})
					Expect(err).NotTo(HaveOccurred())
					testName, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())
					// Get specific fail node
					fails, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: ".fail",
					})
					Expect(err).NotTo(HaveOccurred())
					var errSums []string
					for _, fail := range fails {
						// Get error summary
						msg, err := fail.GetElement(api.Selector{
							Using: "css selector",
							Value: ".test-message",
						})
						errSum, err := msg.GetText()
						Expect(err).NotTo(HaveOccurred())

						// Get diff
						expected, err := fail.GetElements(api.Selector{
							Using: "css selector",
							Value: "del",
						})
						Expect(err).NotTo(HaveOccurred())
						var expectedText string
						if len(expected) > 0 {
							expectedText, err = expected[0].GetText()
							Expect(err).NotTo(HaveOccurred())
						}
						actual, err := fail.GetElements(api.Selector{
							Using: "css selector",
							Value: "ins",
						})
						Expect(err).NotTo(HaveOccurred())
						var actualText string
						if len(actual) > 0 {
							actualText, err = actual[0].GetText()
							Expect(err).NotTo(HaveOccurred())
						}
						if expectedText != "" && actualText != "" {
							errSum = fmt.Sprintf(
								"%s\n\t\t\tExpected: %s\n\t\t\tActual: %s",
								errSum,
								strings.TrimSuffix(expectedText, " "),
								strings.TrimSuffix(actualText, " "),
							)
						}

						errSums = append(errSums, errSum)
					}

					errMsgs = append(errMsgs, fmt.Sprintf("%s:\n\t%s:\n\t\t%s", modName, testName, strings.Join(errSums, "\n\t\t")))
				}

				// Prints each error
				Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
			})
		})
	})
}
<commit_msg>Increase test timeout<commit_after>package webdriver_test

import (
	"fmt"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"

	"github.com/johanbrandhorst/protobuf/test/shared"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	if os.Getenv("GOPHERJS_SERVER_ADDR") != "" {
		if os.Getenv("CHROMEDRIVER_ADDR") != "" {
			browserTest("Google Chrome", os.Getenv("GOPHERJS_SERVER_ADDR"), func(opts ...agouti.Option) (*agouti.Page, error) {
				return agouti.NewPage(fmt.Sprintf("http://%s", os.Getenv("CHROMEDRIVER_ADDR")),
					agouti.Desired(agouti.Capabilities{
						"loggingPrefs": map[string]string{
							"browser": "INFO",
						},
					}))
			})
		}

		if os.Getenv("SELENIUM_ADDR") != "" {
			browserTest("Mozilla Firefox", os.Getenv("GOPHERJS_SERVER_ADDR"), func(opts ...agouti.Option) (*agouti.Page, error) {
				return agouti.NewPage(fmt.Sprintf("http://%s/wd/hub", os.Getenv("SELENIUM_ADDR")),
					agouti.Desired(agouti.Capabilities{
						"loggingPrefs": map[string]string{
							"browser": "INFO",
						},
						"acceptInsecureCerts": true,
					}),
					agouti.Browser("firefox"),
				)
			})
		}
	} else {
		browserTest("ChromeDriver", "localhost"+shared.GopherJSServer, chromeDriver.NewPage)
	}
})

type pageFunc func(...agouti.Option) (*agouti.Page, error)

func browserTest(browserName, address string, newPage pageFunc) {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = newPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	Context(fmt.Sprintf("when testing %s", browserName), func() {
		It("should not find any errors", func() {
			By("Loading the test page", func() {
				Expect(page.Navigate("https://" + address)).NotTo(HaveOccurred())
			})

			By("Finding the number of failures", func() {
				Eventually(page.FirstByClass("failed"), 10).Should(BeFound())
				Eventually(page.FindByID("qunit-testresult").FindByClass("failed"), 10).Should(BeFound())
				numFailures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
				Expect(err).NotTo(HaveOccurred())
				if numFailures == "0" {
					return
				}

				logs, err := page.ReadAllLogs("browser")
				Expect(err).NotTo(HaveOccurred())
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")
				for _, log := range logs {
					fmt.Fprintf(GinkgoWriter, "[%s][%s]\t%s\n", log.Time.Format("15:04:05.000"), log.Level, log.Message)
				}
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")

				// We have at least one failure - lets compile an error message
				Eventually(page.AllByXPath("//li[contains(@id, 'qunit-test-output') and @class='fail']")).Should(BeFound())
				failures := page.AllByXPath("//li[contains(@id, 'qunit-test-output') and @class='fail']")
				elements, err := failures.Elements()
				Expect(err).NotTo(HaveOccurred())
				var errMsgs []string
				for _, element := range elements {
					// Get module name
					msg, err := element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".module-name",
					})
					Expect(err).NotTo(HaveOccurred())
					modName, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())
					// Get test name
					msg, err = element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".test-name",
					})
					Expect(err).NotTo(HaveOccurred())
					testName, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())
					// Get specific fail node
					fails, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: ".fail",
					})
					Expect(err).NotTo(HaveOccurred())
					var errSums []string
					for _, fail := range fails {
						// Get error summary
						msg, err := fail.GetElement(api.Selector{
							Using: "css selector",
							Value: ".test-message",
						})
						errSum, err := msg.GetText()
						Expect(err).NotTo(HaveOccurred())

						// Get diff
						expected, err := fail.GetElements(api.Selector{
							Using: "css selector",
							Value: "del",
						})
						Expect(err).NotTo(HaveOccurred())
						var expectedText string
						if len(expected) > 0 {
							expectedText, err = expected[0].GetText()
							Expect(err).NotTo(HaveOccurred())
						}
						actual, err := fail.GetElements(api.Selector{
							Using: "css selector",
							Value: "ins",
						})
						Expect(err).NotTo(HaveOccurred())
						var actualText string
						if len(actual) > 0 {
							actualText, err = actual[0].GetText()
							Expect(err).NotTo(HaveOccurred())
						}
						if expectedText != "" && actualText != "" {
							errSum = fmt.Sprintf(
								"%s\n\t\t\tExpected: %s\n\t\t\tActual: %s",
								errSum,
								strings.TrimSuffix(expectedText, " "),
								strings.TrimSuffix(actualText, " "),
							)
						}

						errSums = append(errSums, errSum)
					}

					errMsgs = append(errMsgs, fmt.Sprintf("%s:\n\t%s:\n\t\t%s", modName, testName, strings.Join(errSums, "\n\t\t")))
				}

				// Prints each error
				Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
			})
		})
	})
}
<|endoftext|>


<commit_before>//  Copyright 2017 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package daisy

import (
	"context"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"sync"
	"time"
)

const (
	defaultInterval = "10s"
)

var (
	serialOutputValueRegex = regexp.MustCompile(".*<serial-output key:'(.*)' value:'(.*)'>")
)

// WaitForInstancesSignal is a Daisy WaitForInstancesSignal workflow step.
type WaitForInstancesSignal []*InstanceSignal

// FailureMatches is a list of matching failure strings.
type FailureMatches []string

// UnmarshalJSON unmarshals FailureMatches.
func (fms *FailureMatches) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err == nil {
		*fms = []string{s}
		return nil
	}

	//not a string, try unmarshalling into an array. Need a temp type to avoid infinite loop.
	var ss []string
	if err := json.Unmarshal(b, &ss); err != nil {
		return err
	}

	*fms = FailureMatches(ss)
	return nil
}

// SerialOutput describes text signal strings that will be written to the serial
// port.
// A StatusMatch will print out the matching line from the StatusMatch onward.
// This step will not complete until a line in the serial output matches
// SuccessMatch or FailureMatch. A match with FailureMatch will cause the step to fail.
type SerialOutput struct {
	Port         int64          `json:",omitempty"`
	SuccessMatch string         `json:",omitempty"`
	FailureMatch FailureMatches `json:"failureMatch,omitempty"`
	StatusMatch  string         `json:",omitempty"`
}

// InstanceSignal waits for a signal from an instance.
type InstanceSignal struct {
	// Instance name to wait for.
	Name string
	// Interval to check for signal (default is 5s).
	// Must be parsable by https://golang.org/pkg/time/#ParseDuration.
	Interval string `json:",omitempty"`
	interval time.Duration
	// Wait for the instance to stop.
	Stopped bool `json:",omitempty"`
	// Wait for a string match in the serial output.
	SerialOutput *SerialOutput `json:",omitempty"`
}

func waitForInstanceStopped(s *Step, project, zone, name string, interval time.Duration) DError {
	w := s.w
	w.LogStepInfo(s.name, "WaitForInstancesSignal", "Waiting for instance %q to stop.", name)
	tick := time.Tick(interval)
	for {
		select {
		case <-s.w.Cancel:
			return nil
		case <-tick:
			stopped, err := s.w.ComputeClient.InstanceStopped(project, zone, name)
			if err != nil {
				return typedErr(apiError, "failed to check whether instance is stopped", err)
			}
			if stopped {
				w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q stopped.", name)
				return nil
			}
		}
	}
}

func waitForSerialOutput(s *Step, project, zone, name string, so *SerialOutput, interval time.Duration) DError {
	w := s.w
	msg := fmt.Sprintf("Instance %q: watching serial port %d", name, so.Port)
	if so.SuccessMatch != "" {
		msg += fmt.Sprintf(", SuccessMatch: %q", so.SuccessMatch)
	}
	if len(so.FailureMatch) > 0 {
		msg += fmt.Sprintf(", FailureMatch: %q", so.FailureMatch)
	}
	if so.StatusMatch != "" {
		msg += fmt.Sprintf(", StatusMatch: %q", so.StatusMatch)
	}
	w.LogStepInfo(s.name, "WaitForInstancesSignal", msg+".")
	var start int64
	var errs int
	tick := time.Tick(interval)
	for {
		select {
		case <-s.w.Cancel:
			return nil
		case <-tick:
			resp, err := w.ComputeClient.GetSerialPortOutput(project, zone, name, so.Port, start)
			if err != nil {
				status, sErr := w.ComputeClient.InstanceStatus(project, zone, name)
				if sErr != nil {
					err = fmt.Errorf("%v, error getting InstanceStatus: %v", err, sErr)
				} else {
					err = fmt.Errorf("%v, InstanceStatus: %q", err, status)
				}

				// Wait until machine restarts to evaluate SerialOutput.
				if status == "TERMINATED" || status == "STOPPED" || status == "STOPPING" {
					continue
				}

				// Retry up to 3 times in a row on any error if we successfully got InstanceStatus.
				if errs < 3 {
					errs++
					continue
				}

				return Errf("WaitForInstancesSignal: instance %q: error getting serial port: %v", name, err)
			}
			start = resp.Next
			for _, ln := range strings.Split(resp.Contents, "\n") {
				if so.StatusMatch != "" {
					if i := strings.Index(ln, so.StatusMatch); i != -1 {
						w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q: StatusMatch found: %q", name, strings.TrimSpace(ln[i:]))
						extractOutputValue(w, ln)
					}
				}
				if len(so.FailureMatch) > 0 {
					for _, failureMatch := range so.FailureMatch {
						if i := strings.Index(ln, failureMatch); i != -1 {
							errMsg := strings.TrimSpace(ln[i:])
							format := "WaitForInstancesSignal FailureMatch found for %q: %q"
							return newErr(errMsg, fmt.Errorf(format, name, errMsg))
						}
					}
				}
				if so.SuccessMatch != "" {
					if i := strings.Index(ln, so.SuccessMatch); i != -1 {
						w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q: SuccessMatch found %q", name, strings.TrimSpace(ln[i:]))
						return nil
					}
				}
			}
			errs = 0
		}
	}
}

func extractOutputValue(w *Workflow, s string) {
	if matches := serialOutputValueRegex.FindStringSubmatch(s); matches != nil && len(matches) == 3 {
		for w.parent != nil {
			w = w.parent
		}
		w.AddSerialConsoleOutputValue(matches[1], matches[2])
	}
}

func (w *WaitForInstancesSignal) populate(ctx context.Context, s *Step) DError {
	for _, ws := range *w {
		if ws.Interval == "" {
			ws.Interval = defaultInterval
		}
		var err error
		ws.interval, err = time.ParseDuration(ws.Interval)
		if err != nil {
			return newErr("failed to parse duration for step wait_for_instance_signal", err)
		}
	}
	return nil
}

func (w *WaitForInstancesSignal) run(ctx context.Context, s *Step) DError {
	var wg sync.WaitGroup
	e := make(chan DError)

	for _, is := range *w {
		wg.Add(1)
		go func(is *InstanceSignal) {
			defer wg.Done()
			i, ok := s.w.instances.get(is.Name)
			if !ok {
				e <- Errf("unresolved instance %q", is.Name)
				return
			}
			m := namedSubexp(instanceURLRgx, i.link)
			serialSig := make(chan struct{})
			stoppedSig := make(chan struct{})
			if is.Stopped {
				go func() {
					if err := waitForInstanceStopped(s, m["project"], m["zone"], m["instance"], is.interval); err != nil {
						e <- err
					}
					close(stoppedSig)
				}()
			}
			if is.SerialOutput != nil {
				go func() {
					if err := waitForSerialOutput(s, m["project"], m["zone"], m["instance"], is.SerialOutput, is.interval); err != nil {
						e <- err
					}
					close(serialSig)
				}()
			}
			select {
			case <-serialSig:
				return
			case <-stoppedSig:
				return
			}
		}(is)
	}

	go func() {
		wg.Wait()
		e <- nil
	}()

	select {
	case err := <-e:
		return err
	case <-s.w.Cancel:
		return nil
	}
}

func (w *WaitForInstancesSignal) validate(ctx context.Context, s *Step) DError {
	// Instance checking.
	for _, i := range *w {
		if _, err := s.w.instances.regUse(i.Name, s); err != nil {
			return err
		}
		if i.interval == 0*time.Second {
			return Errf("%q: cannot wait for instance signal, no interval given", i.Name)
		}
		if i.SerialOutput == nil && i.Stopped == false {
			return Errf("%q: cannot wait for instance signal, nothing to wait for", i.Name)
		}
		if i.SerialOutput != nil {
			if i.SerialOutput.Port == 0 {
				return Errf("%q: cannot wait for instance signal via SerialOutput, no Port given", i.Name)
			}
			if i.SerialOutput.SuccessMatch == "" && len(i.SerialOutput.FailureMatch) == 0 {
				return Errf("%q: cannot wait for instance signal via SerialOutput, no SuccessMatch or FailureMatch given", i.Name)
			}
		}
	}
	return nil
}
<commit_msg>Added explanation that info about FailureMatch is not an error. This has confused engineers who are not working on Daisy directly so it must be confusing to customers. (#948)<commit_after>//  Copyright 2017 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package daisy

import (
	"context"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"sync"
	"time"
)

const (
	defaultInterval = "10s"
)

var (
	serialOutputValueRegex = regexp.MustCompile(".*<serial-output key:'(.*)' value:'(.*)'>")
)

// WaitForInstancesSignal is a Daisy WaitForInstancesSignal workflow step.
type WaitForInstancesSignal []*InstanceSignal

// FailureMatches is a list of matching failure strings.
type FailureMatches []string

// UnmarshalJSON unmarshals FailureMatches.
func (fms *FailureMatches) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err == nil {
		*fms = []string{s}
		return nil
	}

	//not a string, try unmarshalling into an array. Need a temp type to avoid infinite loop.
	var ss []string
	if err := json.Unmarshal(b, &ss); err != nil {
		return err
	}

	*fms = FailureMatches(ss)
	return nil
}

// SerialOutput describes text signal strings that will be written to the serial
// port.
// A StatusMatch will print out the matching line from the StatusMatch onward.
// This step will not complete until a line in the serial output matches
// SuccessMatch or FailureMatch. A match with FailureMatch will cause the step to fail.
type SerialOutput struct {
	Port         int64          `json:",omitempty"`
	SuccessMatch string         `json:",omitempty"`
	FailureMatch FailureMatches `json:"failureMatch,omitempty"`
	StatusMatch  string         `json:",omitempty"`
}

// InstanceSignal waits for a signal from an instance.
type InstanceSignal struct {
	// Instance name to wait for.
	Name string
	// Interval to check for signal (default is 5s).
	// Must be parsable by https://golang.org/pkg/time/#ParseDuration.
	Interval string `json:",omitempty"`
	interval time.Duration
	// Wait for the instance to stop.
	Stopped bool `json:",omitempty"`
	// Wait for a string match in the serial output.
	SerialOutput *SerialOutput `json:",omitempty"`
}

func waitForInstanceStopped(s *Step, project, zone, name string, interval time.Duration) DError {
	w := s.w
	w.LogStepInfo(s.name, "WaitForInstancesSignal", "Waiting for instance %q to stop.", name)
	tick := time.Tick(interval)
	for {
		select {
		case <-s.w.Cancel:
			return nil
		case <-tick:
			stopped, err := s.w.ComputeClient.InstanceStopped(project, zone, name)
			if err != nil {
				return typedErr(apiError, "failed to check whether instance is stopped", err)
			}
			if stopped {
				w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q stopped.", name)
				return nil
			}
		}
	}
}

func waitForSerialOutput(s *Step, project, zone, name string, so *SerialOutput, interval time.Duration) DError {
	w := s.w
	msg := fmt.Sprintf("Instance %q: watching serial port %d", name, so.Port)
	if so.SuccessMatch != "" {
		msg += fmt.Sprintf(", SuccessMatch: %q", so.SuccessMatch)
	}
	if len(so.FailureMatch) > 0 {
		msg += fmt.Sprintf(", FailureMatch: %q (this is not an error)", so.FailureMatch)
	}
	if so.StatusMatch != "" {
		msg += fmt.Sprintf(", StatusMatch: %q", so.StatusMatch)
	}
	w.LogStepInfo(s.name, "WaitForInstancesSignal", msg+".")
	var start int64
	var errs int
	tick := time.Tick(interval)
	for {
		select {
		case <-s.w.Cancel:
			return nil
		case <-tick:
			resp, err := w.ComputeClient.GetSerialPortOutput(project, zone, name, so.Port, start)
			if err != nil {
				status, sErr := w.ComputeClient.InstanceStatus(project, zone, name)
				if sErr != nil {
					err = fmt.Errorf("%v, error getting InstanceStatus: %v", err, sErr)
				} else {
					err = fmt.Errorf("%v, InstanceStatus: %q", err, status)
				}

				// Wait until machine restarts to evaluate SerialOutput.
				if status == "TERMINATED" || status == "STOPPED" || status == "STOPPING" {
					continue
				}

				// Retry up to 3 times in a row on any error if we successfully got InstanceStatus.
				if errs < 3 {
					errs++
					continue
				}

				return Errf("WaitForInstancesSignal: instance %q: error getting serial port: %v", name, err)
			}
			start = resp.Next
			for _, ln := range strings.Split(resp.Contents, "\n") {
				if so.StatusMatch != "" {
					if i := strings.Index(ln, so.StatusMatch); i != -1 {
						w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q: StatusMatch found: %q", name, strings.TrimSpace(ln[i:]))
						extractOutputValue(w, ln)
					}
				}
				if len(so.FailureMatch) > 0 {
					for _, failureMatch := range so.FailureMatch {
						if i := strings.Index(ln, failureMatch); i != -1 {
							errMsg := strings.TrimSpace(ln[i:])
							format := "WaitForInstancesSignal FailureMatch found for %q: %q"
							return newErr(errMsg, fmt.Errorf(format, name, errMsg))
						}
					}
				}
				if so.SuccessMatch != "" {
					if i := strings.Index(ln, so.SuccessMatch); i != -1 {
						w.LogStepInfo(s.name, "WaitForInstancesSignal", "Instance %q: SuccessMatch found %q", name, strings.TrimSpace(ln[i:]))
						return nil
					}
				}
			}
			errs = 0
		}
	}
}

func extractOutputValue(w *Workflow, s string) {
	if matches := serialOutputValueRegex.FindStringSubmatch(s); matches != nil && len(matches) == 3 {
		for w.parent != nil {
			w = w.parent
		}
		w.AddSerialConsoleOutputValue(matches[1], matches[2])
	}
}

func (w *WaitForInstancesSignal) populate(ctx context.Context, s *Step) DError {
	for _, ws := range *w {
		if ws.Interval == "" {
			ws.Interval = defaultInterval
		}
		var err error
		ws.interval, err = time.ParseDuration(ws.Interval)
		if err != nil {
			return newErr("failed to parse duration for step wait_for_instance_signal", err)
		}
	}
	return nil
}

func (w *WaitForInstancesSignal) run(ctx context.Context, s *Step) DError {
	var wg sync.WaitGroup
	e := make(chan DError)

	for _, is := range *w {
		wg.Add(1)
		go func(is *InstanceSignal) {
			defer wg.Done()
			i, ok := s.w.instances.get(is.Name)
			if !ok {
				e <- Errf("unresolved instance %q", is.Name)
				return
			}
			m := namedSubexp(instanceURLRgx, i.link)
			serialSig := make(chan struct{})
			stoppedSig := make(chan struct{})
			if is.Stopped {
				go func() {
					if err := waitForInstanceStopped(s, m["project"], m["zone"], m["instance"], is.interval); err != nil {
						e <- err
					}
					close(stoppedSig)
				}()
			}
			if is.SerialOutput != nil {
				go func() {
					if err := waitForSerialOutput(s, m["project"], m["zone"], m["instance"], is.SerialOutput, is.interval); err != nil {
						e <- err
					}
					close(serialSig)
				}()
			}
			select {
			case <-serialSig:
				return
			case <-stoppedSig:
				return
			}
		}(is)
	}

	go func() {
		wg.Wait()
		e <- nil
	}()

	select {
	case err := <-e:
		return err
	case <-s.w.Cancel:
		return nil
	}
}

func (w *WaitForInstancesSignal) validate(ctx context.Context, s *Step) DError {
	// Instance checking.
	for _, i := range *w {
		if _, err := s.w.instances.regUse(i.Name, s); err != nil {
			return err
		}
		if i.interval == 0*time.Second {
			return Errf("%q: cannot wait for instance signal, no interval given", i.Name)
		}
		if i.SerialOutput == nil && i.Stopped == false {
			return Errf("%q: cannot wait for instance signal, nothing to wait for", i.Name)
		}
		if i.SerialOutput != nil {
			if i.SerialOutput.Port == 0 {
				return Errf("%q: cannot wait for instance signal via SerialOutput, no Port given", i.Name)
			}
			if i.SerialOutput.SuccessMatch == "" && len(i.SerialOutput.FailureMatch) == 0 {
				return Errf("%q: cannot wait for instance signal via SerialOutput, no SuccessMatch or FailureMatch given", i.Name)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"image"
	_ "image/jpeg"
	"log"
	"mime"
	"os"
	"path/filepath"

	"github.com/agorf/goexif/exif"
	_ "github.com/mattn/go-sqlite3"
)

type Photo struct {
	Aperture      sql.NullFloat64
	Camera        sql.NullString
	ExposureComp  sql.NullInt64
	ExposureTime  sql.NullFloat64
	Flash         sql.NullString
	FocalLength   sql.NullFloat64
	FocalLength35 sql.NullInt64
	Height        int
	ISO           sql.NullInt64
	Lat           sql.NullFloat64
	Lens          sql.NullString
	Lng           sql.NullFloat64
	Path          string
	Size          int64
	Taken         sql.NullString
	Width         int
}

var (
	db                                                             *sql.DB
	selectSetStmt, selectPhotoStmt, insertSetStmt, insertPhotoStmt *sql.Stmt
)

func decodePhotoExif(photo *Photo, x *exif.Exif) {
	taken, err := x.DateTime()
	if err == nil {
		photo.Taken.String = taken.UTC().Format("2006-01-02 15:04:05")
		photo.Taken.Valid = true
	}

	lat, lng, err := x.LatLong()
	if err == nil {
		photo.Lat.Float64 = lat
		photo.Lat.Valid = true
		photo.Lng.Float64 = lng
		photo.Lng.Valid = true
	}

	orientTag, err := x.Get(exif.Orientation)
	if err == nil {
		switch orient, _ := orientTag.Int(0); orient {
		case 5, 6, 7, 8: // rotated
			photo.Width, photo.Height = photo.Height, photo.Width // swap
		}
	}

	camMakeTag, err := x.Get(exif.Make)
	if err == nil {
		photo.Camera.String, _ = camMakeTag.StringVal()
		photo.Camera.Valid = true
	}

	camModelTag, err := x.Get(exif.Model)
	if err == nil {
		cameraModel, _ := camModelTag.StringVal()

		if photo.Camera.Valid {
			photo.Camera.String = fmt.Sprint(photo.Camera.String, " ", cameraModel)
		} else {
			photo.Camera.String = cameraModel
			photo.Camera.Valid = true
		}
	}

	lensMakeTag, err := x.Get(exif.LensMake)
	if err == nil {
		photo.Lens.String, _ = lensMakeTag.StringVal()
		photo.Lens.Valid = true
	}

	lensModelTag, err := x.Get(exif.LensModel)
	if err == nil {
		lensModel, _ := lensModelTag.StringVal()

		if photo.Lens.Valid {
			photo.Lens.String = fmt.Sprint(photo.Lens.String, " ", lensModel)
		} else {
			photo.Lens.String = lensModel
			photo.Lens.Valid = true
		}
	}

	focalLenTag, err := x.Get(exif.FocalLength)
	if err == nil {
		focalLen, _ := focalLenTag.Rat(0)
		photo.FocalLength.Float64, _ = focalLen.Float64()
		photo.FocalLength.Valid = true
	}

	focalLen35Tag, err := x.Get(exif.FocalLengthIn35mmFilm)
	if err == nil {
		photo.FocalLength35.Int64, _ = focalLen35Tag.Int64(0)
		photo.FocalLength35.Valid = true
	}

	apertureTag, err := x.Get(exif.FNumber)
	if err == nil {
		aperture, _ := apertureTag.Rat(0)
		photo.Aperture.Float64, _ = aperture.Float64()
		photo.Aperture.Valid = true
	}

	expTimeTag, err := x.Get(exif.ExposureTime)
	if err == nil {
		expTime, _ := expTimeTag.Rat(0)
		photo.ExposureTime.Float64, _ = expTime.Float64()
		photo.ExposureTime.Valid = true
	}

	isoTag, err := x.Get(exif.ISOSpeedRatings)
	if err == nil {
		photo.ISO.Int64, _ = isoTag.Int64(0)
		photo.ISO.Valid = true
	}

	expBiasTag, err := x.Get(exif.ExposureBiasValue)
	if err == nil {
		photo.ExposureComp.Int64, _ = expBiasTag.Int64(0)
		photo.ExposureComp.Valid = true
	}

	flash, err := x.Flash()
	if err == nil {
		photo.Flash.String = flash
		photo.Flash.Valid = true
	}
}

func decodePhoto(path string) (*Photo, error) {
	var photo Photo

	photo.Path = path

	f, err := os.Open(path)
	if err != nil {
		return &photo, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return &photo, err
	}
	photo.Size = fi.Size()

	img, _, err := image.DecodeConfig(f)
	if err != nil {
		return &photo, err
	}
	photo.Width, photo.Height = img.Width, img.Height

	f.Seek(0, 0) // rewind

	x, err := exif.Decode(f)
	if err == nil { // EXIF data exists
		decodePhotoExif(&photo, x)
	}

	return &photo, nil
}

func storePhoto(photo *Photo) error {
	var setId, photoId int64

	setName := filepath.Base(filepath.Dir(photo.Path))
	row := selectSetStmt.QueryRow(setName)
	if err := row.Scan(&setId); err == sql.ErrNoRows { // set does not exist
		result, err := insertSetStmt.Exec(setName) // create it
		if err != nil {
			return err
		}

		setId, err = result.LastInsertId()
		if err != nil {
			return err
		}
	}

	row = selectPhotoStmt.QueryRow(photo.Path)
	if err := row.Scan(&photoId); err == sql.ErrNoRows { // photo does not exist
		result, err := insertPhotoStmt.Exec(photo.Aperture, photo.Camera,
			photo.ExposureComp, photo.ExposureTime, photo.Flash, photo.FocalLength,
			photo.FocalLength35, photo.Height, photo.ISO, photo.Lat, photo.Lens,
			photo.Lng, photo.Path, setId, photo.Size, photo.Taken, photo.Width) // create it
		if err != nil {
			return err
		}

		photoId, err = result.LastInsertId()
		if err != nil {
			return err
		}

		fmt.Fprintf(os.Stderr, "photos id=%d path=%s\n", photoId, photo.Path)
	}

	return nil
}

func walk(path string, info os.FileInfo, err error) error {
	if err != nil { // error walking "path"
		return nil // skip
	}

	if info.IsDir() {
		return nil // skip
	}

	if mime.TypeByExtension(filepath.Ext(path)) != "image/jpeg" { // not JPEG
		return nil // skip
	}

	photo, err := decodePhoto(path)
	if err == nil {
		storePhoto(photo)
	}

	return nil // next
}

func updatePhotoSiblings() error {
	var prevId, prevSetId int

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	updatePrevPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET prev_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updatePrevPhotoStmt.Close()

	updateNextPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET next_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateNextPhotoStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id FROM photos ORDER BY set_id, taken_at
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId int
		rows.Scan(&id, &setId)

		if setId == prevSetId && prevId > 0 {
			updatePrevPhotoStmt.Exec(prevId, id)
			fmt.Fprintf(os.Stderr, "photos id=%d prev_photo_id=%d\n", id, prevId)
			updateNextPhotoStmt.Exec(id, prevId)
			fmt.Fprintf(os.Stderr, "photos id=%d next_photo_id=%d\n", prevId, id)
		}

		prevId = id
		prevSetId = setId
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func updateSets() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	photosCountStmt, err := tx.Prepare(`
	SELECT COUNT(*) FROM photos WHERE set_id = ?
	`)
	if err != nil {
		return err
	}
	defer photosCountStmt.Close()

	updateSetStmt, err := tx.Prepare(`
	UPDATE sets
	SET photos_count = ?, taken_at = ?, thumb_photo_id = ?
	WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateSetStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id, MIN(taken_at) FROM photos GROUP BY set_id
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId, photosCount int
		var taken sql.NullString

		rows.Scan(&id, &setId, &taken)

		row := photosCountStmt.QueryRow(setId)
		row.Scan(&photosCount)

		updateSetStmt.Exec(photosCount, taken, id, setId)
		fmt.Fprintf(os.Stderr, "sets id=%d photos_count=%d taken=%q thumb_photo_id=%d\n", setId, photosCount, taken.String, id)
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func loadSchema(db *sql.DB) (err error) {
	_, err = db.Exec(`
	CREATE TABLE IF NOT EXISTS sets (
	id integer NOT NULL PRIMARY KEY,
	thumb_photo_id integer UNIQUE REFERENCES photos,
	name varchar(4096) NOT NULL UNIQUE,
	photos_count integer,
	taken_at datetime
	);

	CREATE INDEX IF NOT EXISTS sets_thumb_photo_id_index ON sets (thumb_photo_id);

	CREATE TABLE IF NOT EXISTS photos (
	id integer NOT NULL PRIMARY KEY,
	set_id integer NOT NULL REFERENCES sets,
	prev_photo_id integer UNIQUE REFERENCES photos,
	next_photo_id integer UNIQUE REFERENCES photos,
	path varchar(4096) NOT NULL UNIQUE,
	size integer NOT NULL,
	width integer NOT NULL,
	height integer NOT NULL,
	aperture decimal(2, 1),
	camera varchar(1000),
	exposure_comp integer,
	exposure_time decimal(9, 5),
	flash varchar(51),
	focal_length decimal(3, 1),
	focal_length_35 integer,
	iso integer,
	lat decimal(9, 6),
	lens varchar(1000),
	lng decimal(9, 6),
	taken_at datetime
	);

	CREATE INDEX IF NOT EXISTS photos_set_id_index ON photos (set_id);

	CREATE INDEX IF NOT EXISTS photos_prev_photo_id_index ON photos (prev_photo_id);

	CREATE INDEX IF NOT EXISTS photos_next_photo_id_index ON photos (next_photo_id);
	`)

	return
}

func main() {
	var err error

	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %v path [path...]\n", os.Args[0])
		return
	}

	db, err = sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	if loadSchema(db) != nil {
		log.Fatal(err)
	}

	selectSetStmt, err = db.Prepare("SELECT id FROM sets WHERE name = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectSetStmt.Close()

	selectPhotoStmt, err = db.Prepare("SELECT id FROM photos WHERE path = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectPhotoStmt.Close()

	insertSetStmt, err = db.Prepare("INSERT INTO sets (name) VALUES (?)")
	if err != nil {
		log.Fatal(err)
	}
	defer insertSetStmt.Close()

	insertPhotoStmt, err = db.Prepare(`
	INSERT INTO photos (
	aperture, camera, exposure_comp, exposure_time, flash, focal_length,
	focal_length_35, height, iso, lat, lens, lng, path, set_id, size, taken_at,
	width
	)
	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer insertPhotoStmt.Close()

	for i := 1; i < len(os.Args); i++ {
		filepath.Walk(os.Args[i], walk)
	}

	updatePhotoSiblings()
	updateSets()
}
<commit_msg>Move schema SQL to the top<commit_after>package main

import (
	"database/sql"
	"fmt"
	"image"
	_ "image/jpeg"
	"log"
	"mime"
	"os"
	"path/filepath"

	"github.com/agorf/goexif/exif"
	_ "github.com/mattn/go-sqlite3"
)

type Photo struct {
	Aperture      sql.NullFloat64
	Camera        sql.NullString
	ExposureComp  sql.NullInt64
	ExposureTime  sql.NullFloat64
	Flash         sql.NullString
	FocalLength   sql.NullFloat64
	FocalLength35 sql.NullInt64
	Height        int
	ISO           sql.NullInt64
	Lat           sql.NullFloat64
	Lens          sql.NullString
	Lng           sql.NullFloat64
	Path          string
	Size          int64
	Taken         sql.NullString
	Width         int
}

var (
	db                                                             *sql.DB
	selectSetStmt, selectPhotoStmt, insertSetStmt, insertPhotoStmt *sql.Stmt
)

var createSchemaSQL = `
CREATE TABLE IF NOT EXISTS sets (
	id integer NOT NULL PRIMARY KEY,
	thumb_photo_id integer UNIQUE REFERENCES photos,
	name varchar(4096) NOT NULL UNIQUE,
	photos_count integer,
	taken_at datetime
);

CREATE INDEX IF NOT EXISTS sets_thumb_photo_id_index ON sets (thumb_photo_id);

CREATE TABLE IF NOT EXISTS photos (
	id integer NOT NULL PRIMARY KEY,
	set_id integer NOT NULL REFERENCES sets,
	prev_photo_id integer UNIQUE REFERENCES photos,
	next_photo_id integer UNIQUE REFERENCES photos,
	path varchar(4096) NOT NULL UNIQUE,
	size integer NOT NULL,
	width integer NOT NULL,
	height integer NOT NULL,
	aperture decimal(2, 1),
	camera varchar(1000),
	exposure_comp integer,
	exposure_time decimal(9, 5),
	flash varchar(51),
	focal_length decimal(3, 1),
	focal_length_35 integer,
	iso integer,
	lat decimal(9, 6),
	lens varchar(1000),
	lng decimal(9, 6),
	taken_at datetime
);

CREATE INDEX IF NOT EXISTS photos_set_id_index ON photos (set_id);

CREATE INDEX IF NOT EXISTS photos_prev_photo_id_index ON photos (prev_photo_id);

CREATE INDEX IF NOT EXISTS photos_next_photo_id_index ON photos (next_photo_id);
`

func decodePhotoExif(photo *Photo, x *exif.Exif) {
	taken, err := x.DateTime()
	if err == nil {
		photo.Taken.String = taken.UTC().Format("2006-01-02 15:04:05")
		photo.Taken.Valid = true
	}

	lat, lng, err := x.LatLong()
	if err == nil {
		photo.Lat.Float64 = lat
		photo.Lat.Valid = true
		photo.Lng.Float64 = lng
		photo.Lng.Valid = true
	}

	orientTag, err := x.Get(exif.Orientation)
	if err == nil {
		switch orient, _ := orientTag.Int(0); orient {
		case 5, 6, 7, 8: // rotated
			photo.Width, photo.Height = photo.Height, photo.Width // swap
		}
	}

	camMakeTag, err := x.Get(exif.Make)
	if err == nil {
		photo.Camera.String, _ = camMakeTag.StringVal()
		photo.Camera.Valid = true
	}

	camModelTag, err := x.Get(exif.Model)
	if err == nil {
		cameraModel, _ := camModelTag.StringVal()

		if photo.Camera.Valid {
			photo.Camera.String = fmt.Sprint(photo.Camera.String, " ", cameraModel)
		} else {
			photo.Camera.String = cameraModel
			photo.Camera.Valid = true
		}
	}

	lensMakeTag, err := x.Get(exif.LensMake)
	if err == nil {
		photo.Lens.String, _ = lensMakeTag.StringVal()
		photo.Lens.Valid = true
	}

	lensModelTag, err := x.Get(exif.LensModel)
	if err == nil {
		lensModel, _ := lensModelTag.StringVal()

		if photo.Lens.Valid {
			photo.Lens.String = fmt.Sprint(photo.Lens.String, " ", lensModel)
		} else {
			photo.Lens.String = lensModel
			photo.Lens.Valid = true
		}
	}

	focalLenTag, err := x.Get(exif.FocalLength)
	if err == nil {
		focalLen, _ := focalLenTag.Rat(0)
		photo.FocalLength.Float64, _ = focalLen.Float64()
		photo.FocalLength.Valid = true
	}

	focalLen35Tag, err := x.Get(exif.FocalLengthIn35mmFilm)
	if err == nil {
		photo.FocalLength35.Int64, _ = focalLen35Tag.Int64(0)
		photo.FocalLength35.Valid = true
	}

	apertureTag, err := x.Get(exif.FNumber)
	if err == nil {
		aperture, _ := apertureTag.Rat(0)
		photo.Aperture.Float64, _ = aperture.Float64()
		photo.Aperture.Valid = true
	}

	expTimeTag, err := x.Get(exif.ExposureTime)
	if err == nil {
		expTime, _ := expTimeTag.Rat(0)
		photo.ExposureTime.Float64, _ = expTime.Float64()
		photo.ExposureTime.Valid = true
	}

	isoTag, err := x.Get(exif.ISOSpeedRatings)
	if err == nil {
		photo.ISO.Int64, _ = isoTag.Int64(0)
		photo.ISO.Valid = true
	}

	expBiasTag, err := x.Get(exif.ExposureBiasValue)
	if err == nil {
		photo.ExposureComp.Int64, _ = expBiasTag.Int64(0)
		photo.ExposureComp.Valid = true
	}

	flash, err := x.Flash()
	if err == nil {
		photo.Flash.String = flash
		photo.Flash.Valid = true
	}
}

func decodePhoto(path string) (*Photo, error) {
	var photo Photo

	photo.Path = path

	f, err := os.Open(path)
	if err != nil {
		return &photo, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return &photo, err
	}
	photo.Size = fi.Size()

	img, _, err := image.DecodeConfig(f)
	if err != nil {
		return &photo, err
	}
	photo.Width, photo.Height = img.Width, img.Height

	f.Seek(0, 0) // rewind

	x, err := exif.Decode(f)
	if err == nil { // EXIF data exists
		decodePhotoExif(&photo, x)
	}

	return &photo, nil
}

func storePhoto(photo *Photo) error {
	var setId, photoId int64

	setName := filepath.Base(filepath.Dir(photo.Path))
	row := selectSetStmt.QueryRow(setName)
	if err := row.Scan(&setId); err == sql.ErrNoRows { // set does not exist
		result, err := insertSetStmt.Exec(setName) // create it
		if err != nil {
			return err
		}

		setId, err = result.LastInsertId()
		if err != nil {
			return err
		}
	}

	row = selectPhotoStmt.QueryRow(photo.Path)
	if err := row.Scan(&photoId); err == sql.ErrNoRows { // photo does not exist
		result, err := insertPhotoStmt.Exec(photo.Aperture, photo.Camera,
			photo.ExposureComp, photo.ExposureTime, photo.Flash, photo.FocalLength,
			photo.FocalLength35, photo.Height, photo.ISO, photo.Lat, photo.Lens,
			photo.Lng, photo.Path, setId, photo.Size, photo.Taken, photo.Width) // create it
		if err != nil {
			return err
		}

		photoId, err = result.LastInsertId()
		if err != nil {
			return err
		}

		fmt.Fprintf(os.Stderr, "photos id=%d path=%s\n", photoId, photo.Path)
	}

	return nil
}

func walk(path string, info os.FileInfo, err error) error {
	if err != nil { // error walking "path"
		return nil // skip
	}

	if info.IsDir() {
		return nil // skip
	}

	if mime.TypeByExtension(filepath.Ext(path)) != "image/jpeg" { // not JPEG
		return nil // skip
	}

	photo, err := decodePhoto(path)
	if err == nil {
		storePhoto(photo)
	}

	return nil // next
}

func updatePhotoSiblings() error {
	var prevId, prevSetId int

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	updatePrevPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET prev_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updatePrevPhotoStmt.Close()

	updateNextPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET next_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateNextPhotoStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id FROM photos ORDER BY set_id, taken_at
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId int
		rows.Scan(&id, &setId)

		if setId == prevSetId && prevId > 0 {
			updatePrevPhotoStmt.Exec(prevId, id)
			fmt.Fprintf(os.Stderr, "photos id=%d prev_photo_id=%d\n", id, prevId)
			updateNextPhotoStmt.Exec(id, prevId)
			fmt.Fprintf(os.Stderr, "photos id=%d next_photo_id=%d\n", prevId, id)
		}

		prevId = id
		prevSetId = setId
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func updateSets() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	photosCountStmt, err := tx.Prepare(`
	SELECT COUNT(*) FROM photos WHERE set_id = ?
	`)
	if err != nil {
		return err
	}
	defer photosCountStmt.Close()

	updateSetStmt, err := tx.Prepare(`
	UPDATE sets
	SET photos_count = ?, taken_at = ?, thumb_photo_id = ?
	WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateSetStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id, MIN(taken_at) FROM photos GROUP BY set_id
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId, photosCount int
		var taken sql.NullString

		rows.Scan(&id, &setId, &taken)

		row := photosCountStmt.QueryRow(setId)
		row.Scan(&photosCount)

		updateSetStmt.Exec(photosCount, taken, id, setId)
		fmt.Fprintf(os.Stderr, "sets id=%d photos_count=%d taken=%q thumb_photo_id=%d\n", setId, photosCount, taken.String, id)
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func main() {
	var err error

	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %v path [path...]\n", os.Args[0])
		return
	}

	db, err = sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	_, err = db.Exec(createSchemaSQL)
	if err != nil {
		log.Fatal(err)
	}

	selectSetStmt, err = db.Prepare("SELECT id FROM sets WHERE name = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectSetStmt.Close()

	selectPhotoStmt, err = db.Prepare("SELECT id FROM photos WHERE path = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectPhotoStmt.Close()

	insertSetStmt, err = db.Prepare("INSERT INTO sets (name) VALUES (?)")
	if err != nil {
		log.Fatal(err)
	}
	defer insertSetStmt.Close()

	insertPhotoStmt, err = db.Prepare(`
	INSERT INTO photos (
	aperture, camera, exposure_comp, exposure_time, flash, focal_length,
	focal_length_35, height, iso, lat, lens, lng, path, set_id, size, taken_at,
	width
	)
	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer insertPhotoStmt.Close()

	for i := 1; i < len(os.Args); i++ {
		filepath.Walk(os.Args[i], walk)
	}

	updatePhotoSiblings()
	updateSets()
}
<|endoftext|>


<commit_before>package webseed

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/RoaringBitmap/roaring"
	"github.com/anacrolix/torrent/common"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/segments"
)

type RequestSpec = segments.Extent

type requestPartResult struct {
	resp *http.Response
	err  error
}

type requestPart struct {
	req    *http.Request
	e      segments.Extent
	result chan requestPartResult
}

type Request struct {
	cancel func()
	Result chan RequestResult
}

func (r Request) Cancel() {
	r.cancel()
}

type Client struct {
	HttpClient *http.Client
	Url        string
	fileIndex  segments.Index
	info       *metainfo.Info
	// The pieces we can request with the Url. We're more likely to ban/block at the file-level
	// given that's how requests are mapped to webseeds, but the torrent.Client works at the piece
	// level. We can map our file-level adjustments to the pieces here. This probably need to be
	// private in the future, if Client ever starts removing pieces.
	Pieces roaring.Bitmap
}

func (me *Client) SetInfo(info *metainfo.Info) {
	if !strings.HasSuffix(me.Url, "/") && info.IsDir() {
		// In my experience, this is a non-conforming webseed. For example the
		// http://ia600500.us.archive.org/1/items URLs in archive.org torrents.
		return
	}
	me.fileIndex = segments.NewIndex(common.LengthIterFromUpvertedFiles(info.UpvertedFiles()))
	me.info = info
	me.Pieces.AddRange(0, uint64(info.NumPieces()))
}

type RequestResult struct {
	Bytes []byte
	Err   error
}

func (ws *Client) NewRequest(r RequestSpec) Request {
	ctx, cancel := context.WithCancel(context.Background())
	var requestParts []requestPart
	if !ws.fileIndex.Locate(r, func(i int, e segments.Extent) bool {
		req, err := NewRequest(ws.Url, i, ws.info, e.Start, e.Length)
		if err != nil {
			panic(err)
		}
		req = req.WithContext(ctx)
		part := requestPart{
			req:    req,
			result: make(chan requestPartResult, 1),
			e:      e,
		}
		go func() {
			resp, err := ws.HttpClient.Do(req)
			part.result <- requestPartResult{
				resp: resp,
				err:  err,
			}
		}()
		requestParts = append(requestParts, part)
		return true
	}) {
		panic("request out of file bounds")
	}
	req := Request{
		cancel: cancel,
		Result: make(chan RequestResult, 1),
	}
	go func() {
		b, err := readRequestPartResponses(ctx, requestParts)
		req.Result <- RequestResult{
			Bytes: b,
			Err:   err,
		}
	}()
	return req
}

type ErrBadResponse struct {
	Msg      string
	Response *http.Response
}

func (me ErrBadResponse) Error() string {
	return me.Msg
}

func recvPartResult(ctx context.Context, buf io.Writer, part requestPart) error {
	result := <-part.result
	if result.err != nil {
		return result.err
	}
	defer result.resp.Body.Close()
	if ctx.Err() != nil {
		return ctx.Err()
	}
	switch result.resp.StatusCode {
	case http.StatusPartialContent:
		copied, err := io.Copy(buf, result.resp.Body)
		if err != nil {
			return err
		}
		if copied != part.e.Length {
			return fmt.Errorf("got %v bytes, expected %v", copied, part.e.Length)
		}
		return nil
	case http.StatusOK:
		// This number is based on
		// https://archive.org/download/BloodyPitOfHorror/BloodyPitOfHorror.asr.srt. It seems that
		// archive.org might be using a webserver implementation that refuses to do partial
		// responses to small files.
		if part.e.Start < 48<<10 {
			if part.e.Start != 0 {
				log.Printf("resp status ok but requested range [url=%q, range=%q]",
					part.req.URL,
					part.req.Header.Get("Range"))
			}
			discarded, _ := io.CopyN(io.Discard, result.resp.Body, part.e.Start)
			if discarded != 0 {
				log.Printf("discarded %v bytes in webseed request response part", discarded)
			}
			_, err := io.CopyN(buf, result.resp.Body, part.e.Length)
			return err
		} else {
			return ErrBadResponse{"resp status ok but requested range", result.resp}
		}
	default:
		return ErrBadResponse{
			fmt.Sprintf("unhandled response status code (%v)", result.resp.StatusCode),
			result.resp,
		}
	}
}

func readRequestPartResponses(ctx context.Context, parts []requestPart) ([]byte, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	var buf bytes.Buffer
	firstErr := make(chan error, 1)
	go func() {
		for _, part := range parts {
			err := recvPartResult(ctx, &buf, part)
			if err != nil {
				// Ensure no further unnecessary response reads occur.
				cancel()
				select {
				case firstErr <- fmt.Errorf("reading %q at %q: %w", part.req.URL, part.req.Header.Get("Range"), err):
				default:
				}
			}
		}
		select {
		case firstErr <- nil:
		default:
		}
	}()
	// This can't be merged into the return statement, because buf.Bytes is called first!
	err := <-firstErr
	return buf.Bytes(), err
}
<commit_msg>Add a comment about not discarding in webseed OK response bodies<commit_after>package webseed

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/RoaringBitmap/roaring"
	"github.com/anacrolix/torrent/common"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/segments"
)

type RequestSpec = segments.Extent

type requestPartResult struct {
	resp *http.Response
	err  error
}

type requestPart struct {
	req    *http.Request
	e      segments.Extent
	result chan requestPartResult
}

type Request struct {
	cancel func()
	Result chan RequestResult
}

func (r Request) Cancel() {
	r.cancel()
}

type Client struct {
	HttpClient *http.Client
	Url        string
	fileIndex  segments.Index
	info       *metainfo.Info
	// The pieces we can request with the Url. We're more likely to ban/block at the file-level
	// given that's how requests are mapped to webseeds, but the torrent.Client works at the piece
	// level. We can map our file-level adjustments to the pieces here. This probably need to be
	// private in the future, if Client ever starts removing pieces.
	Pieces roaring.Bitmap
}

func (me *Client) SetInfo(info *metainfo.Info) {
	if !strings.HasSuffix(me.Url, "/") && info.IsDir() {
		// In my experience, this is a non-conforming webseed. For example the
		// http://ia600500.us.archive.org/1/items URLs in archive.org torrents.
		return
	}
	me.fileIndex = segments.NewIndex(common.LengthIterFromUpvertedFiles(info.UpvertedFiles()))
	me.info = info
	me.Pieces.AddRange(0, uint64(info.NumPieces()))
}

type RequestResult struct {
	Bytes []byte
	Err   error
}

func (ws *Client) NewRequest(r RequestSpec) Request {
	ctx, cancel := context.WithCancel(context.Background())
	var requestParts []requestPart
	if !ws.fileIndex.Locate(r, func(i int, e segments.Extent) bool {
		req, err := NewRequest(ws.Url, i, ws.info, e.Start, e.Length)
		if err != nil {
			panic(err)
		}
		req = req.WithContext(ctx)
		part := requestPart{
			req:    req,
			result: make(chan requestPartResult, 1),
			e:      e,
		}
		go func() {
			resp, err := ws.HttpClient.Do(req)
			part.result <- requestPartResult{
				resp: resp,
				err:  err,
			}
		}()
		requestParts = append(requestParts, part)
		return true
	}) {
		panic("request out of file bounds")
	}
	req := Request{
		cancel: cancel,
		Result: make(chan RequestResult, 1),
	}
	go func() {
		b, err := readRequestPartResponses(ctx, requestParts)
		req.Result <- RequestResult{
			Bytes: b,
			Err:   err,
		}
	}()
	return req
}

type ErrBadResponse struct {
	Msg      string
	Response *http.Response
}

func (me ErrBadResponse) Error() string {
	return me.Msg
}

func recvPartResult(ctx context.Context, buf io.Writer, part requestPart) error {
	result := <-part.result
	if result.err != nil {
		return result.err
	}
	defer result.resp.Body.Close()
	if ctx.Err() != nil {
		return ctx.Err()
	}
	switch result.resp.StatusCode {
	case http.StatusPartialContent:
		copied, err := io.Copy(buf, result.resp.Body)
		if err != nil {
			return err
		}
		if copied != part.e.Length {
			return fmt.Errorf("got %v bytes, expected %v", copied, part.e.Length)
		}
		return nil
	case http.StatusOK:
		// This number is based on
		// https://archive.org/download/BloodyPitOfHorror/BloodyPitOfHorror.asr.srt. It seems that
		// archive.org might be using a webserver implementation that refuses to do partial
		// responses to small files.
		if part.e.Start < 48<<10 {
			if part.e.Start != 0 {
				log.Printf("resp status ok but requested range [url=%q, range=%q]",
					part.req.URL,
					part.req.Header.Get("Range"))
			}
			// Instead of discarding, we could try receiving all the chunks present in the response
			// body. I don't know how one would handle multiple chunk requests resulting in an OK
			// response for the same file. The request algorithm might be need to be smarter for
			// that.
			discarded, _ := io.CopyN(io.Discard, result.resp.Body, part.e.Start)
			if discarded != 0 {
				log.Printf("discarded %v bytes in webseed request response part", discarded)
			}
			_, err := io.CopyN(buf, result.resp.Body, part.e.Length)
			return err
		} else {
			return ErrBadResponse{"resp status ok but requested range", result.resp}
		}
	default:
		return ErrBadResponse{
			fmt.Sprintf("unhandled response status code (%v)", result.resp.StatusCode),
			result.resp,
		}
	}
}

func readRequestPartResponses(ctx context.Context, parts []requestPart) ([]byte, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	var buf bytes.Buffer
	firstErr := make(chan error, 1)
	go func() {
		for _, part := range parts {
			err := recvPartResult(ctx, &buf, part)
			if err != nil {
				// Ensure no further unnecessary response reads occur.
				cancel()
				select {
				case firstErr <- fmt.Errorf("reading %q at %q: %w", part.req.URL, part.req.Header.Get("Range"), err):
				default:
				}
			}
		}
		select {
		case firstErr <- nil:
		default:
		}
	}()
	// This can't be merged into the return statement, because buf.Bytes is called first!
	err := <-firstErr
	return buf.Bytes(), err
}
<|endoftext|>


<commit_before>// Copyright 2015 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
	"github.com/tsuru/tsuru/log"
)

const version = "0.6.0"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()
	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}
	log.Debugf("Opening config file: %s ...\n", *configFile)
	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not open gandalf config file at %s (%s).
  For an example, see: gandalf/etc/gandalf.conf
  Note that you can specify a different config file with the --config option -- e.g.: --config=./etc/gandalf.conf`
		log.Fatalf(msg, *configFile, err)
	}
	log.Init()
	log.Debugf("Successfully read config file: %s\n", *configFile)
	router := api.SetupRouter()
	n := negroni.New()
	n.Use(api.NewLoggerMiddleware())
	n.Use(api.NewResponseHeaderMiddleware("Server", "gandalf-webserver/"+version))
	n.Use(api.NewResponseHeaderMiddleware("Cache-Control", "private, max-age=0"))
	n.Use(api.NewResponseHeaderMiddleware("Expires", "-1"))
	n.UseHandler(router)
	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		bareLocation, err := config.GetString("git:bare:location")
		if err != nil {
			panic("You should configure a git:bare:location for gandalf.")
		}
		fmt.Printf("Repository location: %s\n", bareLocation)
		fmt.Printf("gandalf-webserver %s listening on %s\n", version, bind)
		http.ListenAndServe(bind, router)
	}
}
<commit_msg>Release 0.7.0<commit_after>// Copyright 2015 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
	"github.com/tsuru/tsuru/log"
)

const version = "0.7.0"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()
	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}
	log.Debugf("Opening config file: %s ...\n", *configFile)
	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not open gandalf config file at %s (%s).
  For an example, see: gandalf/etc/gandalf.conf
  Note that you can specify a different config file with the --config option -- e.g.: --config=./etc/gandalf.conf`
		log.Fatalf(msg, *configFile, err)
	}
	log.Init()
	log.Debugf("Successfully read config file: %s\n", *configFile)
	router := api.SetupRouter()
	n := negroni.New()
	n.Use(api.NewLoggerMiddleware())
	n.Use(api.NewResponseHeaderMiddleware("Server", "gandalf-webserver/"+version))
	n.Use(api.NewResponseHeaderMiddleware("Cache-Control", "private, max-age=0"))
	n.Use(api.NewResponseHeaderMiddleware("Expires", "-1"))
	n.UseHandler(router)
	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		bareLocation, err := config.GetString("git:bare:location")
		if err != nil {
			panic("You should configure a git:bare:location for gandalf.")
		}
		fmt.Printf("Repository location: %s\n", bareLocation)
		fmt.Printf("gandalf-webserver %s listening on %s\n", version, bind)
		http.ListenAndServe(bind, router)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
)

const version = "0.5.2"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()
	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}
	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not find gandalf config file. Searched on %s.
For an example conf check gandalf/etc/gandalf.conf file.\n %s`
		log.Panicf(msg, *configFile, err)
	}
	router := api.SetupRouter()
	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(bind, router))
	}
}
<commit_msg>Log ip/port gandalf is listening on<commit_after>// Copyright 2014 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
)

const version = "0.5.2"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()
	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}
	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not find gandalf config file. Searched on %s.
For an example conf check gandalf/etc/gandalf.conf file.\n %s`
		log.Panicf(msg, *configFile, err)
	}
	router := api.SetupRouter()
	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		bareLocation, err := config.GetString("git:bare:location")
		if err != nil {
			panic("You should configure a git:bare:location for gandalf.")
		}
		log.Printf("Repository location: %s\n", bareLocation)
		log.Printf("gandalf-webserver %s listening on %s\n", version, bind)
		log.Fatal(http.ListenAndServe(bind, router))
	}
}
<|endoftext|>


<commit_before>package arn

import (
	"errors"

	"sort"

	"github.com/aerogo/nano"
	"github.com/fatih/color"
)

// Quote is a quote made by a character in an anime.
type Quote struct {
	Text          QuoteText `json:"text" editable:"true"`
	CharacterID   string    `json:"characterId" editable:"true"`
	AnimeID       string    `json:"animeId" editable:"true"`
	EpisodeNumber int       `json:"episode" editable:"true"`
	Time          int       `json:"time" editable:"true"`

	HasID
	HasCreator
	HasEditor
	HasLikes
	HasDraft
}

// IsMainQuote returns true if the quote is the main quote of the character.
func (quote *Quote) IsMainQuote() bool {
	return quote.CharacterID != "" && quote.Character().MainQuoteID == quote.ID
}

// Link returns a single quote.
func (quote *Quote) Link() string {
	return "/quote/" + quote.ID
}

// Publish checks the quote and publishes it when no errors were found.
func (quote *Quote) Publish() error {
	// No quote text
	if quote.Text.English == "" {
		return errors.New("English quote text is required")
	}

	// No character
	if quote.CharacterID == "" {
		return errors.New("A character is required")
	}

	// No anime
	if quote.AnimeID == "" {
		return errors.New("An anime is required")
	}

	// // No episode number
	// if quote.EpisodeNumber == -1 {
	// 	return errors.New("An episode number is required")
	// }

	// // No time
	// if quote.Time == -1 {
	// 	return errors.New("Time in minutes is required")
	// }

	// Invalid anime ID
	anime := quote.Anime()

	if anime == nil {
		return errors.New("Invalid anime ID")
	}

	// Invalid episode number
	maxEpisodes := anime.EpisodeCount

	if maxEpisodes != 0 && quote.EpisodeNumber > maxEpisodes {
		return errors.New("Invalid episode number")
	}

	return publish(quote)
}

// Unpublish ...
func (quote *Quote) Unpublish() error {
	return unpublish(quote)
}

// OnLike is called when the quote receives a like.
func (quote *Quote) OnLike(likedBy *User) {
	if likedBy.ID == quote.CreatedBy {
		return
	}

	if !quote.Creator().Settings().Notification.QuoteLikes {
		return
	}

	go func() {
		quote.Creator().SendNotification(&PushNotification{
			Title:   likedBy.Nick + " liked your " + quote.Character().Name.Canonical + " quote",
			Message: quote.Text.English,
			Icon:    "https:" + likedBy.AvatarLink("large"),
			Link:    "https://notify.moe" + likedBy.Link(),
			Type:    NotificationTypeLike,
		})
	}()
}

// String implements the default string serialization.
func (quote *Quote) String() string {
	return quote.Text.English
}

// GetQuote returns a single quote.
func GetQuote(id string) (*Quote, error) {
	obj, err := DB.Get("Quote", id)

	if err != nil {
		return nil, err
	}

	return obj.(*Quote), nil
}

// StreamQuotes returns a stream of all quotes.
func StreamQuotes() chan *Quote {
	channel := make(chan *Quote, nano.ChannelBufferSize)

	go func() {
		for obj := range DB.All("Quote") {
			channel <- obj.(*Quote)
		}

		close(channel)
	}()

	return channel
}

// AllQuotes returns a slice of all quotes.
func AllQuotes() []*Quote {
	var all []*Quote

	stream := StreamQuotes()

	for obj := range stream {
		all = append(all, obj)
	}

	return all
}

// Character returns the character cited in the quote
func (quote *Quote) Character() *Character {
	character, _ := GetCharacter(quote.CharacterID)
	return character
}

// Anime fetches the anime where the quote is said.
func (quote *Quote) Anime() *Anime {
	anime, err := GetAnime(quote.AnimeID)

	if err != nil {
		color.Red("Error fetching anime: %v", err)
	}

	return anime
}

// SortQuotesLatestFirst ...
func SortQuotesLatestFirst(quotes []*Quote) {
	sort.Slice(quotes, func(i, j int) bool {
		return quotes[i].Created > quotes[j].Created
	})
}

// SortQuotesPopularFirst ...
func SortQuotesPopularFirst(quotes []*Quote) {
	sort.Slice(quotes, func(i, j int) bool {
		aLikes := len(quotes[i].Likes)
		bLikes := len(quotes[j].Likes)

		if aLikes == bLikes {
			return quotes[i].Created > quotes[j].Created
		}

		return aLikes > bLikes
	})
}

// FilterQuotes filters all quotes by a custom function.
func FilterQuotes(filter func(*Quote) bool) []*Quote {
	var filtered []*Quote

	for obj := range StreamQuotes() {
		if filter(obj) {
			filtered = append(filtered, obj)
		}
	}

	return filtered
}
<commit_msg>Add check for quote values<commit_after>package arn

import (
	"errors"

	"sort"

	"github.com/aerogo/nano"
	"github.com/fatih/color"
)

// Quote is a quote made by a character in an anime.
type Quote struct {
	Text          QuoteText `json:"text" editable:"true"`
	CharacterID   string    `json:"characterId" editable:"true"`
	AnimeID       string    `json:"animeId" editable:"true"`
	EpisodeNumber int       `json:"episode" editable:"true"`
	Time          int       `json:"time" editable:"true"`

	HasID
	HasCreator
	HasEditor
	HasLikes
	HasDraft
}

// IsMainQuote returns true if the quote is the main quote of the character.
func (quote *Quote) IsMainQuote() bool {
	return quote.CharacterID != "" && quote.Character().MainQuoteID == quote.ID
}

// Link returns a single quote.
func (quote *Quote) Link() string {
	return "/quote/" + quote.ID
}

// Publish checks the quote and publishes it when no errors were found.
func (quote *Quote) Publish() error {
	// No quote text
	if quote.Text.English == "" {
		return errors.New("English quote text is required")
	}

	// No character
	if quote.CharacterID == "" {
		return errors.New("A character is required")
	}

	// No anime
	if quote.AnimeID == "" {
		return errors.New("An anime is required")
	}

	// // No episode number
	// if quote.EpisodeNumber == -1 {
	// 	return errors.New("An episode number is required")
	// }

	// // No time
	// if quote.Time == -1 {
	// 	return errors.New("Time in minutes is required")
	// }

	// Invalid anime ID
	anime := quote.Anime()

	if anime == nil {
		return errors.New("Invalid anime ID")
	}

	// Invalid episode number
	maxEpisodes := anime.EpisodeCount

	if maxEpisodes != 0 && quote.EpisodeNumber > maxEpisodes {
		return errors.New("Invalid episode number")
	}

	return publish(quote)
}

// Unpublish ...
func (quote *Quote) Unpublish() error {
	return unpublish(quote)
}

// OnLike is called when the quote receives a like.
func (quote *Quote) OnLike(likedBy *User) {
	if !quote.IsValid() {
		color.Red("Invalid quote: %s", quote.ID)
		return
	}

	if likedBy.ID == quote.CreatedBy {
		return
	}

	if !quote.Creator().Settings().Notification.QuoteLikes {
		return
	}

	go func() {
		quote.Creator().SendNotification(&PushNotification{
			Title:   likedBy.Nick + " liked your " + quote.Character().Name.Canonical + " quote",
			Message: quote.Text.English,
			Icon:    "https:" + likedBy.AvatarLink("large"),
			Link:    "https://notify.moe" + likedBy.Link(),
			Type:    NotificationTypeLike,
		})
	}()
}

// IsValid tests the field values and returns true if everything is okay.
func (quote *Quote) IsValid() bool {
	return quote.Character() != nil && quote.Anime() != nil && quote.Creator() != nil
}

// String implements the default string serialization.
func (quote *Quote) String() string {
	return quote.Text.English
}

// GetQuote returns a single quote.
func GetQuote(id string) (*Quote, error) {
	obj, err := DB.Get("Quote", id)

	if err != nil {
		return nil, err
	}

	return obj.(*Quote), nil
}

// StreamQuotes returns a stream of all quotes.
func StreamQuotes() chan *Quote {
	channel := make(chan *Quote, nano.ChannelBufferSize)

	go func() {
		for obj := range DB.All("Quote") {
			channel <- obj.(*Quote)
		}

		close(channel)
	}()

	return channel
}

// AllQuotes returns a slice of all quotes.
func AllQuotes() []*Quote {
	var all []*Quote

	stream := StreamQuotes()

	for obj := range stream {
		all = append(all, obj)
	}

	return all
}

// Character returns the character cited in the quote
func (quote *Quote) Character() *Character {
	character, _ := GetCharacter(quote.CharacterID)
	return character
}

// Anime fetches the anime where the quote is said.
func (quote *Quote) Anime() *Anime {
	anime, err := GetAnime(quote.AnimeID)

	if err != nil {
		color.Red("Error fetching anime: %v", err)
	}

	return anime
}

// SortQuotesLatestFirst ...
func SortQuotesLatestFirst(quotes []*Quote) {
	sort.Slice(quotes, func(i, j int) bool {
		return quotes[i].Created > quotes[j].Created
	})
}

// SortQuotesPopularFirst ...
func SortQuotesPopularFirst(quotes []*Quote) {
	sort.Slice(quotes, func(i, j int) bool {
		aLikes := len(quotes[i].Likes)
		bLikes := len(quotes[j].Likes)

		if aLikes == bLikes {
			return quotes[i].Created > quotes[j].Created
		}

		return aLikes > bLikes
	})
}

// FilterQuotes filters all quotes by a custom function.
func FilterQuotes(filter func(*Quote) bool) []*Quote {
	var filtered []*Quote

	for obj := range StreamQuotes() {
		if filter(obj) {
			filtered = append(filtered, obj)
		}
	}

	return filtered
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"template"
)

const defaultAddr = ":31798" // default webserver address

var (
	httpAddr            = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')")
	root                = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl")
	gitwebScript        = flag.String("gitwebscript", "/usr/lib/cgi-bin/gitweb.cgi", "Path to gitweb.cgi, or blank to disable.")
	gitwebFiles         = flag.String("gitwebfiles", "/usr/share/gitweb", "Path to gitweb's static files.")
	pageHtml, errorHtml *template.Template
)

var fmap = template.FormatterMap{
	"":         textFmt,
	"html":     htmlFmt,
	"html-esc": htmlEscFmt,
}

// Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, false, x[0])
}

// Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, true, x[0])
}

// Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
	var buf bytes.Buffer
	writeAny(&buf, false, x[0])
	template.HTMLEscape(w, buf.Bytes())
}

// Write anything to w; optionally html-escaped.
func writeAny(w io.Writer, html bool, x interface{}) {
	switch v := x.(type) {
	case []byte:
		writeText(w, v, html)
	case string:
		writeText(w, []byte(v), html)
	default:
		if html {
			var buf bytes.Buffer
			fmt.Fprint(&buf, x)
			writeText(w, buf.Bytes(), true)
		} else {
			fmt.Fprint(w, x)
		}
	}
}

// Write text to w; optionally html-escaped.
func writeText(w io.Writer, text []byte, html bool) {
	if html {
		template.HTMLEscape(w, text)
		return
	}
	w.Write(text)
}

func applyTemplate(t *template.Template, name string, data interface{}) []byte {
	var buf bytes.Buffer
	if err := t.Execute(data, &buf); err != nil {
		log.Printf("%s.Execute: %s", name, err)
	}
	return buf.Bytes()
}

func servePage(w http.ResponseWriter, title, subtitle string, content []byte) {
	d := struct {
		Title    string
		Subtitle string
		Content  []byte
	}{
		title,
		subtitle,
		content,
	}

	if err := pageHtml.Execute(&d, w); err != nil {
		log.Printf("godocHTML.Execute: %s", err)
	}
}

func readTemplate(name string) *template.Template {
	fileName := path.Join(*root, "tmpl", name)
	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Exitf("ReadFile %s: %v", fileName, err)
	}
	t, err := template.Parse(string(data), fmap)
	if err != nil {
		log.Exitf("%s: %v", fileName, err)
	}
	return t
}

func readTemplates() {
	pageHtml = readTemplate("page.html")
	errorHtml = readTemplate("error.html")
}

func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path!
	w.WriteHeader(http.StatusNotFound)
	servePage(w, "File "+relpath, "", contents)
}

func mainHandler(rw http.ResponseWriter, req *http.Request) {
	relPath := req.URL.Path[1:] // serveFile URL paths start with '/'
	if strings.Contains(relPath, "..") {
		return
	}
	if relPath == "" {
		relPath = "index.html"
	}
	absPath := path.Join(*root, "content", relPath)

	fi, err := os.Lstat(absPath)
	if err != nil {
		log.Print(err)
		serveError(rw, req, relPath, err)
		return
	}

	switch {
	case fi.IsRegular():
		serveFile(rw, req, relPath, absPath)
	}
}

func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) {
	data, err := ioutil.ReadFile(absPath)
	if err != nil {
		serveError(rw, req, absPath, err)
		return
	}
	servePage(rw, "", "", []byte(data))
}

type gitwebHandler struct {
	Cgi    http.Handler
	Static http.Handler
}

func (h *gitwebHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if r.URL.RawPath == "/code/" ||
		strings.HasPrefix(r.URL.RawPath, "/code/?") {
		h.Cgi.ServeHTTP(rw, r)
	} else {
		h.Static.ServeHTTP(rw, r)
	}
}

func main() {
	flag.Parse()
	readTemplates()

	if *root == "" {
		var err os.Error
		*root, err = os.Getwd()
		if err != nil {
			log.Exitf("Failed to getwd: %v", err)
		}
	}

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/static/", http.FileServer(path.Join(*root, "static"), "/static/"))

	testCgi := &CgiHandler{ExecutablePath: path.Join(*root, "test.cgi"),
		Root: "/test.cgi",
	}
	mux.Handle("/test.cgi", testCgi)
	mux.Handle("/test.cgi/foo", testCgi)

	mux.Handle("/code", http.RedirectHandler("/code/", http.StatusFound))
	if *gitwebScript != "" {
		env := os.Environ()
		env = append(env, "GITWEB_CONFIG="+path.Join(*root, "gitweb-camli.conf"))
		env = append(env, "CAMWEB_ROOT="+path.Join(*root))
		mux.Handle("/code/", &gitwebHandler{
			Cgi: &CgiHandler{
				ExecutablePath: *gitwebScript,
				Root:           "/code/",
				Environ:        env,
			},
			Static: http.FileServer(*gitwebFiles, "/code/"),
		})
	}
	mux.HandleFunc("/", mainHandler)

	if err := http.ListenAndServe(*httpAddr, mux); err != nil {
		log.Exitf("ListenAndServe %s: %v", *httpAddr, err)
	}
}
<commit_msg>Start of stripping www.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"template"
)

const defaultAddr = ":31798" // default webserver address

var (
	httpAddr            = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')")
	root                = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl")
	gitwebScript        = flag.String("gitwebscript", "/usr/lib/cgi-bin/gitweb.cgi", "Path to gitweb.cgi, or blank to disable.")
	gitwebFiles         = flag.String("gitwebfiles", "/usr/share/gitweb", "Path to gitweb's static files.")
	pageHtml, errorHtml *template.Template
)

var fmap = template.FormatterMap{
	"":         textFmt,
	"html":     htmlFmt,
	"html-esc": htmlEscFmt,
}

// Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, false, x[0])
}

// Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) {
	writeAny(w, true, x[0])
}

// Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
	var buf bytes.Buffer
	writeAny(&buf, false, x[0])
	template.HTMLEscape(w, buf.Bytes())
}

// Write anything to w; optionally html-escaped.
func writeAny(w io.Writer, html bool, x interface{}) {
	switch v := x.(type) {
	case []byte:
		writeText(w, v, html)
	case string:
		writeText(w, []byte(v), html)
	default:
		if html {
			var buf bytes.Buffer
			fmt.Fprint(&buf, x)
			writeText(w, buf.Bytes(), true)
		} else {
			fmt.Fprint(w, x)
		}
	}
}

// Write text to w; optionally html-escaped.
func writeText(w io.Writer, text []byte, html bool) {
	if html {
		template.HTMLEscape(w, text)
		return
	}
	w.Write(text)
}

func applyTemplate(t *template.Template, name string, data interface{}) []byte {
	var buf bytes.Buffer
	if err := t.Execute(data, &buf); err != nil {
		log.Printf("%s.Execute: %s", name, err)
	}
	return buf.Bytes()
}

func servePage(w http.ResponseWriter, title, subtitle string, content []byte) {
	d := struct {
		Title    string
		Subtitle string
		Content  []byte
	}{
		title,
		subtitle,
		content,
	}

	if err := pageHtml.Execute(&d, w); err != nil {
		log.Printf("godocHTML.Execute: %s", err)
	}
}

func readTemplate(name string) *template.Template {
	fileName := path.Join(*root, "tmpl", name)
	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Exitf("ReadFile %s: %v", fileName, err)
	}
	t, err := template.Parse(string(data), fmap)
	if err != nil {
		log.Exitf("%s: %v", fileName, err)
	}
	return t
}

func readTemplates() {
	pageHtml = readTemplate("page.html")
	errorHtml = readTemplate("error.html")
}

func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path!
	w.WriteHeader(http.StatusNotFound)
	servePage(w, "File "+relpath, "", contents)
}

func mainHandler(rw http.ResponseWriter, req *http.Request) {
	relPath := req.URL.Path[1:] // serveFile URL paths start with '/'
	if strings.Contains(relPath, "..") {
		return
	}
	if relPath == "" {
		relPath = "index.html"
	}
	absPath := path.Join(*root, "content", relPath)

	fi, err := os.Lstat(absPath)
	if err != nil {
		log.Print(err)
		serveError(rw, req, relPath, err)
		return
	}

	switch {
	case fi.IsRegular():
		serveFile(rw, req, relPath, absPath)
	}
}

func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) {
	data, err := ioutil.ReadFile(absPath)
	if err != nil {
		serveError(rw, req, absPath, err)
		return
	}
	servePage(rw, "", "", []byte(data))
}

type gitwebHandler struct {
	Cgi    http.Handler
	Static http.Handler
}

func (h *gitwebHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if r.URL.RawPath == "/code/" ||
		strings.HasPrefix(r.URL.RawPath, "/code/?") {
		h.Cgi.ServeHTTP(rw, r)
	} else {
		h.Static.ServeHTTP(rw, r)
	}
}

type noWwwHandler struct {
	Handler http.Handler
}

func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	// TODO: strip www.
	h.Handler.ServeHTTP(rw, r)
}

func main() {
	flag.Parse()
	readTemplates()

	if *root == "" {
		var err os.Error
		*root, err = os.Getwd()
		if err != nil {
			log.Exitf("Failed to getwd: %v", err)
		}
	}

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(path.Join(*root, "static"), "/"))
	mux.Handle("/static/", http.FileServer(path.Join(*root, "static"), "/static/"))

	testCgi := &CgiHandler{ExecutablePath: path.Join(*root, "test.cgi"),
		Root: "/test.cgi",
	}
	mux.Handle("/test.cgi", testCgi)
	mux.Handle("/test.cgi/foo", testCgi)

	mux.Handle("/code", http.RedirectHandler("/code/", http.StatusFound))
	if *gitwebScript != "" {
		env := os.Environ()
		env = append(env, "GITWEB_CONFIG="+path.Join(*root, "gitweb-camli.conf"))
		env = append(env, "CAMWEB_ROOT="+path.Join(*root))
		mux.Handle("/code/", &gitwebHandler{
			Cgi: &CgiHandler{
				ExecutablePath: *gitwebScript,
				Root:           "/code/",
				Environ:        env,
			},
			Static: http.FileServer(*gitwebFiles, "/code/"),
		})
	}
	mux.HandleFunc("/", mainHandler)

	if err := http.ListenAndServe(*httpAddr, &noWwwHandler{Handler: mux}); err != nil {
		log.Exitf("ListenAndServe %s: %v", *httpAddr, err)
	}
}
<|endoftext|>