<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package https provides a helper for starting an HTTPS server.
package https // import "upspin.io/cloud/https"

import (
	"crypto/tls"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/crypto/acme/autocert"
	gContext "golang.org/x/net/context"
	"google.golang.org/api/option"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/storage"

	"upspin.io/access"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/shutdown"
)

// Options permits the configuration of TLS certificates for servers running
// outside GCE. The default is the self-signed certificate in
// upspin.io/rpc/testdata.
type Options struct {
	// LetsEncryptCache specifies the cache file for Let's Encrypt.
	// If non-empty, enables Let's Encrypt certificates for this server.
	LetsEncryptCache string

	// LetsEncryptHosts specifies the list of hosts for which we should
	// obtain TLS certificates through Let's Encrypt. If LetsEncryptCache
	// is specified this should be specified also.
	LetsEncryptHosts []string

	// CertFile and KeyFile specifies the TLS certificates to use.
	// It has no effect if LetsEncryptCache is non-empty.
	CertFile string
	KeyFile  string

	// InsecureHTTP specifies whether to serve insecure HTTP without TLS.
	// An error occurs if this is attempted with a non-loopback address.
	InsecureHTTP bool
}

var defaultOptions = &Options{
	CertFile: filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/cert.pem"),
	KeyFile:  filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/key.pem"),
}

func (opt *Options) applyDefaults() {
	if opt.CertFile == "" {
		opt.CertFile = defaultOptions.CertFile
	}
	if opt.KeyFile == "" {
		opt.KeyFile = defaultOptions.KeyFile
	}
}

// ListenAndServe serves the http.DefaultServeMux by HTTPS (and HTTP,
// redirecting to HTTPS), storing SSL credentials in the Google Cloud Storage
// buckets letsencrypt*.
//
// If the server is running outside GCE, instead an HTTPS server is started on
// the address specified by addr using the certificate details specified by opt.
//
// The given channel, if any, is closed when the TCP listener has succeeded.
// It may be used to signal that the server is ready to start serving requests.
//
// ListenAndServe does not return. It exits the program when the server is
// shut down (via SIGTERM or due to an error) and calls shutdown.Shutdown.
func ListenAndServe(ready chan<- struct{}, serverName, addr string, opt *Options) {
	if opt == nil {
		opt = defaultOptions
	} else {
		opt.applyDefaults()
	}

	var m autocert.Manager
	m.Prompt = autocert.AcceptTOS
	if h := opt.LetsEncryptHosts; len(h) > 0 {
		m.HostPolicy = autocert.HostWhitelist(h...)
	}

	var config *tls.Config
	if opt.InsecureHTTP {
		log.Info.Printf("https: serving insecure HTTP on %q", addr)
		host, _, err := net.SplitHostPort(addr)
		if err != nil {
			log.Fatalf("https: couldn't parse address: %v", err)
		}
		if host != "localhost" && host != "127.0.0.1" && host != "::1" {
			log.Fatalf("https: cannot serve insecure HTTP on non-loopback address %q", addr)
		}
	} else if file := opt.LetsEncryptCache; file != "" {
		log.Info.Printf("https: serving HTTPS on %q using Let's Encrypt certificates", addr)
		m.Cache = autocert.DirCache(file)
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else if metadata.OnGCE() {
		addr = ":443"
		log.Info.Printf("https: serving HTTPS on GCE %q using Let's Encrypt certificates", addr)
		const key = "letsencrypt-bucket"
		bucket, err := metadata.InstanceAttributeValue(key)
		if err != nil {
			log.Fatalf("https: couldn't read %q metadata value: %v", key, err)
		}
		cache, err := newAutocertCache(bucket, serverName)
		if err != nil {
			log.Fatalf("https: couldn't set up letsencrypt cache: %v", err)
		}
		m.Cache = cache
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else {
		log.Info.Printf("https: not on GCE; serving HTTPS on %q using provided certificates", addr)
		if opt.CertFile == defaultOptions.CertFile || opt.KeyFile == defaultOptions.KeyFile {
			log.Error.Print("https: WARNING: using self-signed test certificates.")
		}
		var err error
		config, err = newDefaultTLSConfig(opt.CertFile, opt.KeyFile)
		if err != nil {
			log.Fatalf("https: setting up TLS config: %v", err)
		}
	}
	// WriteTimeout is set to 0 because it also pertains to streaming
	// replies, e.g., the DirServer.Watch interface.
	server := &http.Server{
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       15 * time.Second,
		WriteTimeout:      0,
		IdleTimeout:       60 * time.Second,
		TLSConfig:         config,
	}
	// TODO(adg): enable HTTP/2 once it's fast enough
	//err := http2.ConfigureServer(server, nil)
	//if err != nil {
	//	log.Fatalf("https: %v", err)
	//}

	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("https: %v", err)
	}
	if ready != nil {
		close(ready)
	}
	shutdown.Handle(func() {
		// Stop accepting connections and forces the server to stop
		// its serving loop.
		ln.Close()
	})
	if !opt.InsecureHTTP {
		ln = tls.NewListener(ln, config)
	}
	err = server.Serve(ln)
	log.Printf("https: %v", err)
	shutdown.Now(1)
}

// ListenAndServeFromFlags is the same as ListenAndServe, but it determines the
// listen address and Options from command-line flags in the flags package.
func ListenAndServeFromFlags(ready chan<- struct{}, serverName string) {
	addr := flags.HTTPSAddr
	if flags.InsecureHTTP {
		addr = flags.HTTPAddr
	}
	ListenAndServe(ready, serverName, addr, &Options{
		LetsEncryptCache: flags.LetsEncryptCache,
		CertFile:         flags.TLSCertFile,
		KeyFile:          flags.TLSKeyFile,
		InsecureHTTP:     flags.InsecureHTTP,
	})
}

// newDefaultTLSConfig creates a new TLS config based on the certificate files given.
func newDefaultTLSConfig(certFile string, certKeyFile string) (*tls.Config, error) {
	const op = "cloud/https.newDefaultTLSConfig"
	certReadable, err := isReadableFile(certFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL certificate in %q: %q", certFile, err))
	}
	if !certReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate file %q not readable", certFile))
	}
	keyReadable, err := isReadableFile(certKeyFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL key in %q: %v", certKeyFile, err))
	}
	if !keyReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate key file %q not readable", certKeyFile))
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		return nil, errors.E(op, err)
	}

	tlsConfig := &tls.Config{
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		},
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true, // Use our choice, not the client's choice
		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519},
		Certificates:             []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()
	return tlsConfig, nil
}

// isReadableFile reports whether the file exists and is readable.
// If the error is non-nil, it means there might be a file or directory
// with that name but we cannot read it.
func isReadableFile(path string) (bool, error) {
	// Is it stattable and is it a plain file?
	info, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil // Item does not exist.
		}
		return false, err // Item is problematic.
	}
	if info.IsDir() {
		return false, errors.Str("is directory")
	}
	// Is it readable?
	fd, err := os.Open(path)
	if err != nil {
		return false, access.ErrPermissionDenied
	}
	fd.Close()
	return true, nil // Item exists and is readable.
}

// autocertCache implements autocert.Cache.
type autocertCache struct {
	b      *storage.BucketHandle
	server string
}

func newAutocertCache(bucket, prefix string) (cache autocertCache, err error) {
	ctx := gContext.Background()
	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl))
	if err != nil {
		return
	}
	cache.b = client.Bucket(bucket)
	cache.server = prefix + "-"
	return
}

func (cache autocertCache) Get(ctx gContext.Context, name string) ([]byte, error) {
	r, err := cache.b.Object(cache.server + name).NewReader(ctx)
	if err == storage.ErrObjectNotExist {
		return nil, autocert.ErrCacheMiss
	}
	if err != nil {
		return nil, err
	}
	defer r.Close()
	return ioutil.ReadAll(r)
}

func (cache autocertCache) Put(ctx gContext.Context, name string, data []byte) error {
	// TODO(ehg) Do we need to add contentType="text/plain; charset=utf-8"?
	w := cache.b.Object(cache.server + name).NewWriter(ctx)
	_, err := w.Write(data)
	if err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	if err := w.Close(); err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	return err
}

func (cache autocertCache) Delete(ctx gContext.Context, name string) error {
	return cache.b.Object(cache.server + name).Delete(ctx)
}
<commit_msg>cloud/https: update to track changes to x/crypto/acme/autocert<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package https provides a helper for starting an HTTPS server.
package https // import "upspin.io/cloud/https"

import (
	"context"
	"crypto/tls"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"golang.org/x/crypto/acme/autocert"
	"google.golang.org/api/option"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/storage"

	"upspin.io/access"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/shutdown"
)

// Options permits the configuration of TLS certificates for servers running
// outside GCE. The default is the self-signed certificate in
// upspin.io/rpc/testdata.
type Options struct {
	// LetsEncryptCache specifies the cache file for Let's Encrypt.
	// If non-empty, enables Let's Encrypt certificates for this server.
	LetsEncryptCache string

	// LetsEncryptHosts specifies the list of hosts for which we should
	// obtain TLS certificates through Let's Encrypt. If LetsEncryptCache
	// is specified this should be specified also.
	LetsEncryptHosts []string

	// CertFile and KeyFile specifies the TLS certificates to use.
	// It has no effect if LetsEncryptCache is non-empty.
	CertFile string
	KeyFile  string

	// InsecureHTTP specifies whether to serve insecure HTTP without TLS.
	// An error occurs if this is attempted with a non-loopback address.
	InsecureHTTP bool
}

var defaultOptions = &Options{
	CertFile: filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/cert.pem"),
	KeyFile:  filepath.Join(os.Getenv("GOPATH"), "/src/upspin.io/rpc/testdata/key.pem"),
}

func (opt *Options) applyDefaults() {
	if opt.CertFile == "" {
		opt.CertFile = defaultOptions.CertFile
	}
	if opt.KeyFile == "" {
		opt.KeyFile = defaultOptions.KeyFile
	}
}

// ListenAndServe serves the http.DefaultServeMux by HTTPS (and HTTP,
// redirecting to HTTPS), storing SSL credentials in the Google Cloud Storage
// buckets letsencrypt*.
//
// If the server is running outside GCE, instead an HTTPS server is started on
// the address specified by addr using the certificate details specified by opt.
//
// The given channel, if any, is closed when the TCP listener has succeeded.
// It may be used to signal that the server is ready to start serving requests.
//
// ListenAndServe does not return. It exits the program when the server is
// shut down (via SIGTERM or due to an error) and calls shutdown.Shutdown.
func ListenAndServe(ready chan<- struct{}, serverName, addr string, opt *Options) {
	if opt == nil {
		opt = defaultOptions
	} else {
		opt.applyDefaults()
	}

	var m autocert.Manager
	m.Prompt = autocert.AcceptTOS
	if h := opt.LetsEncryptHosts; len(h) > 0 {
		m.HostPolicy = autocert.HostWhitelist(h...)
	}

	var config *tls.Config
	if opt.InsecureHTTP {
		log.Info.Printf("https: serving insecure HTTP on %q", addr)
		host, _, err := net.SplitHostPort(addr)
		if err != nil {
			log.Fatalf("https: couldn't parse address: %v", err)
		}
		if host != "localhost" && host != "127.0.0.1" && host != "::1" {
			log.Fatalf("https: cannot serve insecure HTTP on non-loopback address %q", addr)
		}
	} else if file := opt.LetsEncryptCache; file != "" {
		log.Info.Printf("https: serving HTTPS on %q using Let's Encrypt certificates", addr)
		m.Cache = autocert.DirCache(file)
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else if metadata.OnGCE() {
		addr = ":443"
		log.Info.Printf("https: serving HTTPS on GCE %q using Let's Encrypt certificates", addr)
		const key = "letsencrypt-bucket"
		bucket, err := metadata.InstanceAttributeValue(key)
		if err != nil {
			log.Fatalf("https: couldn't read %q metadata value: %v", key, err)
		}
		cache, err := newAutocertCache(bucket, serverName)
		if err != nil {
			log.Fatalf("https: couldn't set up letsencrypt cache: %v", err)
		}
		m.Cache = cache
		config = &tls.Config{GetCertificate: m.GetCertificate}
	} else {
		log.Info.Printf("https: not on GCE; serving HTTPS on %q using provided certificates", addr)
		if opt.CertFile == defaultOptions.CertFile || opt.KeyFile == defaultOptions.KeyFile {
			log.Error.Print("https: WARNING: using self-signed test certificates.")
		}
		var err error
		config, err = newDefaultTLSConfig(opt.CertFile, opt.KeyFile)
		if err != nil {
			log.Fatalf("https: setting up TLS config: %v", err)
		}
	}
	// WriteTimeout is set to 0 because it also pertains to streaming
	// replies, e.g., the DirServer.Watch interface.
	server := &http.Server{
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       15 * time.Second,
		WriteTimeout:      0,
		IdleTimeout:       60 * time.Second,
		TLSConfig:         config,
	}
	// TODO(adg): enable HTTP/2 once it's fast enough
	//err := http2.ConfigureServer(server, nil)
	//if err != nil {
	//	log.Fatalf("https: %v", err)
	//}

	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("https: %v", err)
	}
	if ready != nil {
		close(ready)
	}
	shutdown.Handle(func() {
		// Stop accepting connections and forces the server to stop
		// its serving loop.
		ln.Close()
	})
	if !opt.InsecureHTTP {
		ln = tls.NewListener(ln, config)
	}
	err = server.Serve(ln)
	log.Printf("https: %v", err)
	shutdown.Now(1)
}

// ListenAndServeFromFlags is the same as ListenAndServe, but it determines the
// listen address and Options from command-line flags in the flags package.
func ListenAndServeFromFlags(ready chan<- struct{}, serverName string) {
	addr := flags.HTTPSAddr
	if flags.InsecureHTTP {
		addr = flags.HTTPAddr
	}
	ListenAndServe(ready, serverName, addr, &Options{
		LetsEncryptCache: flags.LetsEncryptCache,
		CertFile:         flags.TLSCertFile,
		KeyFile:          flags.TLSKeyFile,
		InsecureHTTP:     flags.InsecureHTTP,
	})
}

// newDefaultTLSConfig creates a new TLS config based on the certificate files given.
func newDefaultTLSConfig(certFile string, certKeyFile string) (*tls.Config, error) {
	const op = "cloud/https.newDefaultTLSConfig"
	certReadable, err := isReadableFile(certFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL certificate in %q: %q", certFile, err))
	}
	if !certReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate file %q not readable", certFile))
	}
	keyReadable, err := isReadableFile(certKeyFile)
	if err != nil {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("SSL key in %q: %v", certKeyFile, err))
	}
	if !keyReadable {
		return nil, errors.E(op, errors.Invalid, errors.Errorf("certificate key file %q not readable", certKeyFile))
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		return nil, errors.E(op, err)
	}

	tlsConfig := &tls.Config{
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		},
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true, // Use our choice, not the client's choice
		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519},
		Certificates:             []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()
	return tlsConfig, nil
}

// isReadableFile reports whether the file exists and is readable.
// If the error is non-nil, it means there might be a file or directory
// with that name but we cannot read it.
func isReadableFile(path string) (bool, error) {
	// Is it stattable and is it a plain file?
	info, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil // Item does not exist.
		}
		return false, err // Item is problematic.
	}
	if info.IsDir() {
		return false, errors.Str("is directory")
	}
	// Is it readable?
	fd, err := os.Open(path)
	if err != nil {
		return false, access.ErrPermissionDenied
	}
	fd.Close()
	return true, nil // Item exists and is readable.
}

// autocertCache implements autocert.Cache.
type autocertCache struct {
	b      *storage.BucketHandle
	server string
}

func newAutocertCache(bucket, prefix string) (cache autocertCache, err error) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithScopes(storage.ScopeFullControl))
	if err != nil {
		return
	}
	cache.b = client.Bucket(bucket)
	cache.server = prefix + "-"
	return
}

func (cache autocertCache) Get(ctx context.Context, name string) ([]byte, error) {
	r, err := cache.b.Object(cache.server + name).NewReader(ctx)
	if err == storage.ErrObjectNotExist {
		return nil, autocert.ErrCacheMiss
	}
	if err != nil {
		return nil, err
	}
	defer r.Close()
	return ioutil.ReadAll(r)
}

func (cache autocertCache) Put(ctx context.Context, name string, data []byte) error {
	// TODO(ehg) Do we need to add contentType="text/plain; charset=utf-8"?
	w := cache.b.Object(cache.server + name).NewWriter(ctx)
	_, err := w.Write(data)
	if err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	if err := w.Close(); err != nil {
		log.Printf("https: writing letsencrypt cache: %s %v", name, err)
	}
	return err
}

func (cache autocertCache) Delete(ctx context.Context, name string) error {
	return cache.b.Object(cache.server + name).Delete(ctx)
}
<|endoftext|>


<commit_before>// Package besticon includes functions
// finding icons for a given web site.
package besticon

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"image"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"sort"
	"strings"
	"time"

	"image/color"

	// Load supported image formats.
	_ "image/gif"
	_ "image/png"

	"github.com/mat/besticon/colorfinder"

	// ...even more image formats.
	_ "github.com/mat/besticon/ico"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
	"golang.org/x/net/publicsuffix"
)

var defaultFormats []string

// Icon holds icon information.
type Icon struct {
	URL       string `json:"url"`
	Width     int    `json:"width"`
	Height    int    `json:"height"`
	Format    string `json:"format"`
	Bytes     int    `json:"bytes"`
	Error     error  `json:"error"`
	Sha1sum   string `json:"sha1sum"`
	ImageData []byte `json:",omitempty"`
}

type IconFinder struct {
	FormatsAllowed []string
	KeepImageBytes bool
	icons          []Icon
}

func (f *IconFinder) FetchIcons(url string) (error, []Icon) {
	var err error

	if cacheEnabled() {
		f.icons, err = resultFromCache(url)
	} else {
		f.icons, err = FetchIcons(url)
	}

	return err, f.Icons()
}

func (f *IconFinder) IconWithMinSize(minSize int) *Icon {
	SortIcons(f.icons, false)

	for _, ico := range f.icons {
		if ico.Width >= minSize && ico.Height >= minSize {
			return &ico
		}
	}
	return nil
}

func (f *IconFinder) MainColorForIcons() *color.RGBA {
	return MainColorForIcons(f.icons)
}

func (f *IconFinder) Icons() []Icon {
	return discardUnwantedFormats(f.icons, f.FormatsAllowed)
}

func (ico *Icon) Image() (*image.Image, error) {
	img, _, err := image.Decode(bytes.NewReader(ico.ImageData))
	return &img, err
}

func discardUnwantedFormats(icons []Icon, wantedFormats []string) []Icon {
	formats := defaultFormats
	if len(wantedFormats) > 0 {
		formats = wantedFormats
	}

	return filterIcons(icons, func(ico Icon) bool {
		return includesString(formats, ico.Format)
	})
}

type iconPredicate func(Icon) bool

func filterIcons(icons []Icon, pred iconPredicate) []Icon {
	result := []Icon{}
	for _, ico := range icons {
		if pred(ico) {
			result = append(result, ico)
		}
	}
	return result
}

func includesString(arr []string, str string) bool {
	for _, e := range arr {
		if e == str {
			return true
		}
	}
	return false
}

// FetchBestIcon takes a siteURL and returns the icon with
// the largest dimensions for this site or an error.
func FetchBestIcon(siteURL string) (*Icon, error) {
	icons, e := FetchIcons(siteURL)
	if e != nil {
		return nil, e
	}

	if len(icons) < 1 {
		return nil, errors.New("besticon: no icons found for site")
	}

	best := icons[0]
	return &best, nil
}

func FetchOrGenerateIcon(siteURL string, size int) (*Icon, error) {
	icons, err := FetchIcons(siteURL)
	if err != nil {
		return nil, err
	}
	if len(icons) < 1 {
		return nil, errors.New("besticon: no icons found for site")
	}

	best := icons[0]
	if best.Width >= size && best.Height > size {
		return &best, nil
	}

	return &Icon{
		Width:  size,
		Height: size,
		Format: "png",
		URL:    fmt.Sprintf("/siteicon?url=%s&size=%d", siteURL, size),
	}, nil
}

// FetchIcons takes a siteURL and returns all icons for this site
// or an error.
func FetchIcons(siteURL string) ([]Icon, error) {
	siteURL = strings.TrimSpace(siteURL)
	if !strings.HasPrefix(siteURL, "http") {
		siteURL = "http://" + siteURL
	}

	html, url, e := fetchHTML(siteURL)
	if e != nil {
		return nil, e
	}

	links, e := findIconLinks(url, html)
	if e != nil {
		return nil, e
	}

	icons := fetchAllIcons(links)
	icons = rejectBrokenIcons(icons)
	SortIcons(icons, true)

	return icons, nil
}

const maxResponseBodySize = 10485760 // 10MB

func fetchHTML(url string) ([]byte, *url.URL, error) {
	r, e := get(url)
	if e != nil {
		return nil, nil, e
	}

	if !(r.StatusCode >= 200 && r.StatusCode < 300) {
		return nil, nil, errors.New("besticon: not found")
	}

	b, e := getBodyBytes(r)
	if e != nil {
		return nil, nil, e
	}
	if len(b) == 0 {
		return nil, nil, errors.New("besticon: empty response")
	}

	reader := bytes.NewReader(b)
	contentType := r.Header.Get("Content-Type")
	utf8reader, e := charset.NewReader(reader, contentType)
	if e != nil {
		return nil, nil, e
	}
	utf8bytes, e := ioutil.ReadAll(utf8reader)
	if e != nil {
		return nil, nil, e
	}

	return utf8bytes, r.Request.URL, nil
}

var iconPaths = []string{
	"/favicon.ico",
	"/apple-touch-icon.png",
	"/apple-touch-icon-precomposed.png",
}

type empty struct{}

func findIconLinks(siteURL *url.URL, html []byte) ([]string, error) {
	doc, e := docFromHTML(html)
	if e != nil {
		return nil, e
	}

	baseURL := determineBaseURL(siteURL, doc)
	links := make(map[string]empty)

	// Add common, hard coded icon paths
	for _, path := range iconPaths {
		links[urlFromBase(baseURL, path)] = empty{}
	}

	// Add icons found in page
	urls := extractIconTags(doc)
	for _, url := range urls {
		url, e := absoluteURL(baseURL, url)
		if e == nil {
			links[url] = empty{}
		}
	}

	// Turn unique keys into array
	result := []string{}
	for u := range links {
		result = append(result, u)
	}
	sort.Strings(result)

	return result, nil
}

func determineBaseURL(siteURL *url.URL, doc *goquery.Document) *url.URL {
	baseTagHref := extractBaseTag(doc)
	if baseTagHref != "" {
		baseTagURL, e := url.Parse(baseTagHref)
		if e != nil {
			return siteURL
		}
		return baseTagURL
	}

	return siteURL
}

func docFromHTML(html []byte) (*goquery.Document, error) {
	doc, e := goquery.NewDocumentFromReader(bytes.NewReader(html))
	if e != nil || doc == nil {
		return nil, errParseHTML
	}
	return doc, nil
}

var csspaths = strings.Join([]string{
	"link[rel='icon']",
	"link[rel='shortcut icon']",
	"link[rel='apple-touch-icon']",
	"link[rel='apple-touch-icon-precomposed']",
}, ", ")

var errParseHTML = errors.New("besticon: could not parse html")

func extractBaseTag(doc *goquery.Document) string {
	href := ""
	doc.Find("head base[href]").First().Each(func(i int, s *goquery.Selection) {
		href, _ = s.Attr("href")
	})
	return href
}

func extractIconTags(doc *goquery.Document) []string {
	hits := []string{}
	doc.Find(csspaths).Each(func(i int, s *goquery.Selection) {
		href, ok := s.Attr("href")
		if ok && href != "" {
			hits = append(hits, href)
		}
	})
	return hits
}

func MainColorForIcons(icons []Icon) *color.RGBA {
	if len(icons) == 0 {
		return nil
	}

	var icon *Icon
	for _, ico := range icons {
		if ico.Format == "png" {
			icon = &ico
			break
		}
	}
	if icon == nil {
		return nil
	}

	img, err := icon.Image()
	if err != nil {
		return nil
	}

	cf := colorfinder.ColorFinder{}
	color, err := cf.FindMainColor(*img)
	if err != nil {
		return nil
	}

	return &color
}

func fetchAllIcons(urls []string) []Icon {
	ch := make(chan Icon)

	for _, u := range urls {
		go func(u string) { ch <- fetchIconDetails(u) }(u)
	}

	icons := []Icon{}
	for range urls {
		icon := <-ch
		icons = append(icons, icon)
	}
	return icons
}

func fetchIconDetails(url string) Icon {
	i := Icon{URL: url}

	response, e := get(url)
	if e != nil {
		i.Error = e
		return i
	}

	b, e := getBodyBytes(response)
	if e != nil {
		i.Error = e
		return i
	}

	cfg, format, e := image.DecodeConfig(bytes.NewReader(b))
	if e != nil {
		i.Error = fmt.Errorf("besticon: unknown image format: %s", e)
		return i
	}

	i.Width = cfg.Width
	i.Height = cfg.Height
	i.Format = format
	i.Bytes = len(b)
	i.Sha1sum = sha1Sum(b)
	if keepImageBytes {
		i.ImageData = b
	}

	return i
}

func get(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	if e != nil {
		return nil, e
	}

	setDefaultHeaders(req)

	start := time.Now()
	resp, err := client.Do(req)
	end := time.Now()
	duration := end.Sub(start)

	if err != nil {
		logger.Printf("Error: %s %s %s %.2fms",
			req.Method,
			req.URL,
			err,
			float64(duration)/float64(time.Millisecond),
		)
	} else {
		logger.Printf("%s %s %d %.2fms %d",
			req.Method,
			req.URL,
			resp.StatusCode,
			float64(duration)/float64(time.Millisecond),
			resp.ContentLength,
		)
	}

	return resp, err
}

func getBodyBytes(r *http.Response) ([]byte, error) {
	limitReader := io.LimitReader(r.Body, maxResponseBodySize)
	b, e := ioutil.ReadAll(limitReader)
	r.Body.Close()

	if len(b) >= maxResponseBodySize {
		return nil, errors.New("body too large")
	}
	return b, e
}

func setDefaultHeaders(req *http.Request) {
	req.Header.Set("Accept", "*/*")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10")
}

func mustInitCookieJar() *cookiejar.Jar {
	options := cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	}
	jar, e := cookiejar.New(&options)
	if e != nil {
		panic(e)
	}

	return jar
}

func checkRedirect(req *http.Request, via []*http.Request) error {
	setDefaultHeaders(req)

	if len(via) >= 10 {
		return errors.New("stopped after 10 redirects")
	}
	return nil
}

func absoluteURL(baseURL *url.URL, path string) (string, error) {
	url, e := url.Parse(path)
	if e != nil {
		return "", e
	}

	url.Scheme = baseURL.Scheme
	if url.Scheme == "" {
		url.Scheme = "http"
	}

	if url.Host == "" {
		url.Host = baseURL.Host
	}
	return url.String(), nil
}

func urlFromBase(baseURL *url.URL, path string) string {
	url := *baseURL
	url.Path = path
	if url.Scheme == "" {
		url.Scheme = "http"
	}

	return url.String()
}

func rejectBrokenIcons(icons []Icon) []Icon {
	result := []Icon{}
	for _, img := range icons {
		if img.Error == nil && (img.Width > 1 && img.Height > 1) {
			result = append(result, img)
		}
	}
	return result
}

func sha1Sum(b []byte) string {
	hash := sha1.New()
	hash.Write(b)
	bs := hash.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

var client *http.Client
var keepImageBytes bool

func init() {
	setHTTPClient(&http.Client{Timeout: 60 * time.Second})

	// Needs to be kept in sync with those image/... imports
	defaultFormats = []string{"png", "gif", "ico"}
}

func setHTTPClient(c *http.Client) {
	c.Jar = mustInitCookieJar()
	c.CheckRedirect = checkRedirect
	client = c
}

var logger *log.Logger

// SetLogOutput sets the output for the package's logger.
func SetLogOutput(w io.Writer) {
	logger = log.New(w, "http:  ", log.LstdFlags|log.Lmicroseconds)
}

func init() {
	SetLogOutput(os.Stdout)
	keepImageBytes = true
}
<commit_msg>Also look at gif images for fallback colors<commit_after>// Package besticon includes functions
// finding icons for a given web site.
package besticon

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"image"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"sort"
	"strings"
	"time"

	"image/color"

	// Load supported image formats.
	_ "image/gif"
	_ "image/png"

	"github.com/mat/besticon/colorfinder"

	// ...even more image formats.
	_ "github.com/mat/besticon/ico"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
	"golang.org/x/net/publicsuffix"
)

var defaultFormats []string

// Icon holds icon information.
type Icon struct {
	URL       string `json:"url"`
	Width     int    `json:"width"`
	Height    int    `json:"height"`
	Format    string `json:"format"`
	Bytes     int    `json:"bytes"`
	Error     error  `json:"error"`
	Sha1sum   string `json:"sha1sum"`
	ImageData []byte `json:",omitempty"`
}

type IconFinder struct {
	FormatsAllowed []string
	KeepImageBytes bool
	icons          []Icon
}

func (f *IconFinder) FetchIcons(url string) (error, []Icon) {
	var err error

	if cacheEnabled() {
		f.icons, err = resultFromCache(url)
	} else {
		f.icons, err = FetchIcons(url)
	}

	return err, f.Icons()
}

func (f *IconFinder) IconWithMinSize(minSize int) *Icon {
	SortIcons(f.icons, false)

	for _, ico := range f.icons {
		if ico.Width >= minSize && ico.Height >= minSize {
			return &ico
		}
	}
	return nil
}

func (f *IconFinder) MainColorForIcons() *color.RGBA {
	return MainColorForIcons(f.icons)
}

func (f *IconFinder) Icons() []Icon {
	return discardUnwantedFormats(f.icons, f.FormatsAllowed)
}

func (ico *Icon) Image() (*image.Image, error) {
	img, _, err := image.Decode(bytes.NewReader(ico.ImageData))
	return &img, err
}

func discardUnwantedFormats(icons []Icon, wantedFormats []string) []Icon {
	formats := defaultFormats
	if len(wantedFormats) > 0 {
		formats = wantedFormats
	}

	return filterIcons(icons, func(ico Icon) bool {
		return includesString(formats, ico.Format)
	})
}

type iconPredicate func(Icon) bool

func filterIcons(icons []Icon, pred iconPredicate) []Icon {
	result := []Icon{}
	for _, ico := range icons {
		if pred(ico) {
			result = append(result, ico)
		}
	}
	return result
}

func includesString(arr []string, str string) bool {
	for _, e := range arr {
		if e == str {
			return true
		}
	}
	return false
}

// FetchBestIcon takes a siteURL and returns the icon with
// the largest dimensions for this site or an error.
func FetchBestIcon(siteURL string) (*Icon, error) {
	icons, e := FetchIcons(siteURL)
	if e != nil {
		return nil, e
	}

	if len(icons) < 1 {
		return nil, errors.New("besticon: no icons found for site")
	}

	best := icons[0]
	return &best, nil
}

func FetchOrGenerateIcon(siteURL string, size int) (*Icon, error) {
	icons, err := FetchIcons(siteURL)
	if err != nil {
		return nil, err
	}
	if len(icons) < 1 {
		return nil, errors.New("besticon: no icons found for site")
	}

	best := icons[0]
	if best.Width >= size && best.Height > size {
		return &best, nil
	}

	return &Icon{
		Width:  size,
		Height: size,
		Format: "png",
		URL:    fmt.Sprintf("/siteicon?url=%s&size=%d", siteURL, size),
	}, nil
}

// FetchIcons takes a siteURL and returns all icons for this site
// or an error.
func FetchIcons(siteURL string) ([]Icon, error) {
	siteURL = strings.TrimSpace(siteURL)
	if !strings.HasPrefix(siteURL, "http") {
		siteURL = "http://" + siteURL
	}

	html, url, e := fetchHTML(siteURL)
	if e != nil {
		return nil, e
	}

	links, e := findIconLinks(url, html)
	if e != nil {
		return nil, e
	}

	icons := fetchAllIcons(links)
	icons = rejectBrokenIcons(icons)
	SortIcons(icons, true)

	return icons, nil
}

const maxResponseBodySize = 10485760 // 10MB

func fetchHTML(url string) ([]byte, *url.URL, error) {
	r, e := get(url)
	if e != nil {
		return nil, nil, e
	}

	if !(r.StatusCode >= 200 && r.StatusCode < 300) {
		return nil, nil, errors.New("besticon: not found")
	}

	b, e := getBodyBytes(r)
	if e != nil {
		return nil, nil, e
	}
	if len(b) == 0 {
		return nil, nil, errors.New("besticon: empty response")
	}

	reader := bytes.NewReader(b)
	contentType := r.Header.Get("Content-Type")
	utf8reader, e := charset.NewReader(reader, contentType)
	if e != nil {
		return nil, nil, e
	}
	utf8bytes, e := ioutil.ReadAll(utf8reader)
	if e != nil {
		return nil, nil, e
	}

	return utf8bytes, r.Request.URL, nil
}

var iconPaths = []string{
	"/favicon.ico",
	"/apple-touch-icon.png",
	"/apple-touch-icon-precomposed.png",
}

type empty struct{}

func findIconLinks(siteURL *url.URL, html []byte) ([]string, error) {
	doc, e := docFromHTML(html)
	if e != nil {
		return nil, e
	}

	baseURL := determineBaseURL(siteURL, doc)
	links := make(map[string]empty)

	// Add common, hard coded icon paths
	for _, path := range iconPaths {
		links[urlFromBase(baseURL, path)] = empty{}
	}

	// Add icons found in page
	urls := extractIconTags(doc)
	for _, url := range urls {
		url, e := absoluteURL(baseURL, url)
		if e == nil {
			links[url] = empty{}
		}
	}

	// Turn unique keys into array
	result := []string{}
	for u := range links {
		result = append(result, u)
	}
	sort.Strings(result)

	return result, nil
}

func determineBaseURL(siteURL *url.URL, doc *goquery.Document) *url.URL {
	baseTagHref := extractBaseTag(doc)
	if baseTagHref != "" {
		baseTagURL, e := url.Parse(baseTagHref)
		if e != nil {
			return siteURL
		}
		return baseTagURL
	}

	return siteURL
}

func docFromHTML(html []byte) (*goquery.Document, error) {
	doc, e := goquery.NewDocumentFromReader(bytes.NewReader(html))
	if e != nil || doc == nil {
		return nil, errParseHTML
	}
	return doc, nil
}

var csspaths = strings.Join([]string{
	"link[rel='icon']",
	"link[rel='shortcut icon']",
	"link[rel='apple-touch-icon']",
	"link[rel='apple-touch-icon-precomposed']",
}, ", ")

var errParseHTML = errors.New("besticon: could not parse html")

func extractBaseTag(doc *goquery.Document) string {
	href := ""
	doc.Find("head base[href]").First().Each(func(i int, s *goquery.Selection) {
		href, _ = s.Attr("href")
	})
	return href
}

func extractIconTags(doc *goquery.Document) []string {
	hits := []string{}
	doc.Find(csspaths).Each(func(i int, s *goquery.Selection) {
		href, ok := s.Attr("href")
		if ok && href != "" {
			hits = append(hits, href)
		}
	})
	return hits
}

func MainColorForIcons(icons []Icon) *color.RGBA {
	if len(icons) == 0 {
		return nil
	}

	var icon *Icon
	for _, ico := range icons {
		if ico.Format == "png" || ico.Format == "gif" {
			icon = &ico
			break
		}
	}
	if icon == nil {
		return nil
	}

	img, err := icon.Image()
	if err != nil {
		return nil
	}

	cf := colorfinder.ColorFinder{}
	color, err := cf.FindMainColor(*img)
	if err != nil {
		return nil
	}

	return &color
}

func fetchAllIcons(urls []string) []Icon {
	ch := make(chan Icon)

	for _, u := range urls {
		go func(u string) { ch <- fetchIconDetails(u) }(u)
	}

	icons := []Icon{}
	for range urls {
		icon := <-ch
		icons = append(icons, icon)
	}
	return icons
}

func fetchIconDetails(url string) Icon {
	i := Icon{URL: url}

	response, e := get(url)
	if e != nil {
		i.Error = e
		return i
	}

	b, e := getBodyBytes(response)
	if e != nil {
		i.Error = e
		return i
	}

	cfg, format, e := image.DecodeConfig(bytes.NewReader(b))
	if e != nil {
		i.Error = fmt.Errorf("besticon: unknown image format: %s", e)
		return i
	}

	i.Width = cfg.Width
	i.Height = cfg.Height
	i.Format = format
	i.Bytes = len(b)
	i.Sha1sum = sha1Sum(b)
	if keepImageBytes {
		i.ImageData = b
	}

	return i
}

func get(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	if e != nil {
		return nil, e
	}

	setDefaultHeaders(req)

	start := time.Now()
	resp, err := client.Do(req)
	end := time.Now()
	duration := end.Sub(start)

	if err != nil {
		logger.Printf("Error: %s %s %s %.2fms",
			req.Method,
			req.URL,
			err,
			float64(duration)/float64(time.Millisecond),
		)
	} else {
		logger.Printf("%s %s %d %.2fms %d",
			req.Method,
			req.URL,
			resp.StatusCode,
			float64(duration)/float64(time.Millisecond),
			resp.ContentLength,
		)
	}

	return resp, err
}

func getBodyBytes(r *http.Response) ([]byte, error) {
	limitReader := io.LimitReader(r.Body, maxResponseBodySize)
	b, e := ioutil.ReadAll(limitReader)
	r.Body.Close()

	if len(b) >= maxResponseBodySize {
		return nil, errors.New("body too large")
	}
	return b, e
}

func setDefaultHeaders(req *http.Request) {
	req.Header.Set("Accept", "*/*")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10")
}

func mustInitCookieJar() *cookiejar.Jar {
	options := cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	}
	jar, e := cookiejar.New(&options)
	if e != nil {
		panic(e)
	}

	return jar
}

func checkRedirect(req *http.Request, via []*http.Request) error {
	setDefaultHeaders(req)

	if len(via) >= 10 {
		return errors.New("stopped after 10 redirects")
	}
	return nil
}

func absoluteURL(baseURL *url.URL, path string) (string, error) {
	url, e := url.Parse(path)
	if e != nil {
		return "", e
	}

	url.Scheme = baseURL.Scheme
	if url.Scheme == "" {
		url.Scheme = "http"
	}

	if url.Host == "" {
		url.Host = baseURL.Host
	}
	return url.String(), nil
}

func urlFromBase(baseURL *url.URL, path string) string {
	url := *baseURL
	url.Path = path
	if url.Scheme == "" {
		url.Scheme = "http"
	}

	return url.String()
}

func rejectBrokenIcons(icons []Icon) []Icon {
	result := []Icon{}
	for _, img := range icons {
		if img.Error == nil && (img.Width > 1 && img.Height > 1) {
			result = append(result, img)
		}
	}
	return result
}

func sha1Sum(b []byte) string {
	hash := sha1.New()
	hash.Write(b)
	bs := hash.Sum(nil)
	return fmt.Sprintf("%x", bs)
}

var client *http.Client
var keepImageBytes bool

func init() {
	setHTTPClient(&http.Client{Timeout: 60 * time.Second})

	// Needs to be kept in sync with those image/... imports
	defaultFormats = []string{"png", "gif", "ico"}
}

func setHTTPClient(c *http.Client) {
	c.Jar = mustInitCookieJar()
	c.CheckRedirect = checkRedirect
	client = c
}

var logger *log.Logger

// SetLogOutput sets the output for the package's logger.
func SetLogOutput(w io.Writer) {
	logger = log.New(w, "http:  ", log.LstdFlags|log.Lmicroseconds)
}

func init() {
	SetLogOutput(os.Stdout)
	keepImageBytes = true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"testing"
)

// save a repo and try to load it using stdout
func TestSaveAndLoadRepoStdout(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to create a container: %v %v", out, err))

	cleanedContainerID := stripTrailingCharacters(out)

	repoName := "foobar-save-load-test"

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
	out, _, err = runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("output should've been a container id: %v %v", cleanedContainerID, err))

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName)
	out, _, err = runCommandWithOutput(commitCmd)
	errorOut(err, t, fmt.Sprintf("failed to commit container: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	before, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist before saving it: %v %v", before, err))

	saveCmdTemplate := `%v save %v > /tmp/foobar-save-load-test.tar`
	saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo: %v %v", out, err))

	deleteImages(repoName)

	loadCmdFinal := `cat /tmp/foobar-save-load-test.tar | docker load`
	loadCmd := exec.Command("bash", "-c", loadCmdFinal)
	out, _, err = runCommandWithOutput(loadCmd)
	errorOut(err, t, fmt.Sprintf("failed to load repo: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	after, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist after loading it: %v %v", after, err))

	if before != after {
		t.Fatalf("inspect is not the same after a save / load")
	}

	deleteContainer(cleanedContainerID)
	deleteImages(repoName)

	os.Remove("/tmp/foobar-save-load-test.tar")

	logDone("save - save a repo using stdout")
	logDone("load - load a repo using stdout")
}

func TestSaveSingleTag(t *testing.T) {
	repoName := "foobar-save-single-tag-test"

	tagCmdFinal := fmt.Sprintf("%v tag busybox:latest %v:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	idCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName)
	idCmd := exec.Command("bash", "-c", idCmdFinal)
	out, _, err = runCommandWithOutput(idCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err))

	cleanedImageID := stripTrailingCharacters(out)

	saveCmdFinal := fmt.Sprintf("%v save %v:latest | tar t | grep -E '(^repositories$|%v)'", dockerBinary, repoName, cleanedImageID)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo with image ID and 'repositories' file: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save a specific image:tag")
}

func TestSaveImageId(t *testing.T) {
	repoName := "foobar-save-image-id-test"

	tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	idLongCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName)
	idLongCmd := exec.Command("bash", "-c", idLongCmdFinal)
	out, _, err = runCommandWithOutput(idLongCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err))

	cleanedLongImageID := stripTrailingCharacters(out)

	idShortCmdFinal := fmt.Sprintf("%v images -q %v", dockerBinary, repoName)
	idShortCmd := exec.Command("bash", "-c", idShortCmdFinal)
	out, _, err = runCommandWithOutput(idShortCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo short ID: %v %v", out, err))

	cleanedShortImageID := stripTrailingCharacters(out)

	saveCmdFinal := fmt.Sprintf("%v save %v | tar t | grep %v", dockerBinary, cleanedShortImageID, cleanedLongImageID)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo with image ID: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save a image by ID")
}

// save a repo and try to load it using flags
func TestSaveAndLoadRepoFlags(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to create a container: %v %v", out, err))

	cleanedContainerID := stripTrailingCharacters(out)

	repoName := "foobar-save-load-test"

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
	out, _, err = runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("output should've been a container id: %v %v", cleanedContainerID, err))

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName)
	out, _, err = runCommandWithOutput(commitCmd)
	errorOut(err, t, fmt.Sprintf("failed to commit container: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	before, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist before saving it: %v %v", before, err))

	saveCmdTemplate := `%v save -o /tmp/foobar-save-load-test.tar %v`
	saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo: %v %v", out, err))

	deleteImages(repoName)

	loadCmdFinal := `docker load -i /tmp/foobar-save-load-test.tar`
	loadCmd := exec.Command("bash", "-c", loadCmdFinal)
	out, _, err = runCommandWithOutput(loadCmd)
	errorOut(err, t, fmt.Sprintf("failed to load repo: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	after, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist after loading it: %v %v", after, err))

	if before != after {
		t.Fatalf("inspect is not the same after a save / load")
	}

	deleteContainer(cleanedContainerID)
	deleteImages(repoName)

	os.Remove("/tmp/foobar-save-load-test.tar")

	logDone("save - save a repo using -o")
	logDone("load - load a repo using -i")
}

func TestSaveMultipleNames(t *testing.T) {
	repoName := "foobar-save-multi-name-test"

	// Make one image
	tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v-one:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))
	// Make two images
	tagCmdFinal = fmt.Sprintf("%v tag scratch:latest %v-two:latest", dockerBinary, repoName)
	tagCmd = exec.Command("bash", "-c", tagCmdFinal)
	out, _, err = runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	saveCmdFinal := fmt.Sprintf("%v save %v-one %v-two:latest | tar xO repositories | grep -q -E '(-one|-two)'", dockerBinary, repoName, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save multiple repos: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save by multiple names")
}

// Issue #6722 #5892 ensure directories are included in changes
func TestSaveDirectoryPermissions(t *testing.T) {
	layerEntries := []string{"opt/", "opt/a/", "opt/a/b/", "opt/a/b/c"}

	name := "save-directory-permissions"
	tmpDir, err := ioutil.TempDir("", "save-layers-with-directories")
	extractionDirectory := filepath.Join(tmpDir, "image-extraction-dir")
	os.Mkdir(extractionDirectory, 0777)

	if err != nil {
		t.Errorf("failed to create temporary directory: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	defer deleteImages(name)
	_, err = buildImage(name,
		`FROM busybox
	RUN adduser -D user && mkdir -p /opt/a/b && chown -R user:user /opt/a
	RUN touch /opt/a/b/c && chown user:user /opt/a/b/c`,
		true)
	if err != nil {
		t.Fatal(err)
	}

	saveCmdFinal := fmt.Sprintf("%s save %s | tar -xf - -C %s", dockerBinary, name, extractionDirectory)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err := runCommandWithOutput(saveCmd)
	if err != nil {
		t.Errorf("failed to save and extract image: %s", out)
	}

	dirs, err := ioutil.ReadDir(extractionDirectory)
	if err != nil {
		t.Errorf("failed to get a listing of the layer directories: %s", err)
	}

	found := false
	for _, entry := range dirs {
		if entry.IsDir() {
			layerPath := filepath.Join(extractionDirectory, entry.Name(), "layer.tar")

			f, err := os.Open(layerPath)
			if err != nil {
				t.Fatalf("failed to open %s: %s", layerPath, err)
			}

			entries, err := ListTar(f)
			if err != nil {
				t.Fatalf("encountered error while listing tar entries: %s", err)
			}

			if reflect.DeepEqual(entries, layerEntries) {
				found = true
			}
		}
	}

	if !found {
		t.Fatalf("failed to find the layer with the right content listing")
	}

	logDone("save - ensure directories exist in exported layers")
}
<commit_msg>integcli: fix permission detection for aufs<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"testing"
)

// save a repo and try to load it using stdout
func TestSaveAndLoadRepoStdout(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to create a container: %v %v", out, err))

	cleanedContainerID := stripTrailingCharacters(out)

	repoName := "foobar-save-load-test"

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
	out, _, err = runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("output should've been a container id: %v %v", cleanedContainerID, err))

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName)
	out, _, err = runCommandWithOutput(commitCmd)
	errorOut(err, t, fmt.Sprintf("failed to commit container: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	before, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist before saving it: %v %v", before, err))

	saveCmdTemplate := `%v save %v > /tmp/foobar-save-load-test.tar`
	saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo: %v %v", out, err))

	deleteImages(repoName)

	loadCmdFinal := `cat /tmp/foobar-save-load-test.tar | docker load`
	loadCmd := exec.Command("bash", "-c", loadCmdFinal)
	out, _, err = runCommandWithOutput(loadCmd)
	errorOut(err, t, fmt.Sprintf("failed to load repo: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	after, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist after loading it: %v %v", after, err))

	if before != after {
		t.Fatalf("inspect is not the same after a save / load")
	}

	deleteContainer(cleanedContainerID)
	deleteImages(repoName)

	os.Remove("/tmp/foobar-save-load-test.tar")

	logDone("save - save a repo using stdout")
	logDone("load - load a repo using stdout")
}

func TestSaveSingleTag(t *testing.T) {
	repoName := "foobar-save-single-tag-test"

	tagCmdFinal := fmt.Sprintf("%v tag busybox:latest %v:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	idCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName)
	idCmd := exec.Command("bash", "-c", idCmdFinal)
	out, _, err = runCommandWithOutput(idCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err))

	cleanedImageID := stripTrailingCharacters(out)

	saveCmdFinal := fmt.Sprintf("%v save %v:latest | tar t | grep -E '(^repositories$|%v)'", dockerBinary, repoName, cleanedImageID)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo with image ID and 'repositories' file: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save a specific image:tag")
}

func TestSaveImageId(t *testing.T) {
	repoName := "foobar-save-image-id-test"

	tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	idLongCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName)
	idLongCmd := exec.Command("bash", "-c", idLongCmdFinal)
	out, _, err = runCommandWithOutput(idLongCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err))

	cleanedLongImageID := stripTrailingCharacters(out)

	idShortCmdFinal := fmt.Sprintf("%v images -q %v", dockerBinary, repoName)
	idShortCmd := exec.Command("bash", "-c", idShortCmdFinal)
	out, _, err = runCommandWithOutput(idShortCmd)
	errorOut(err, t, fmt.Sprintf("failed to get repo short ID: %v %v", out, err))

	cleanedShortImageID := stripTrailingCharacters(out)

	saveCmdFinal := fmt.Sprintf("%v save %v | tar t | grep %v", dockerBinary, cleanedShortImageID, cleanedLongImageID)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo with image ID: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save a image by ID")
}

// save a repo and try to load it using flags
func TestSaveAndLoadRepoFlags(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	errorOut(err, t, fmt.Sprintf("failed to create a container: %v %v", out, err))

	cleanedContainerID := stripTrailingCharacters(out)

	repoName := "foobar-save-load-test"

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
	out, _, err = runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("output should've been a container id: %v %v", cleanedContainerID, err))

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName)
	out, _, err = runCommandWithOutput(commitCmd)
	errorOut(err, t, fmt.Sprintf("failed to commit container: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	before, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist before saving it: %v %v", before, err))

	saveCmdTemplate := `%v save -o /tmp/foobar-save-load-test.tar %v`
	saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save repo: %v %v", out, err))

	deleteImages(repoName)

	loadCmdFinal := `docker load -i /tmp/foobar-save-load-test.tar`
	loadCmd := exec.Command("bash", "-c", loadCmdFinal)
	out, _, err = runCommandWithOutput(loadCmd)
	errorOut(err, t, fmt.Sprintf("failed to load repo: %v %v", out, err))

	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
	after, _, err := runCommandWithOutput(inspectCmd)
	errorOut(err, t, fmt.Sprintf("the repo should exist after loading it: %v %v", after, err))

	if before != after {
		t.Fatalf("inspect is not the same after a save / load")
	}

	deleteContainer(cleanedContainerID)
	deleteImages(repoName)

	os.Remove("/tmp/foobar-save-load-test.tar")

	logDone("save - save a repo using -o")
	logDone("load - load a repo using -i")
}

func TestSaveMultipleNames(t *testing.T) {
	repoName := "foobar-save-multi-name-test"

	// Make one image
	tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v-one:latest", dockerBinary, repoName)
	tagCmd := exec.Command("bash", "-c", tagCmdFinal)
	out, _, err := runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))
	// Make two images
	tagCmdFinal = fmt.Sprintf("%v tag scratch:latest %v-two:latest", dockerBinary, repoName)
	tagCmd = exec.Command("bash", "-c", tagCmdFinal)
	out, _, err = runCommandWithOutput(tagCmd)
	errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err))

	saveCmdFinal := fmt.Sprintf("%v save %v-one %v-two:latest | tar xO repositories | grep -q -E '(-one|-two)'", dockerBinary, repoName, repoName)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err = runCommandWithOutput(saveCmd)
	errorOut(err, t, fmt.Sprintf("failed to save multiple repos: %v %v", out, err))

	deleteImages(repoName)

	logDone("save - save by multiple names")
}

// Issue #6722 #5892 ensure directories are included in changes
func TestSaveDirectoryPermissions(t *testing.T) {
	layerEntries := []string{"opt/", "opt/a/", "opt/a/b/", "opt/a/b/c"}
	layerEntriesAUFS := []string{"./", ".wh..wh.aufs", ".wh..wh.orph/", ".wh..wh.plnk/", "opt/", "opt/a/", "opt/a/b/", "opt/a/b/c"}

	name := "save-directory-permissions"
	tmpDir, err := ioutil.TempDir("", "save-layers-with-directories")
	extractionDirectory := filepath.Join(tmpDir, "image-extraction-dir")
	os.Mkdir(extractionDirectory, 0777)

	if err != nil {
		t.Errorf("failed to create temporary directory: %s", err)
	}
	defer os.RemoveAll(tmpDir)
	defer deleteImages(name)
	_, err = buildImage(name,
		`FROM busybox
	RUN adduser -D user && mkdir -p /opt/a/b && chown -R user:user /opt/a
	RUN touch /opt/a/b/c && chown user:user /opt/a/b/c`,
		true)
	if err != nil {
		t.Fatal(err)
	}

	saveCmdFinal := fmt.Sprintf("%s save %s | tar -xf - -C %s", dockerBinary, name, extractionDirectory)
	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
	out, _, err := runCommandWithOutput(saveCmd)
	if err != nil {
		t.Errorf("failed to save and extract image: %s", out)
	}

	dirs, err := ioutil.ReadDir(extractionDirectory)
	if err != nil {
		t.Errorf("failed to get a listing of the layer directories: %s", err)
	}

	found := false
	for _, entry := range dirs {
		if entry.IsDir() {
			layerPath := filepath.Join(extractionDirectory, entry.Name(), "layer.tar")

			f, err := os.Open(layerPath)
			if err != nil {
				t.Fatalf("failed to open %s: %s", layerPath, err)
			}

			entries, err := ListTar(f)
			if err != nil {
				t.Fatalf("encountered error while listing tar entries: %s", err)
			}

			if reflect.DeepEqual(entries, layerEntries) || reflect.DeepEqual(entries, layerEntriesAUFS) {
				found = true
				break
			}
		}
	}

	if !found {
		t.Fatalf("failed to find the layer with the right content listing")
	}

	logDone("save - ensure directories exist in exported layers")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"code.sajari.com/docconv"
)

var (
	inputPath                     = flag.String("input", "", "The file path to convert and exit; no server")
	listenAddr                    = flag.String("addr", ":8888", "The address to listen on (e.g. 127.0.0.1:8888)")
	logLevel                      = flag.Uint("log-level", 0, "The verbosity of the log")
	readabilityLengthLow          = flag.Int("readability-length-low", 70, "Sets the readability length low")
	readabilityLengthHigh         = flag.Int("readability-length-high", 200, "Sets the readability length high")
	readabilityStopwordsLow       = flag.Float64("readability-stopwords-low", 0.2, "Sets the readability stopwords low")
	readabilityStopwordsHigh      = flag.Float64("readability-stopwords-high", 0.3, "Sets the readability stopwords high")
	readabilityMaxLinkDensity     = flag.Float64("readability-max-link-density", 0.2, "Sets the readability max link density")
	readabilityMaxHeadingDistance = flag.Int("readability-max-heading-distance", 200, "Sets the readability max heading distance")
	readabilityUseClasses         = flag.String("readability-use-classes", "good,neargood", "Comma separated list of readability classes to use")
)

func main() {
	flag.Parse()

	// TODO: Improve this (remove the need for it!)
	docconv.HTMLReadabilityOptionsValues = docconv.HTMLReadabilityOptions{
		LengthLow:             *readabilityLengthLow,
		LengthHigh:            *readabilityLengthHigh,
		StopwordsLow:          *readabilityStopwordsLow,
		StopwordsHigh:         *readabilityStopwordsHigh,
		MaxLinkDensity:        *readabilityMaxLinkDensity,
		MaxHeadingDistance:    *readabilityMaxHeadingDistance,
		ReadabilityUseClasses: *readabilityUseClasses,
	}

	if *inputPath != "" {
		resp, err := docconv.ConvertPath(*inputPath)
		if err != nil {
			log.Printf("error converting file '%v': %v", *inputPath, err)
		}
		fmt.Print(string(resp.Body))
		return
	}
	serve()
}

// Convert a file given a path
func convertPath(path string, readability bool) ([]byte, error) {
	mimeType := docconv.MimeTypeByExtension(path)
	if *logLevel >= 1 {
		log.Println("Converting file: " + path + " (" + mimeType + ")")
	}

	// Open file
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	data, err := docconv.Convert(f, mimeType, readability)
	if err != nil {
		return nil, err
	}
	b, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}
	if *logLevel >= 2 {
		log.Println(string(b))
	}
	return b, nil
}

// Start the conversion web service
func serve() {
	http.HandleFunc("/convert", func(w http.ResponseWriter, r *http.Request) {
		// Readability flag. Currently only used for HTML
		var readability bool
		if r.FormValue("readability") == "1" {
			readability = true
			if *logLevel >= 2 {
				log.Println("Readability is on")
			}
		}

		path := r.FormValue("path")
		if path != "" {
			b, err := docconv.ConvertPathReadability(path, readability)
			if err != nil {
				// TODO: return a sensible status code for errors like this.
				log.Printf("error converting path '%v': %v", path, err)
				return
			}
			w.Write(b)
			return
		}

		// Get uploaded file
		file, info, err := r.FormFile("input")
		if err != nil {
			log.Println("File upload", err)
			return
		}
		defer file.Close()

		// Abort if file doesn't have a mime type
		if len(info.Header["Content-Type"]) == 0 {
			log.Println("No content type", info.Filename)
			return
		}

		// If a generic mime type was provided then use file extension to determine mimetype
		mimeType := info.Header["Content-Type"][0]
		if mimeType == "application/octet-stream" {
			mimeType = docconv.MimeTypeByExtension(info.Filename)
		}

		if *logLevel >= 1 {
			log.Println("Received file: " + info.Filename + " (" + mimeType + ")")
		}

		data, err := docconv.Convert(file, mimeType, readability)
		if err != nil {
			log.Printf("error converting data: %v", err)
			data = &docconv.Response{
				Error: err.Error(),
			}
		}

		if err := json.NewEncoder(w).Encode(data); err != nil {
			log.Printf("error marshaling JSON data: %v", err)
			return
		}
	})

	// Start webserver
	log.Println("Setting log level to", *logLevel)
	log.Println("Starting docconv on", *listenAddr)
	log.Fatal(http.ListenAndServe(*listenAddr, nil))
}
<commit_msg>docd: remove unused convertPath function<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"

	"code.sajari.com/docconv"
)

var (
	inputPath                     = flag.String("input", "", "The file path to convert and exit; no server")
	listenAddr                    = flag.String("addr", ":8888", "The address to listen on (e.g. 127.0.0.1:8888)")
	logLevel                      = flag.Uint("log-level", 0, "The verbosity of the log")
	readabilityLengthLow          = flag.Int("readability-length-low", 70, "Sets the readability length low")
	readabilityLengthHigh         = flag.Int("readability-length-high", 200, "Sets the readability length high")
	readabilityStopwordsLow       = flag.Float64("readability-stopwords-low", 0.2, "Sets the readability stopwords low")
	readabilityStopwordsHigh      = flag.Float64("readability-stopwords-high", 0.3, "Sets the readability stopwords high")
	readabilityMaxLinkDensity     = flag.Float64("readability-max-link-density", 0.2, "Sets the readability max link density")
	readabilityMaxHeadingDistance = flag.Int("readability-max-heading-distance", 200, "Sets the readability max heading distance")
	readabilityUseClasses         = flag.String("readability-use-classes", "good,neargood", "Comma separated list of readability classes to use")
)

func main() {
	flag.Parse()

	// TODO: Improve this (remove the need for it!)
	docconv.HTMLReadabilityOptionsValues = docconv.HTMLReadabilityOptions{
		LengthLow:             *readabilityLengthLow,
		LengthHigh:            *readabilityLengthHigh,
		StopwordsLow:          *readabilityStopwordsLow,
		StopwordsHigh:         *readabilityStopwordsHigh,
		MaxLinkDensity:        *readabilityMaxLinkDensity,
		MaxHeadingDistance:    *readabilityMaxHeadingDistance,
		ReadabilityUseClasses: *readabilityUseClasses,
	}

	if *inputPath != "" {
		resp, err := docconv.ConvertPath(*inputPath)
		if err != nil {
			log.Printf("error converting file '%v': %v", *inputPath, err)
		}
		fmt.Print(string(resp.Body))
		return
	}
	serve()
}

// Start the conversion web service
func serve() {
	http.HandleFunc("/convert", func(w http.ResponseWriter, r *http.Request) {
		// Readability flag. Currently only used for HTML
		var readability bool
		if r.FormValue("readability") == "1" {
			readability = true
			if *logLevel >= 2 {
				log.Println("Readability is on")
			}
		}

		path := r.FormValue("path")
		if path != "" {
			b, err := docconv.ConvertPathReadability(path, readability)
			if err != nil {
				// TODO: return a sensible status code for errors like this.
				log.Printf("error converting path '%v': %v", path, err)
				return
			}
			w.Write(b)
			return
		}

		// Get uploaded file
		file, info, err := r.FormFile("input")
		if err != nil {
			log.Println("File upload", err)
			return
		}
		defer file.Close()

		// Abort if file doesn't have a mime type
		if len(info.Header["Content-Type"]) == 0 {
			log.Println("No content type", info.Filename)
			return
		}

		// If a generic mime type was provided then use file extension to determine mimetype
		mimeType := info.Header["Content-Type"][0]
		if mimeType == "application/octet-stream" {
			mimeType = docconv.MimeTypeByExtension(info.Filename)
		}

		if *logLevel >= 1 {
			log.Println("Received file: " + info.Filename + " (" + mimeType + ")")
		}

		data, err := docconv.Convert(file, mimeType, readability)
		if err != nil {
			log.Printf("error converting data: %v", err)
			data = &docconv.Response{
				Error: err.Error(),
			}
		}

		if err := json.NewEncoder(w).Encode(data); err != nil {
			log.Printf("error marshaling JSON data: %v", err)
			return
		}
	})

	// Start webserver
	log.Println("Setting log level to", *logLevel)
	log.Println("Starting docconv on", *listenAddr)
	log.Fatal(http.ListenAndServe(*listenAddr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/codegangsta/cli"
	"github.com/gernest/apidemic"
)

func server(ctx *cli.Context) {
	port := ctx.Int("port")
	s := apidemic.NewServer()

	log.Println("starting server on port :", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), s))
}

func main() {
	app := cli.NewApp()
	app.Name = "apidemic"
	app.Usage = "Fake JSON API Responses"
	app.Authors = []cli.Author{
		{"Geofrey Ernest", "geofreyernest@live.com"},
	}
	app.Version = apidemic.Version
	app.Commands = []cli.Command{
		cli.Command{
			Name:      "start",
			ShortName: "s",
			Usage:     "starts apidemic server",
			Action:    server,
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:   "port",
					Usage:  "http port to run",
					Value:  3000,
					EnvVar: "POSRT",
				},
			},
		},
	}
	app.RunAndExitOnError()
}
<commit_msg>Fix typo in port environment<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/codegangsta/cli"
	"github.com/gernest/apidemic"
)

func server(ctx *cli.Context) {
	port := ctx.Int("port")
	s := apidemic.NewServer()

	log.Println("starting server on port :", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), s))
}

func main() {
	app := cli.NewApp()
	app.Name = "apidemic"
	app.Usage = "Fake JSON API Responses"
	app.Authors = []cli.Author{
		{"Geofrey Ernest", "geofreyernest@live.com"},
	}
	app.Version = apidemic.Version
	app.Commands = []cli.Command{
		cli.Command{
			Name:      "start",
			ShortName: "s",
			Usage:     "starts apidemic server",
			Action:    server,
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:   "port",
					Usage:  "http port to run",
					Value:  3000,
					EnvVar: "PORT",
				},
			},
		},
	}
	app.RunAndExitOnError()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	"image/draw"
	_ "image/jpeg"
	"image/png"
	"io/ioutil"
	"os"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"
)

func mustGetImage(path string) image.Image {
	image, err := getImage(path)
	if err != nil {
		panic(err)
	}
	return image
}

func getImage(path string) (image.Image, error) {
	imageFd, err := os.Open(path)
	if err != nil {
		return image.Black, err
	}
	defer imageFd.Close()

	img, _, err := image.Decode(imageFd)
	if err != nil {
		return image.Black, err
	}
	return img, nil
}

func generateBasicTemplate() draw.Image {
	templateImage := mustGetImage("template.png")
	destinationImage := image.NewNRGBA(templateImage.Bounds())

	// put base template into our destination
	draw.Draw(
		destinationImage,
		destinationImage.Bounds(),
		templateImage,
		image.ZP,
		draw.Src,
	)
	return destinationImage
}

func writeBackground(destinationImage draw.Image) draw.Image {
	templateMask := mustGetImage("template_mask.png")
	backgroundImage := mustGetImage("background")

	draw.DrawMask(
		destinationImage,
		destinationImage.Bounds(),
		backgroundImage,
		image.ZP,
		templateMask,
		image.ZP,
		draw.Over,
	)

	return destinationImage
}

func getFont() *truetype.Font {
	fontFd, err := os.Open("Loveletter_TW.ttf")
	if err != nil {
		panic(err)
	}

	fontBytes, err := ioutil.ReadAll(fontFd)
	if err != nil {
		panic(err)
	}

	font, err := truetype.Parse(fontBytes)
	if err != nil {
		panic(err)
	}

	return font
}

const (
	fontSize              = 14.0
	baselineX             = 30
	baselineY             = 120
	textBackgroundPadding = 3
)

func withPadding(rect image.Rectangle, padding int) image.Rectangle {
	return image.Rect(
		rect.Min.X-padding,
		rect.Min.Y-padding,
		rect.Max.X+padding,
		rect.Max.Y+padding,
	)
}

var panelToTopLeft = map[int]image.Point{
	0: image.Pt(13, 37),
	1: image.Pt(254, 37),
	2: image.Pt(493, 38),
}

var panelRectangle = image.Rect(
	0, 0,
	212, 216,
)

var panelToRectangle = func() map[int]image.Rectangle {
	m := make(map[int]image.Rectangle)
	for panelNumber, topLeft := range panelToTopLeft {
		m[panelNumber] = panelRectangle.Add(topLeft)
	}
	return m
}()

func copyImage(img image.Image) draw.Image {
	// create a new image
	copyTo := image.NewNRGBA(img.Bounds())

	// copy stuff to that image
	draw.Draw(
		copyTo,
		copyTo.Bounds(),
		img,
		image.ZP,
		draw.Src,
	)
	return copyTo
}

func writeTextList(textConfig []string, destinationImage draw.Image) draw.Image {
	// copy for easier semantics
	destinationImage = copyImage(destinationImage)

	for i, text := range textConfig {
		// writing an empty string still does a background, so let's not do that
		if text == "" {
			continue
		}
		// create text image for panel
		textImage := writeSingleText(text)
		// write text image on top of panel
		draw.DrawMask(
			destinationImage,
			panelToRectangle[i],
			textImage,
			image.ZP,
			image.Black,
			image.ZP,
			draw.Over,
		)
	}
	return destinationImage
}

func writeSingleText(text string) draw.Image {
	// create a panel image to draw our text to
	destinationImage := image.NewNRGBA(panelRectangle)

	// create font face for our font
	fontFace := truetype.NewFace(
		getFont(),
		&truetype.Options{Size: fontSize},
	)

	// create the start point for our baseline for the text
	startPoint := fixed.P(baselineX, baselineY)

	// create a drawer to draw the text starting at the baseline point, in the font
	drawer := &font.Drawer{
		Face: fontFace,
		Dot:  startPoint,
	}

	// measure the distance of the string
	drawDistance := drawer.MeasureString(text)
	borderRect := withPadding(
		// create a rectangle for the border
		image.Rect(
			// top left x is the same as the baseline
			baselineX,
			// top left y is the baseline y moved up by the ascent of the font (the distance between the baseline and the top of the font)
			baselineY-fontFace.Metrics().Ascent.Round(),
			// bottom right x is the baseline start point x plus the calculated distance for drawing
			baselineX+drawDistance.Round(),
			// bottom right y is the same as the baseline
			baselineY,
		),
		// pad that rectangle
		textBackgroundPadding,
	)

	// draw the background rectangle into the destination image in white
	draw.DrawMask(
		destinationImage,
		destinationImage.Bounds(),
		image.White,
		image.ZP,
		borderRect,
		image.ZP,
		draw.Over,
	)

	// draw the text, in black to the return value
	drawer.Dst = destinationImage
	drawer.Src = image.Black
	drawer.DrawString(text)

	return destinationImage
}

func writeImage(path string, image image.Image) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fd.Close()

	return png.Encode(fd, image)
}

func main() {
	destinationImage := writeTextList([]string{"foo", "", "baz"}, writeBackground(generateBasicTemplate()))
	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(1)
		}
	}()

	err := writeImage("out.png", destinationImage)
	if err != nil {
		panic(err)
	}
}
<commit_msg>refactor drawer using code in writeSingleText to be clearer<commit_after>package main

import (
	"fmt"
	"image"
	"image/draw"
	_ "image/jpeg"
	"image/png"
	"io/ioutil"
	"os"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"
)

func mustGetImage(path string) image.Image {
	image, err := getImage(path)
	if err != nil {
		panic(err)
	}
	return image
}

func getImage(path string) (image.Image, error) {
	imageFd, err := os.Open(path)
	if err != nil {
		return image.Black, err
	}
	defer imageFd.Close()

	img, _, err := image.Decode(imageFd)
	if err != nil {
		return image.Black, err
	}
	return img, nil
}

func generateBasicTemplate() draw.Image {
	templateImage := mustGetImage("template.png")
	destinationImage := image.NewNRGBA(templateImage.Bounds())

	// put base template into our destination
	draw.Draw(
		destinationImage,
		destinationImage.Bounds(),
		templateImage,
		image.ZP,
		draw.Src,
	)
	return destinationImage
}

func writeBackground(destinationImage draw.Image) draw.Image {
	templateMask := mustGetImage("template_mask.png")
	backgroundImage := mustGetImage("background")

	draw.DrawMask(
		destinationImage,
		destinationImage.Bounds(),
		backgroundImage,
		image.ZP,
		templateMask,
		image.ZP,
		draw.Over,
	)

	return destinationImage
}

func getFont() *truetype.Font {
	fontFd, err := os.Open("Loveletter_TW.ttf")
	if err != nil {
		panic(err)
	}

	fontBytes, err := ioutil.ReadAll(fontFd)
	if err != nil {
		panic(err)
	}

	font, err := truetype.Parse(fontBytes)
	if err != nil {
		panic(err)
	}

	return font
}

const (
	fontSize              = 14.0
	baselineX             = 30
	baselineY             = 120
	textBackgroundPadding = 3
)

func withPadding(rect image.Rectangle, padding int) image.Rectangle {
	return image.Rect(
		rect.Min.X-padding,
		rect.Min.Y-padding,
		rect.Max.X+padding,
		rect.Max.Y+padding,
	)
}

var panelToTopLeft = map[int]image.Point{
	0: image.Pt(13, 37),
	1: image.Pt(254, 37),
	2: image.Pt(493, 38),
}

var panelRectangle = image.Rect(
	0, 0,
	212, 216,
)

var panelToRectangle = func() map[int]image.Rectangle {
	m := make(map[int]image.Rectangle)
	for panelNumber, topLeft := range panelToTopLeft {
		m[panelNumber] = panelRectangle.Add(topLeft)
	}
	return m
}()

func copyImage(img image.Image) draw.Image {
	// create a new image
	copyTo := image.NewNRGBA(img.Bounds())

	// copy stuff to that image
	draw.Draw(
		copyTo,
		copyTo.Bounds(),
		img,
		image.ZP,
		draw.Src,
	)
	return copyTo
}

func writeTextList(textConfig []string, destinationImage draw.Image) draw.Image {
	// copy for easier semantics
	destinationImage = copyImage(destinationImage)

	for i, text := range textConfig {
		// writing an empty string still does a background, so let's not do that
		if text == "" {
			continue
		}
		// create text image for panel
		textImage := writeSingleText(text)
		// write text image on top of panel
		draw.DrawMask(
			destinationImage,
			panelToRectangle[i],
			textImage,
			image.ZP,
			image.Black,
			image.ZP,
			draw.Over,
		)
	}
	return destinationImage
}

func writeSingleText(text string) draw.Image {
	// create a panel image to draw our text to
	destinationImage := image.NewNRGBA(panelRectangle)

	// create font face for our font
	fontFace := truetype.NewFace(
		getFont(),
		&truetype.Options{Size: fontSize},
	)

	// create the start point for our baseline for the text
	startPoint := fixed.P(baselineX, baselineY)

	// create a drawer to draw the text starting at the baseline point, in the font and measure the distance of the string
	drawDistance := (&font.Drawer{Face: fontFace, Dot: startPoint}).MeasureString(text)

	borderRect := withPadding(
		// create a rectangle for the border
		image.Rect(
			// top left x is the same as the baseline
			baselineX,
			// top left y is the baseline y moved up by the ascent of the font (the distance between the baseline and the top of the font)
			baselineY-fontFace.Metrics().Ascent.Round(),
			// bottom right x is the baseline start point x plus the calculated distance for drawing
			baselineX+drawDistance.Round(),
			// bottom right y is the same as the baseline
			baselineY,
		),
		// pad that rectangle
		textBackgroundPadding,
	)

	// draw the background rectangle into the destination image in white
	draw.DrawMask(
		destinationImage,
		destinationImage.Bounds(),
		image.White,
		image.ZP,
		borderRect,
		image.ZP,
		draw.Over,
	)

	// draw the text, in black to the return value
	drawer := &font.Drawer{
		Dst:  destinationImage,
		Src:  image.Black,
		Face: fontFace,
		Dot:  startPoint,
	}
	drawer.DrawString(text)

	return destinationImage
}

func writeImage(path string, image image.Image) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fd.Close()

	return png.Encode(fd, image)
}

func main() {
	destinationImage := writeTextList([]string{"foo", "", "baz"}, writeBackground(generateBasicTemplate()))
	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(1)
		}
	}()

	err := writeImage("out.png", destinationImage)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"

	broker "cloud-testenv-broker/broker"
	glog "github.com/golang/glog"
	jsonpb "github.com/golang/protobuf/jsonpb"
	proto "github.com/golang/protobuf/proto"
	emulators "google/emulators"
	pb "google/protobuf"
)

const (
	// BRKR - get it?
	defaultBrokerPort = 8939
)

var (
	host = flag.String("host", "localhost", "The server host or IP address.")
	port = flag.Int("port", defaultBrokerPort,
		fmt.Sprintf("The server port. If specified as a non-default value, "+
			"overrides the value of the %s environment variable.",
			broker.BrokerAddressEnv))
	// TODO(hbchai): Should we accept multiple config files?
	configFile = flag.String("config_file", "", "The json config file of the Cloud Broker.")
)

// Returns the port the broker should serve on.
func brokerPort() int {
	if *port != defaultBrokerPort {
		return *port
	}
	brokerPort := broker.BrokerPortFromEnv()
	if brokerPort != 0 {
		return brokerPort
	}
	return defaultBrokerPort
}

func main() {
	flag.Set("alsologtostderr", "true")
	flag.Parse()
	brokerDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		glog.Fatalf("Failed to obtain broker directory: %v", err)
	}
	glog.Infof("Broker starting up (%s)...", brokerDir)

	config := emulators.BrokerConfig{DefaultEmulatorStartDeadline: &pb.Duration{Seconds: 10}}
	if *configFile != "" {
		// Parse configFile and use it.
		f, err := os.Open(*configFile)
		if err != nil {
			glog.Fatalf("Failed to open config file: %v", err)
		}
		err = jsonpb.Unmarshal(f, &config)
		if err != nil {
			glog.Fatalf("Failed to parse config file: %v", err)
		}
	}
	glog.Infof("Using configuration:\n%s", proto.MarshalTextString(&config))

	b, err := broker.NewGrpcServer(*host, brokerPort(), brokerDir, &config)
	if err != nil {
		glog.Fatalf("Failed to create broker: %v", err)
	}
	err = b.Start()
	if err != nil {
		glog.Fatalf("Failed to start broker: %v", err)
	}
	die := make(chan os.Signal, 1)
	signal.Notify(die, os.Interrupt, os.Kill)
	go func() {
		<-die
		b.Shutdown()
		os.Exit(1)
	}()
	defer b.Shutdown()
	glog.Infof("Broker listening on %s:%d.", *host, *port)
	b.Wait()
	glog.Infof("Broker shut down.")
}
<commit_msg>Close the config file.<commit_after>/*
Copyright 2014 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"

	broker "cloud-testenv-broker/broker"
	glog "github.com/golang/glog"
	jsonpb "github.com/golang/protobuf/jsonpb"
	proto "github.com/golang/protobuf/proto"
	emulators "google/emulators"
	pb "google/protobuf"
)

const (
	// BRKR - get it?
	defaultBrokerPort = 8939
)

var (
	host = flag.String("host", "localhost", "The server host or IP address.")
	port = flag.Int("port", defaultBrokerPort,
		fmt.Sprintf("The server port. If specified as a non-default value, "+
			"overrides the value of the %s environment variable.",
			broker.BrokerAddressEnv))
	// TODO(hbchai): Should we accept multiple config files?
	configFile = flag.String("config_file", "", "The json config file of the Cloud Broker.")
)

// Returns the port the broker should serve on.
func brokerPort() int {
	if *port != defaultBrokerPort {
		return *port
	}
	brokerPort := broker.BrokerPortFromEnv()
	if brokerPort != 0 {
		return brokerPort
	}
	return defaultBrokerPort
}

func main() {
	flag.Set("alsologtostderr", "true")
	flag.Parse()
	brokerDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		glog.Fatalf("Failed to obtain broker directory: %v", err)
	}
	glog.Infof("Broker starting up (%s)...", brokerDir)

	config := emulators.BrokerConfig{DefaultEmulatorStartDeadline: &pb.Duration{Seconds: 10}}
	if *configFile != "" {
		// Parse configFile and use it.
		f, err := os.Open(*configFile)
		if err != nil {
			glog.Fatalf("Failed to open config file: %v", err)
		}
		defer f.Close()
		err = jsonpb.Unmarshal(f, &config)
		if err != nil {
			glog.Fatalf("Failed to parse config file: %v", err)
		}
	}
	glog.Infof("Using configuration:\n%s", proto.MarshalTextString(&config))

	b, err := broker.NewGrpcServer(*host, brokerPort(), brokerDir, &config)
	if err != nil {
		glog.Fatalf("Failed to create broker: %v", err)
	}
	err = b.Start()
	if err != nil {
		glog.Fatalf("Failed to start broker: %v", err)
	}
	die := make(chan os.Signal, 1)
	signal.Notify(die, os.Interrupt, os.Kill)
	go func() {
		<-die
		b.Shutdown()
		os.Exit(1)
	}()
	defer b.Shutdown()
	glog.Infof("Broker listening on %s:%d.", *host, *port)
	b.Wait()
	glog.Infof("Broker shut down.")
}
<|endoftext|>