<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.3.1"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if file != "" && url != "" {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}

func ParseUserData(contents string) (interface{}, error) {
	header := strings.SplitN(contents, "\n", 2)[0]

	if strings.HasPrefix(header, "#!") {
		log.Printf("Parsing user-data as script")
		return system.Script(contents), nil

	} else if header == "#cloud-config" {
		log.Printf("Parsing user-data as cloud-config")
		cfg, err := initialize.NewCloudConfig(contents)
		if err != nil {
			log.Fatal(err.Error())
		}
		return *cfg, nil
	} else {
		return nil, fmt.Errorf("Unrecognized user-data header: %s", header)
	}
}
<commit_msg>chore(release): Bump version to v0.3.1+git<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.3.1+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if file != "" && url != "" {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else {
		fmt.Println("Provide one of --from-file or --from-url")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}

func ParseUserData(contents string) (interface{}, error) {
	header := strings.SplitN(contents, "\n", 2)[0]

	if strings.HasPrefix(header, "#!") {
		log.Printf("Parsing user-data as script")
		return system.Script(contents), nil

	} else if header == "#cloud-config" {
		log.Printf("Parsing user-data as cloud-config")
		cfg, err := initialize.NewCloudConfig(contents)
		if err != nil {
			log.Fatal(err.Error())
		}
		return *cfg, nil
	} else {
		return nil, fmt.Errorf("Unrecognized user-data header: %s", header)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.6.0+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}
<commit_msg>chore(coreos-cloudinit): bump to 0.7.0<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.7.0"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	rss "github.com/jteeuwen/go-pkg-rss"

	"bytes"
	"net/smtp"
	"strings"

	"database/sql"
	"github.com/coopernurse/gorp"
	_ "github.com/mattn/go-sqlite3"
)

type Recipient struct {
	To string
}

type Smtp struct {
	Host       string
	Auth       bool
	From       string
	Sender     string
	Recipients []Recipient
	Debug      bool
}

type Feed struct {
	Uri string
}

type Sqlite3 struct {
	DbFile string
	Table  string
}

type Config struct {
	Smtp    Smtp
	Feeds   []Feed
	Sqlite3 Sqlite3
}

type DataStore struct {
	Id         int
	Url        string
	Update     string
	UpdateDate string
}

var wg sync.WaitGroup
var config Config

const utcformat = "2006-01-02T15:04:05Z"
const timeformat = "2006-01-02 15:04:05"

func main() {
	flag.Parse()

	arg := flag.Arg(0)
	log.Println("arg : ", arg)

	configFile := "feed2mail.json"
	jsonBuffer, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Println("ERROR: ", err)
		os.Exit(1)
	}
	//fmt.Println(string(jsonBuffer))

	err = json.Unmarshal(jsonBuffer, &config)
	if err != nil {
		log.Println("ERROR: ", err)
		os.Exit(1)
	}

	log.Printf("%#v\n", config)

	// This sets up a new feed and polls it for new channels/items.
	// Invoking it with 'go PollFeed(...)' to have the polling performed in a
	// separate goroutine, so we can poll mutiple feeds.
	//go PollFeed("http://blog.case.edu/news/feed.atom", 5, nil)

	log.Printf("%#v\n", config.Feeds)

	for _, feed := range config.Feeds {
		log.Printf("%#v\n", feed.Uri)
		wg.Add(1)
		go PollFeed(feed.Uri, 2)
		wg.Wait()
	}
}

//func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
func PollFeed(uri string, timeout int) {
	defer wg.Done()
	feed := rss.New(timeout, true, chanHandler, itemHandler)

	//for {
	if err := feed.Fetch(uri, nil); err != nil {
		fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
		return
	}

	//<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	//}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	log.Printf("%d new channel(s) in %s\n", len(newchannels), feed.Url)
}

func itemHandler(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
	log.Printf("%d new item(s) in %s\n", len(newitems), feed.Url)

	//db, err := sql.Open("sqlite3", "./feed2mail.db")
	db, err := sql.Open("sqlite3", config.Sqlite3.DbFile)
	if err != nil {
		panic(err.Error())
	}
	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}

	//t := dbmap.AddTableWithName(DataStore{}, "feed2mail").SetKeys(true, "Id")
	t := dbmap.AddTableWithName(DataStore{}, config.Sqlite3.Table).SetKeys(true, "Id")
	t.ColMap("Id").Rename("id")
	t.ColMap("Url").Rename("url")
	t.ColMap("Update").Rename("update")
	t.ColMap("UpdateDate").Rename("update_date")

	tx, _ := dbmap.Begin()
	for _, item := range newitems {
		update, _ := time.Parse(utcformat, item.Updated)
		now := time.Now()
		log.Printf("%s %s(%s) %s\n", item.Links[0].Href, update.Format(timeformat), item.Updated, now.Format(timeformat))
		count, _ := dbmap.SelectInt("select count(*) from feed2mail where `URL` = $1", item.Links[0].Href, update.Format(timeformat))
		if count == 0 {
			log.Printf("now %s \n", now.Format(timeformat))
			err = tx.Insert(&DataStore{0, item.Links[0].Href, update.Format(timeformat), now.Format(timeformat)})
			if err != nil {
				log.Println("Insert Error!")
				log.Println(err.Error())
			}
			sendmail(item)
		} else {
			id, _ := dbmap.SelectStr("select `ID` from feed2mail where `URL` = $1 and `UPDATE` < $2", item.Links[0].Href, update.Format(timeformat))
			log.Printf("id = %s", id)
			if id != "" {
				uid, _ := strconv.Atoi(id)
				count, err = tx.Update(&DataStore{uid, item.Links[0].Href, update.Format(timeformat), now.Format(timeformat)})
				if err != nil {
					log.Println("Update Error!")
					log.Println(err.Error())
				}
				sendmail(item)
			}
		}
	}
	tx.Commit()
}

func subjectSplit(title string, length int) []string {
	result := []string{}
	var buf bytes.Buffer

	for k, c := range strings.Split(title, "") {
		buf.WriteString(c)
		if k%length == length-1 {
			result = append(result, buf.String())
			buf.Reset()
		}
	}

	if buf.Len() > 0 {
		result = append(result, buf.String())
	}

	return result
}

func encodeSubject(subject string) string {
	var buf bytes.Buffer

	buf.WriteString("Subject:")
	for _, line := range subjectSplit(subject, 13) {
		buf.WriteString(" =?utf-8?B?")
		buf.WriteString(base64.StdEncoding.EncodeToString([]byte(line)))
		buf.WriteString("?=\r\n")
	}

	return buf.String()
}

func sendmail(item *rss.Item) {
	log.Println("title:", item.Title)
	if len(item.Links) > 0 {
		for i, link := range item.Links {
			log.Println("links", i, ":", link)
		}
	}
	log.Println("author:", item.Author)
	if len(item.Categories) > 0 {
		for i, category := range item.Categories {
			log.Println("categories", i, ":", category)
		}
	}

	pubdate, _ := time.Parse(utcformat, item.PubDate)
	update, _ := time.Parse(utcformat, item.Updated)

	log.Println("pubdate:", item.PubDate)
	log.Println("pubdate:", pubdate.Format(timeformat))
	log.Println("updated:", item.Updated)
	log.Println("updated:", update.Format(timeformat))
	log.Println("content:", item.Content.Text)
	fmt.Println("")

	c, err := smtp.Dial(config.Smtp.Host)
	if err != nil {
		log.Println(err)
	}

	c.Mail(config.Smtp.From)
	for _, recipient := range config.Smtp.Recipients {
		c.Rcpt(recipient.To)
	}

	wc, err := c.Data()
	if err != nil {
		log.Println("Data() ", err)
	}
	defer wc.Close()

	var message bytes.Buffer
	message.WriteString("Title: ")
	message.WriteString(item.Title)
	message.WriteString("
\r\n")
	message.WriteString("Auther: ")
	message.WriteString(item.Author.Name)
	if item.Author.Email != "" {
		message.WriteString(" <")
		message.WriteString(item.Author.Email)
		message.WriteString("> ")
	}
	message.WriteString("
\r\n")
	if item.PubDate != "" {
		message.WriteString("Publish Date: ")
		message.WriteString(pubdate.Format(timeformat))
		message.WriteString("
\r\n")
	}
	message.WriteString("Update Date: ")
	message.WriteString(update.Format(timeformat))
	message.WriteString("
\r\n")
	if len(item.Links) > 0 {
		for _, link := range item.Links {
			message.WriteString("URL: ")
			message.WriteString(link.Href)
			message.WriteString("
\r\n")
		}
	}
	if len(item.Categories) > 0 {
		for _, category := range item.Categories {
			message.WriteString("Category: ")
			message.WriteString(category.Text)
			message.WriteString("
\r\n")
		}
	}
	message.WriteString("---
\r\n")
	message.WriteString(item.Content.Text)

	buf := bytes.NewBufferString("")
	for i, recipient := range config.Smtp.Recipients {
		if i == 0 {
			buf.WriteString("To: " + recipient.To)
		} else {
			buf.WriteString("Bcc: " + recipient.To)
		}
		buf.WriteString("\r\n")
	}
	buf.WriteString(encodeSubject("feed2mail: " + item.Title))
	buf.WriteString("MIME-Version: 1.0\r\n")
	//buf.WriteString("Content-Type: text/plain; charset=\"utf-8\"\r\n")
	buf.WriteString("Content-Type: text/html; charset=\"utf-8\"\r\n")
	buf.WriteString("Content-Transfer-Encoding: base64\r\n")
	buf.WriteString("\r\n")
	buf.WriteString(base64.StdEncoding.EncodeToString(message.Bytes()))
	if _, err = buf.WriteTo(wc); err != nil {
		log.Println("WriteTo() ", err)
	}

	c.Quit()
}
<commit_msg>Linuxのcronで動かしたときの調整<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	rss "github.com/jteeuwen/go-pkg-rss"

	"bytes"
	"net/smtp"
	"strings"

	"database/sql"
	"github.com/coopernurse/gorp"
	_ "github.com/mattn/go-sqlite3"
)

type Recipient struct {
	To string
}

type Smtp struct {
	Host       string
	Auth       bool
	From       string
	Sender     string
	Recipients []Recipient
	Debug      bool
}

type Feed struct {
	Uri string
}

type Sqlite3 struct {
	DbFile string
	Table  string
}

type Config struct {
	Smtp    Smtp
	Feeds   []Feed
	Sqlite3 Sqlite3
}

type DataStore struct {
	Id         int
	Url        string
	Update     string
	UpdateDate string
}

var wg sync.WaitGroup
var config Config

const utcformat = "2006-01-02T15:04:05Z"
const timeformat = "2006-01-02 15:04:05"

func main() {
	flag.Parse()

	arg := flag.Arg(0)
	log.Println("arg : ", arg)

	configFile := "feed2mail.json"
	for _, p := range filepath.SplitList(os.Getenv("PATH")) {
		fmt.Println(p)
		f := filepath.Join(p, configFile)
		_, err := os.Stat(f)
		if err == nil {
			fmt.Println(f)
			configFile = f
			break
		}
	}
	jsonBuffer, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Println("ERROR: ", err)
		os.Exit(1)
	}
	//fmt.Println(string(jsonBuffer))

	err = json.Unmarshal(jsonBuffer, &config)
	if err != nil {
		log.Println("ERROR: ", err)
		os.Exit(1)
	}

	log.Printf("%#v\n", config)

	// This sets up a new feed and polls it for new channels/items.
	// Invoking it with 'go PollFeed(...)' to have the polling performed in a
	// separate goroutine, so we can poll mutiple feeds.
	//go PollFeed("http://blog.case.edu/news/feed.atom", 5, nil)

	log.Printf("%#v\n", config.Feeds)

	for _, feed := range config.Feeds {
		log.Printf("%#v\n", feed.Uri)
		wg.Add(1)
		go PollFeed(feed.Uri, 2)
		wg.Wait()
	}
}

func PollFeed(uri string, timeout int) {
	defer wg.Done()
	feed := rss.New(timeout, true, chanHandler, itemHandler)

	//for {
	if err := feed.Fetch(uri, nil); err != nil {
		fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
		return
	}

	//<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	//}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	log.Printf("%d new channel(s) in %s\n", len(newchannels), feed.Url)
}

func itemHandler(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
	log.Printf("%d new item(s) in %s\n", len(newitems), feed.Url)

	db, err := sql.Open("sqlite3", config.Sqlite3.DbFile)
	if err != nil {
		panic(err.Error())
	}
	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}

	t := dbmap.AddTableWithName(DataStore{}, config.Sqlite3.Table).SetKeys(true, "Id")
	t.ColMap("Id").Rename("id")
	t.ColMap("Url").Rename("url")
	t.ColMap("Update").Rename("update")
	t.ColMap("UpdateDate").Rename("update_date")

	dbmap.CreateTablesIfNotExists()

	tx, _ := dbmap.Begin()
	for _, item := range newitems {
		update, _ := time.Parse(utcformat, item.Updated)
		now := time.Now()
		log.Printf("%s %s(%s) %s\n", item.Links[0].Href, update.Format(timeformat), item.Updated, now.Format(timeformat))
		count, _ := dbmap.SelectInt("select count(*) from feed2mail where `URL` = $1", item.Links[0].Href, update.Format(timeformat))
		if count == 0 {
			log.Printf("now %s \n", now.Format(timeformat))
			err = tx.Insert(&DataStore{0, item.Links[0].Href, update.Format(timeformat), now.Format(timeformat)})
			if err != nil {
				log.Println("Insert Error!")
				log.Println(err.Error())
			}
			sendmail(item)
		} else {
			id, _ := dbmap.SelectStr("select `ID` from feed2mail where `URL` = $1 and `UPDATE` < $2", item.Links[0].Href, update.Format(timeformat))
			log.Printf("id = %s", id)
			if id != "" {
				uid, _ := strconv.Atoi(id)
				count, err = tx.Update(&DataStore{uid, item.Links[0].Href, update.Format(timeformat), now.Format(timeformat)})
				if err != nil {
					log.Println("Update Error!")
					log.Println(err.Error())
				}
				sendmail(item)
			}
		}
	}
	tx.Commit()
}

func subjectSplit(title string, length int) []string {
	result := []string{}
	var buf bytes.Buffer

	for k, c := range strings.Split(title, "") {
		buf.WriteString(c)
		if k%length == length-1 {
			result = append(result, buf.String())
			buf.Reset()
		}
	}

	if buf.Len() > 0 {
		result = append(result, buf.String())
	}

	return result
}

func encodeSubject(subject string) string {
	var buf bytes.Buffer

	buf.WriteString("Subject:")
	for _, line := range subjectSplit(subject, 13) {
		buf.WriteString(" =?utf-8?B?")
		buf.WriteString(base64.StdEncoding.EncodeToString([]byte(line)))
		buf.WriteString("?=\r\n")
	}

	return buf.String()
}

func sendmail(item *rss.Item) {
	log.Println("title:", item.Title)
	if len(item.Links) > 0 {
		for i, link := range item.Links {
			log.Println("links", i, ":", link)
		}
	}
	log.Println("author:", item.Author)
	if len(item.Categories) > 0 {
		for i, category := range item.Categories {
			log.Println("categories", i, ":", category)
		}
	}

	pubdate, _ := time.Parse(utcformat, item.PubDate)
	update, _ := time.Parse(utcformat, item.Updated)

	log.Println("pubdate:", item.PubDate)
	log.Println("pubdate:", pubdate.Format(timeformat))
	log.Println("updated:", item.Updated)
	log.Println("updated:", update.Format(timeformat))
	log.Println("content:", item.Content.Text)
	fmt.Println("")

	c, err := smtp.Dial(config.Smtp.Host)
	if err != nil {
		log.Println(err)
	}

	c.Mail(config.Smtp.From)
	for _, recipient := range config.Smtp.Recipients {
		c.Rcpt(recipient.To)
	}

	wc, err := c.Data()
	if err != nil {
		log.Println("Data() ", err)
	}
	defer wc.Close()

	var message bytes.Buffer
	message.WriteString("Title: ")
	message.WriteString(item.Title)
	message.WriteString("
\r\n")
	message.WriteString("Auther: ")
	message.WriteString(item.Author.Name)
	if item.Author.Email != "" {
		message.WriteString(" <")
		message.WriteString(item.Author.Email)
		message.WriteString("> ")
	}
	message.WriteString("
\r\n")
	if item.PubDate != "" {
		message.WriteString("Publish Date: ")
		message.WriteString(pubdate.Format(timeformat))
		message.WriteString("
\r\n")
	}
	message.WriteString("Update Date: ")
	message.WriteString(update.Format(timeformat))
	message.WriteString("
\r\n")
	if len(item.Links) > 0 {
		for _, link := range item.Links {
			message.WriteString("URL: ")
			message.WriteString(link.Href)
			message.WriteString("
\r\n")
		}
	}
	if len(item.Categories) > 0 {
		for _, category := range item.Categories {
			message.WriteString("Category: ")
			message.WriteString(category.Text)
			message.WriteString("
\r\n")
		}
	}
	message.WriteString("---
\r\n")
	message.WriteString(item.Content.Text)

	buf := bytes.NewBufferString("")
	for i, recipient := range config.Smtp.Recipients {
		if i == 0 {
			buf.WriteString("To: " + recipient.To)
		} else {
			buf.WriteString("Bcc: " + recipient.To)
		}
		buf.WriteString("\r\n")
	}
	buf.WriteString(encodeSubject("feed2mail: " + item.Title))
	buf.WriteString("MIME-Version: 1.0\r\n")
	//buf.WriteString("Content-Type: text/plain; charset=\"utf-8\"\r\n")
	buf.WriteString("Content-Type: text/html; charset=\"utf-8\"\r\n")
	buf.WriteString("Content-Transfer-Encoding: base64\r\n")
	buf.WriteString("\r\n")
	buf.WriteString(base64.StdEncoding.EncodeToString(message.Bytes()))
	if _, err = buf.WriteTo(wc); err != nil {
		log.Println("WriteTo() ", err)
	}

	c.Quit()
}
<|endoftext|>


<commit_before>// package merkledag implements the IPFS Merkle DAG datastructures.
package merkledag

import (
	"context"
	"fmt"
	"strings"
	"sync"

	blocks "github.com/ipfs/go-ipfs/blocks"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	offline "github.com/ipfs/go-ipfs/exchange/offline"

	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
	node "gx/ipfs/QmYDscK7dmdo2GZ9aumS8s5auUUAH5mR1jvj5pYhWusfK7/go-ipld-node"
	ipldcbor "gx/ipfs/QmdaC21UyoyN3t9QdapHZfsaUo3mqVf5p4CEuFaYVFqwap/go-ipld-cbor"
)

var log = logging.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(node.Node) (*cid.Cid, error)
	Get(context.Context, *cid.Cid) (node.Node, error)
	Remove(node.Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetMany(context.Context, []*cid.Cid) <-chan *NodeOption

	Batch() *Batch

	LinkService
}

type LinkService interface {
	// Return all links for a node, may be more effect than
	// calling Get in DAGService
	GetLinks(context.Context, *cid.Cid) ([]*node.Link, error)

	GetOfflineLinkService() LinkService
}

func NewDAGService(bs bserv.BlockService) *dagService {
	return &dagService{Blocks: bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return nil, fmt.Errorf("dagService is nil")
	}

	return n.Blocks.AddBlock(nd)
}

func (n *dagService) Batch() *Batch {
	return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	b, err := n.Blocks.GetBlock(ctx, c)
	if err != nil {
		if err == bserv.ErrNotFound {
			return nil, ErrNotFound
		}
		return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
	}

	return decodeBlock(b)
}

func decodeBlock(b blocks.Block) (node.Node, error) {
	c := b.Cid()

	switch c.Type() {
	case cid.DagProtobuf:
		decnd, err := DecodeProtobuf(b.RawData())
		if err != nil {
			if strings.Contains(err.Error(), "Unmarshal failed") {
				return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c)
			}
			return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
		}

		decnd.cached = b.Cid()
		decnd.Prefix = b.Cid().Prefix()
		return decnd, nil
	case cid.Raw:
		return NewRawNode(b.RawData()), nil
	case cid.DagCBOR:
		return ipldcbor.Decode(b.RawData())
	default:
		return nil, fmt.Errorf("unrecognized object type: %s", c.Type())
	}
}

func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) {
	if c.Type() == cid.Raw {
		return nil, nil
	}
	node, err := n.Get(ctx, c)
	if err != nil {
		return nil, err
	}
	return node.Links(), nil
}

func (n *dagService) GetOfflineLinkService() LinkService {
	if n.Blocks.Exchange().IsOnline() {
		bsrv := bserv.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
		return NewDAGService(bsrv)
	} else {
		return n
	}
}

func (n *dagService) Remove(nd node.Node) error {
	return n.Blocks.DeleteBlock(nd)
}

// FetchGraph fetches all nodes that are children of the given node
func FetchGraph(ctx context.Context, c *cid.Cid, serv DAGService) error {
	return EnumerateChildrenAsync(ctx, serv, c, cid.NewSet().Visit)
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(links []*cid.Cid, c *cid.Cid, start int) []int {
	var out []int
	for i, lnk_c := range links[start:] {
		if c.Equals(lnk_c) {
			out = append(out, i+start)
		}
	}
	return out
}

type NodeOption struct {
	Node node.Node
	Err  error
}

func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *NodeOption {
	out := make(chan *NodeOption, len(keys))
	blocks := ds.Blocks.GetBlocks(ctx, keys)
	var count int

	go func() {
		defer close(out)
		for {
			select {
			case b, ok := <-blocks:
				if !ok {
					if count != len(keys) {
						out <- &NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
					}
					return
				}

				nd, err := decodeBlock(b)
				if err != nil {
					out <- &NodeOption{Err: err}
					return
				}

				out <- &NodeOption{Node: nd}
				count++

			case <-ctx.Done():
				out <- &NodeOption{Err: ctx.Err()}
				return
			}
		}
	}()
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func GetDAG(ctx context.Context, ds DAGService, root node.Node) []NodeGetter {
	var cids []*cid.Cid
	for _, lnk := range root.Links() {
		cids = append(cids, lnk.Cid)
	}

	return GetNodes(ctx, ds, cids)
}

// GetNodes returns an array of 'NodeGetter' promises, with each corresponding
// to the key with the same index as the passed in keys
func GetNodes(ctx context.Context, ds DAGService, keys []*cid.Cid) []NodeGetter {

	// Early out if no work to do
	if len(keys) == 0 {
		return nil
	}

	promises := make([]NodeGetter, len(keys))
	for i := range keys {
		promises[i] = newNodePromise(ctx)
	}

	dedupedKeys := dedupeKeys(keys)
	go func() {
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		nodechan := ds.GetMany(ctx, dedupedKeys)

		for count := 0; count < len(keys); {
			select {
			case opt, ok := <-nodechan:
				if !ok {
					for _, p := range promises {
						p.Fail(ErrNotFound)
					}
					return
				}

				if opt.Err != nil {
					for _, p := range promises {
						p.Fail(opt.Err)
					}
					return
				}

				nd := opt.Node
				is := FindLinks(keys, nd.Cid(), 0)
				for _, i := range is {
					count++
					promises[i].Send(nd)
				}
			case <-ctx.Done():
				return
			}
		}
	}()
	return promises
}

// Remove duplicates from a list of keys
func dedupeKeys(cids []*cid.Cid) []*cid.Cid {
	set := cid.NewSet()
	for _, c := range cids {
		set.Add(c)
	}
	return set.Keys()
}

func newNodePromise(ctx context.Context) NodeGetter {
	return &nodePromise{
		recv: make(chan node.Node, 1),
		ctx:  ctx,
		err:  make(chan error, 1),
	}
}

type nodePromise struct {
	cache node.Node
	clk   sync.Mutex
	recv  chan node.Node
	ctx   context.Context
	err   chan error
}

// NodeGetter provides a promise like interface for a dag Node
// the first call to Get will block until the Node is received
// from its internal channels, subsequent calls will return the
// cached node.
type NodeGetter interface {
	Get(context.Context) (node.Node, error)
	Fail(err error)
	Send(node.Node)
}

func (np *nodePromise) Fail(err error) {
	np.clk.Lock()
	v := np.cache
	np.clk.Unlock()

	// if promise has a value, don't fail it
	if v != nil {
		return
	}

	np.err <- err
}

func (np *nodePromise) Send(nd node.Node) {
	var already bool
	np.clk.Lock()
	if np.cache != nil {
		already = true
	}
	np.cache = nd
	np.clk.Unlock()

	if already {
		panic("sending twice to the same promise is an error!")
	}

	np.recv <- nd
}

func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
	np.clk.Lock()
	c := np.cache
	np.clk.Unlock()
	if c != nil {
		return c, nil
	}

	select {
	case nd := <-np.recv:
		return nd, nil
	case <-np.ctx.Done():
		return nil, np.ctx.Err()
	case <-ctx.Done():
		return nil, ctx.Err()
	case err := <-np.err:
		return nil, err
	}
}

type Batch struct {
	ds *dagService

	blocks  []blocks.Block
	size    int
	MaxSize int
}

func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
	t.blocks = append(t.blocks, nd)
	t.size += len(nd.RawData())
	if t.size > t.MaxSize {
		return nd.Cid(), t.Commit()
	}
	return nd.Cid(), nil
}

func (t *Batch) Commit() error {
	_, err := t.ds.Blocks.AddBlocks(t.blocks)
	t.blocks = nil
	t.size = 0
	return err
}

// EnumerateChildren will walk the dag below the given root node and add all
// unseen children to the passed in set.
// TODO: parallelize to avoid disk latency perf hits?
func EnumerateChildren(ctx context.Context, ds LinkService, root *cid.Cid, visit func(*cid.Cid) bool, bestEffort bool) error {
	links, err := ds.GetLinks(ctx, root)
	if bestEffort && err == ErrNotFound {
		return nil
	} else if err != nil {
		return err
	}
	for _, lnk := range links {
		c := lnk.Cid
		if visit(c) {
			err = EnumerateChildren(ctx, ds, c, visit, bestEffort)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// FetchGraphConcurrency is total number of concurrent fetches that
// 'fetchNodes' will start at a time
var FetchGraphConcurrency = 8

func EnumerateChildrenAsync(ctx context.Context, ds DAGService, c *cid.Cid, visit func(*cid.Cid) bool) error {
	feed := make(chan *cid.Cid)
	out := make(chan node.Node)
	done := make(chan struct{})

	var setlk sync.Mutex

	errChan := make(chan error)
	fetchersCtx, cancel := context.WithCancel(ctx)

	defer cancel()

	for i := 0; i < FetchGraphConcurrency; i++ {
		go func() {
			for ic := range feed {
				n, err := ds.Get(ctx, ic)
				if err != nil {
					errChan <- err
					return
				}

				setlk.Lock()
				unseen := visit(ic)
				setlk.Unlock()

				if unseen {
					select {
					case out <- n:
					case <-fetchersCtx.Done():
						return
					}
				}
				select {
				case done <- struct{}{}:
				case <-fetchersCtx.Done():
				}
			}
		}()
	}
	defer close(feed)

	send := feed
	var todobuffer []*cid.Cid
	var inProgress int

	next := c
	for {
		select {
		case send <- next:
			inProgress++
			if len(todobuffer) > 0 {
				next = todobuffer[0]
				todobuffer = todobuffer[1:]
			} else {
				next = nil
				send = nil
			}
		case <-done:
			inProgress--
			if inProgress == 0 && next == nil {
				return nil
			}
		case nd := <-out:
			for _, lnk := range nd.Links() {
				if next == nil {
					next = lnk.Cid
					send = feed
				} else {
					todobuffer = append(todobuffer, lnk.Cid)
				}
			}
		case err := <-errChan:
			return err

		case <-ctx.Done():
			return ctx.Err()
		}
	}

}
<commit_msg>Fix formatting in merkledag.go<commit_after>// package merkledag implements the IPFS Merkle DAG datastructures.
package merkledag

import (
	"context"
	"fmt"
	"strings"
	"sync"

	blocks "github.com/ipfs/go-ipfs/blocks"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	offline "github.com/ipfs/go-ipfs/exchange/offline"

	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
	node "gx/ipfs/QmYDscK7dmdo2GZ9aumS8s5auUUAH5mR1jvj5pYhWusfK7/go-ipld-node"
	ipldcbor "gx/ipfs/QmdaC21UyoyN3t9QdapHZfsaUo3mqVf5p4CEuFaYVFqwap/go-ipld-cbor"
)

var log = logging.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(node.Node) (*cid.Cid, error)
	Get(context.Context, *cid.Cid) (node.Node, error)
	Remove(node.Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetMany(context.Context, []*cid.Cid) <-chan *NodeOption

	Batch() *Batch

	LinkService
}

type LinkService interface {
	// Return all links for a node, may be more effect than
	// calling Get in DAGService
	GetLinks(context.Context, *cid.Cid) ([]*node.Link, error)

	GetOfflineLinkService() LinkService
}

func NewDAGService(bs bserv.BlockService) *dagService {
	return &dagService{Blocks: bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return nil, fmt.Errorf("dagService is nil")
	}

	return n.Blocks.AddBlock(nd)
}

func (n *dagService) Batch() *Batch {
	return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	b, err := n.Blocks.GetBlock(ctx, c)
	if err != nil {
		if err == bserv.ErrNotFound {
			return nil, ErrNotFound
		}
		return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
	}

	return decodeBlock(b)
}

func decodeBlock(b blocks.Block) (node.Node, error) {
	c := b.Cid()

	switch c.Type() {
	case cid.DagProtobuf:
		decnd, err := DecodeProtobuf(b.RawData())
		if err != nil {
			if strings.Contains(err.Error(), "Unmarshal failed") {
				return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c)
			}
			return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
		}

		decnd.cached = b.Cid()
		decnd.Prefix = b.Cid().Prefix()
		return decnd, nil
	case cid.Raw:
		return NewRawNode(b.RawData()), nil
	case cid.DagCBOR:
		return ipldcbor.Decode(b.RawData())
	default:
		return nil, fmt.Errorf("unrecognized object type: %s", c.Type())
	}
}

func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) {
	if c.Type() == cid.Raw {
		return nil, nil
	}
	node, err := n.Get(ctx, c)
	if err != nil {
		return nil, err
	}
	return node.Links(), nil
}

func (n *dagService) GetOfflineLinkService() LinkService {
	if n.Blocks.Exchange().IsOnline() {
		bsrv := bserv.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
		return NewDAGService(bsrv)
	} else {
		return n
	}
}

func (n *dagService) Remove(nd node.Node) error {
	return n.Blocks.DeleteBlock(nd)
}

// FetchGraph fetches all nodes that are children of the given node
func FetchGraph(ctx context.Context, c *cid.Cid, serv DAGService) error {
	return EnumerateChildrenAsync(ctx, serv, c, cid.NewSet().Visit)
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(links []*cid.Cid, c *cid.Cid, start int) []int {
	var out []int
	for i, lnk_c := range links[start:] {
		if c.Equals(lnk_c) {
			out = append(out, i+start)
		}
	}
	return out
}

type NodeOption struct {
	Node node.Node
	Err  error
}

func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *NodeOption {
	out := make(chan *NodeOption, len(keys))
	blocks := ds.Blocks.GetBlocks(ctx, keys)
	var count int

	go func() {
		defer close(out)
		for {
			select {
			case b, ok := <-blocks:
				if !ok {
					if count != len(keys) {
						out <- &NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
					}
					return
				}

				nd, err := decodeBlock(b)
				if err != nil {
					out <- &NodeOption{Err: err}
					return
				}

				out <- &NodeOption{Node: nd}
				count++

			case <-ctx.Done():
				out <- &NodeOption{Err: ctx.Err()}
				return
			}
		}
	}()
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func GetDAG(ctx context.Context, ds DAGService, root node.Node) []NodeGetter {
	var cids []*cid.Cid
	for _, lnk := range root.Links() {
		cids = append(cids, lnk.Cid)
	}

	return GetNodes(ctx, ds, cids)
}

// GetNodes returns an array of 'NodeGetter' promises, with each corresponding
// to the key with the same index as the passed in keys
func GetNodes(ctx context.Context, ds DAGService, keys []*cid.Cid) []NodeGetter {

	// Early out if no work to do
	if len(keys) == 0 {
		return nil
	}

	promises := make([]NodeGetter, len(keys))
	for i := range keys {
		promises[i] = newNodePromise(ctx)
	}

	dedupedKeys := dedupeKeys(keys)
	go func() {
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		nodechan := ds.GetMany(ctx, dedupedKeys)

		for count := 0; count < len(keys); {
			select {
			case opt, ok := <-nodechan:
				if !ok {
					for _, p := range promises {
						p.Fail(ErrNotFound)
					}
					return
				}

				if opt.Err != nil {
					for _, p := range promises {
						p.Fail(opt.Err)
					}
					return
				}

				nd := opt.Node
				is := FindLinks(keys, nd.Cid(), 0)
				for _, i := range is {
					count++
					promises[i].Send(nd)
				}
			case <-ctx.Done():
				return
			}
		}
	}()
	return promises
}

// Remove duplicates from a list of keys
func dedupeKeys(cids []*cid.Cid) []*cid.Cid {
	set := cid.NewSet()
	for _, c := range cids {
		set.Add(c)
	}
	return set.Keys()
}

func newNodePromise(ctx context.Context) NodeGetter {
	return &nodePromise{
		recv: make(chan node.Node, 1),
		ctx:  ctx,
		err:  make(chan error, 1),
	}
}

type nodePromise struct {
	cache node.Node
	clk   sync.Mutex
	recv  chan node.Node
	ctx   context.Context
	err   chan error
}

// NodeGetter provides a promise like interface for a dag Node
// the first call to Get will block until the Node is received
// from its internal channels, subsequent calls will return the
// cached node.
type NodeGetter interface {
	Get(context.Context) (node.Node, error)
	Fail(err error)
	Send(node.Node)
}

func (np *nodePromise) Fail(err error) {
	np.clk.Lock()
	v := np.cache
	np.clk.Unlock()

	// if promise has a value, don't fail it
	if v != nil {
		return
	}

	np.err <- err
}

func (np *nodePromise) Send(nd node.Node) {
	var already bool
	np.clk.Lock()
	if np.cache != nil {
		already = true
	}
	np.cache = nd
	np.clk.Unlock()

	if already {
		panic("sending twice to the same promise is an error!")
	}

	np.recv <- nd
}

func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
	np.clk.Lock()
	c := np.cache
	np.clk.Unlock()
	if c != nil {
		return c, nil
	}

	select {
	case nd := <-np.recv:
		return nd, nil
	case <-np.ctx.Done():
		return nil, np.ctx.Err()
	case <-ctx.Done():
		return nil, ctx.Err()
	case err := <-np.err:
		return nil, err
	}
}

type Batch struct {
	ds *dagService

	blocks  []blocks.Block
	size    int
	MaxSize int
}

func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
	t.blocks = append(t.blocks, nd)
	t.size += len(nd.RawData())
	if t.size > t.MaxSize {
		return nd.Cid(), t.Commit()
	}
	return nd.Cid(), nil
}

func (t *Batch) Commit() error {
	_, err := t.ds.Blocks.AddBlocks(t.blocks)
	t.blocks = nil
	t.size = 0
	return err
}

// EnumerateChildren will walk the dag below the given root node and add all
// unseen children to the passed in set.
// TODO: parallelize to avoid disk latency perf hits?
func EnumerateChildren(ctx context.Context, ds LinkService, root *cid.Cid, visit func(*cid.Cid) bool, bestEffort bool) error {
	links, err := ds.GetLinks(ctx, root)
	if bestEffort && err == ErrNotFound {
		return nil
	} else if err != nil {
		return err
	}
	for _, lnk := range links {
		c := lnk.Cid
		if visit(c) {
			err = EnumerateChildren(ctx, ds, c, visit, bestEffort)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// FetchGraphConcurrency is total number of concurrent fetches that
// 'fetchNodes' will start at a time
var FetchGraphConcurrency = 8

func EnumerateChildrenAsync(ctx context.Context, ds DAGService, c *cid.Cid, visit func(*cid.Cid) bool) error {
	feed := make(chan *cid.Cid)
	out := make(chan node.Node)
	done := make(chan struct{})

	var setlk sync.Mutex

	errChan := make(chan error)
	fetchersCtx, cancel := context.WithCancel(ctx)

	defer cancel()

	for i := 0; i < FetchGraphConcurrency; i++ {
		go func() {
			for ic := range feed {
				n, err := ds.Get(ctx, ic)
				if err != nil {
					errChan <- err
					return
				}

				setlk.Lock()
				unseen := visit(ic)
				setlk.Unlock()

				if unseen {
					select {
					case out <- n:
					case <-fetchersCtx.Done():
						return
					}
				}
				select {
				case done <- struct{}{}:
				case <-fetchersCtx.Done():
				}
			}
		}()
	}
	defer close(feed)

	send := feed
	var todobuffer []*cid.Cid
	var inProgress int

	next := c
	for {
		select {
		case send <- next:
			inProgress++
			if len(todobuffer) > 0 {
				next = todobuffer[0]
				todobuffer = todobuffer[1:]
			} else {
				next = nil
				send = nil
			}
		case <-done:
			inProgress--
			if inProgress == 0 && next == nil {
				return nil
			}
		case nd := <-out:
			for _, lnk := range nd.Links() {
				if next == nil {
					next = lnk.Cid
					send = feed
				} else {
					todobuffer = append(todobuffer, lnk.Cid)
				}
			}
		case err := <-errChan:
			return err

		case <-ctx.Done():
			return ctx.Err()
		}
	}

}
<|endoftext|>


<commit_before>package presenter

import (
	"io"

	"github.com/anchore/imgbom/imgbom/pkg"
	"github.com/anchore/imgbom/imgbom/presenter/json"
	"github.com/anchore/imgbom/imgbom/presenter/text"
	"github.com/anchore/stereoscope/pkg/image"
)

type Presenter interface {
	Present(io.Writer) error
}

func GetPresenter(option Option, img *image.Image, catalog *pkg.Catalog) Presenter {
	switch option {
	case JSONPresenter:
		return json.NewPresenter(img, catalog)
	case TextPresenter:
		return text.NewPresenter(img, catalog)

	default:
		return nil
	}
}
<commit_msg>presenter can pick text-dir, text-img, json-text, or json-img for reporting<commit_after>package presenter

import (
	"io"

	"github.com/anchore/imgbom/imgbom/pkg"
	json_dirs "github.com/anchore/imgbom/imgbom/presenter/json/dirs"
	json_imgs "github.com/anchore/imgbom/imgbom/presenter/json/imgs"
	text_dirs "github.com/anchore/imgbom/imgbom/presenter/text/dirs"
	text_imgs "github.com/anchore/imgbom/imgbom/presenter/text/imgs"
	"github.com/anchore/stereoscope/pkg/image"
)

type Presenter interface {
	Present(io.Writer) error
}

func GetImgPresenter(option Option, img *image.Image, catalog *pkg.Catalog) Presenter {
	switch option {
	case JSONPresenter:
		return json_imgs.NewPresenter(img, catalog)
	case TextPresenter:
		return text_imgs.NewPresenter(img, catalog)
	default:
		return nil
	}
}

func GetDirPresenter(option Option, path string, catalog *pkg.Catalog) Presenter {
	switch option {
	case JSONPresenter:
		return json_dirs.NewPresenter(catalog, path)
	case TextPresenter:
		return text_dirs.NewPresenter(catalog, path)
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>// +build !windows

package timeout

import (
	"os/exec"
	"syscall"
	"testing"
	"time"
)

func TestRunSimple_withStop(t *testing.T) {
	tio := &Timeout{
		Duration:  2 * time.Second,
		KillAfter: 1 * time.Second,
		Cmd:       exec.Command(shellcmd, shellflag, "sleep 10"),
	}
	ch, err := tio.RunCommand()
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	tio.Cmd.Process.Signal(syscall.SIGSTOP)
	st := <-ch

	expect := 128 + 15
	if st.Code != expect {
		t.Errorf("exit code invalid. out: %d, expect: %d", st.Code, expect)
	}
}
<commit_msg>test for signaled<commit_after>// +build !windows

package timeout

import (
	"os/exec"
	"syscall"
	"testing"
	"time"
)

func TestRunSimple_withStop(t *testing.T) {
	tio := &Timeout{
		Duration:  2 * time.Second,
		KillAfter: 1 * time.Second,
		Cmd:       exec.Command(shellcmd, shellflag, "sleep 10"),
	}
	ch, err := tio.RunCommand()
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	tio.Cmd.Process.Signal(syscall.SIGSTOP)
	st := <-ch

	expect := 128 + 15
	if st.Code != expect {
		t.Errorf("exit code invalid. out: %d, expect: %d", st.Code, expect)
	}
}

func TestRunCommand_signaled(t *testing.T) {
	testCases := []struct {
		name     string
		cmd      *exec.Cmd
		exit     int
		signaled bool
	}{
		{
			name:     "signal handled",
			cmd:      exec.Command("perl", "testdata/exit_with_23.pl"),
			exit:     23,
			signaled: false,
		},
		{
			name:     "termed by sigterm",
			cmd:      exec.Command("sleep", "3"),
			exit:     143,
			signaled: true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			tio := &Timeout{
				Duration:  time.Second,
				KillAfter: 3 * time.Second,
				Cmd:       tc.cmd,
			}
			st, _, _, err := tio.Run()

			if err != nil {
				t.Errorf("error should be nil but: %s", err)
			}

			if st.GetChildExitCode() != tc.exit {
				t.Errorf("expected exitcode: %d, but: %d", tc.exit, st.GetChildExitCode())
			}
			if st.Signaled != tc.signaled {
				t.Errorf("something went wrong")
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package file

// package gt

import (
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// FileExist checks whether a file or directory exists.
// It returns false when the file or directory does not exist.
func FileExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func PathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// FileSize returns file size in bytes and possible error.
func FileSize(file string) (int64, error) {
	f, err := os.Stat(file)
	if err != nil {
		return 0, err
	}
	return f.Size(), nil
}

func OFileSha(filepath string, args ...string) (string, error) {
	file, err := os.Open(filepath)
	if err != nil {
		return "", err
	}
	defer file.Close()

	var sha string

	if len(args) > 0 {
		sha, err = FileSha(file, args[0])
	} else {
		sha, err = FileSha(file)
	}

	if err != nil {
		return "", err
	}

	return sha, nil
}

func FileSha(file *os.File, args ...string) (string, error) {
	var h hash.Hash

	if len(args) > 0 {
		h = sha256.New()
	} else {
		h = sha1.New()
	}

	_, err := io.Copy(h, file)
	if err != nil {
		return "", err
	}

	sha := fmt.Sprintf("%x", h.Sum(nil))

	return sha, nil
}

// Copy copies file from source to target path.
func Copy(src, dest string) error {
	// Gather file information to set back later.
	si, err := os.Lstat(src)
	if err != nil {
		return err
	}

	// Handle symbolic link.
	if si.Mode()&os.ModeSymlink != 0 {
		target, err := os.Readlink(src)
		if err != nil {
			return err
		}
		// NOTE: os.Chmod and os.Chtimes don't recoganize symbolic link,
		// which will lead "no such file or directory" error.
		return os.Symlink(target, dest)
	}

	sr, err := os.Open(src)
	if err != nil {
		return err
	}
	defer sr.Close()

	dw, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer dw.Close()

	if _, err = io.Copy(dw, sr); err != nil {
		return err
	}

	// Set back file information.
	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
		return err
	}
	return os.Chmod(dest, si.Mode())
}

// CopyFile copies file from source to target path.
func CopyFile(src, dst string) (w int64, err error) {
	srcFile, err := os.Open(src)
	if err != nil {
		return
	}
	defer srcFile.Close()
	// if FileExist(dst) != true {
	if !FileExist(dst) {
		Writefile("", dst)
	}
	dstFile, err := os.Create(dst)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	defer dstFile.Close()
	return io.Copy(dstFile, srcFile)
}

func CopyOFile(srcName, dstName string) (written int64, err error) {
	src, err := os.Open(srcName)
	if err != nil {
		return
	}
	defer src.Close()
	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer dst.Close()
	return io.Copy(dst, src)
}

// Readfile read file and return string
func Readfile(fname string) (string, error) {
	userFile := fname
	fin, err := os.Open(userFile)
	defer fin.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return "", err
	}
	var restr string = ""

	buf := make([]byte, 1024)
	for {
		n, _ := fin.Read(buf)
		if 0 == n {
			break
		}
		// os.Stdout.Write(buf[:n])

		strbuf := string(buf[:n])

		restr += strbuf
	}

	return restr, nil
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func WriteFile(filename string, data []byte) error {
	os.MkdirAll(path.Dir(filename), os.ModePerm)
	return ioutil.WriteFile(filename, data, 0655)
}

// Writefile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func Writefile(wirtestr string, userFile string) {
	os.MkdirAll(path.Dir(userFile), os.ModePerm)

	fout, err := os.Create(userFile)
	defer fout.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return
	}

	fout.WriteString(wirtestr)
}

func ListFile(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func ListDir(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if !fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func WalkFile(dirPth, suffix string) (files []string, err error) {
	files = make([]string, 0, 30)
	suffix = strings.ToUpper(suffix)
	err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}

		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, filename)
		}
		return nil
	})
	return files, err
}

func WalkDir(dirPth, suffix string) (files []string, err error) {
	files = make([]string, 0, 30)
	suffix = strings.ToUpper(suffix)
	err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {

		if !fi.IsDir() {
			return nil
		}

		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, filename)
		}
		return nil
	})
	return files, err
}
<commit_msg>Fix name err<commit_after>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package file

// package gt

import (
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// FileExist checks whether a file or directory exists.
// It returns false when the file or directory does not exist.
func FileExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func PathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// FileSize returns file size in bytes and possible error.
func FileSize(file string) (int64, error) {
	f, err := os.Stat(file)
	if err != nil {
		return 0, err
	}
	return f.Size(), nil
}

func OFileSha(filepath string, args ...string) (string, error) {
	file, err := os.Open(filepath)
	if err != nil {
		return "", err
	}
	defer file.Close()

	var sha string

	if len(args) > 0 {
		sha, err = FileSha(file, args[0])
	} else {
		sha, err = FileSha(file)
	}

	if err != nil {
		return "", err
	}

	return sha, nil
}

func FileSha(file *os.File, args ...string) (string, error) {
	var h hash.Hash

	if len(args) > 0 {
		h = sha256.New()
	} else {
		h = sha1.New()
	}

	_, err := io.Copy(h, file)
	if err != nil {
		return "", err
	}

	sha := fmt.Sprintf("%x", h.Sum(nil))

	return sha, nil
}

// Copy copies file from source to target path.
func Copy(src, dest string) error {
	// Gather file information to set back later.
	si, err := os.Lstat(src)
	if err != nil {
		return err
	}

	// Handle symbolic link.
	if si.Mode()&os.ModeSymlink != 0 {
		target, err := os.Readlink(src)
		if err != nil {
			return err
		}
		// NOTE: os.Chmod and os.Chtimes don't recoganize symbolic link,
		// which will lead "no such file or directory" error.
		return os.Symlink(target, dest)
	}

	sr, err := os.Open(src)
	if err != nil {
		return err
	}
	defer sr.Close()

	dw, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer dw.Close()

	if _, err = io.Copy(dw, sr); err != nil {
		return err
	}

	// Set back file information.
	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
		return err
	}
	return os.Chmod(dest, si.Mode())
}

// CopyFile copies file from source to target path.
func CopyFile(src, dst string) (w int64, err error) {
	srcFile, err := os.Open(src)
	if err != nil {
		return
	}
	defer srcFile.Close()
	// if FileExist(dst) != true {
	if !FileExist(dst) {
		Writefile("", dst)
	}
	dstFile, err := os.Create(dst)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	defer dstFile.Close()
	return io.Copy(dstFile, srcFile)
}

func CopyOFile(srcName, dstName string) (written int64, err error) {
	src, err := os.Open(srcName)
	if err != nil {
		return
	}
	defer src.Close()
	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer dst.Close()
	return io.Copy(dst, src)
}

// Readfile read file and return string
func Readfile(fname string) (string, error) {
	userFile := fname
	fin, err := os.Open(userFile)
	defer fin.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return "", err
	}
	var restr string = ""

	buf := make([]byte, 1024)
	for {
		n, _ := fin.Read(buf)
		if 0 == n {
			break
		}
		// os.Stdout.Write(buf[:n])

		strbuf := string(buf[:n])

		restr += strbuf
	}

	return restr, nil
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func WriteFile(filename string, data []byte) error {
	os.MkdirAll(path.Dir(filename), os.ModePerm)
	return ioutil.WriteFile(filename, data, 0655)
}

// Writefile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it
// and its upper level paths.
func Writefile(writeStr string, userFile string) {
	os.MkdirAll(path.Dir(userFile), os.ModePerm)

	fout, err := os.Create(userFile)
	defer fout.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return
	}

	fout.WriteString(writeStr)
}

func ListFile(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func ListDir(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if !fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func WalkFile(dirPth, suffix string) (files []string, err error) {
	files = make([]string, 0, 30)
	suffix = strings.ToUpper(suffix)
	err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}

		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, filename)
		}
		return nil
	})
	return files, err
}

func WalkDir(dirPth, suffix string) (files []string, err error) {
	files = make([]string, 0, 30)
	suffix = strings.ToUpper(suffix)
	err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {

		if !fi.IsDir() {
			return nil
		}

		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, filename)
		}
		return nil
	})
	return files, err
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2016 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package file

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
)

const (
	Name    = "file"
	Version = 3
)

type filePublisher struct {
}

type MetricToPublish struct {
	// The timestamp from when the metric was created.
	Timestamp time.Time         `json:"timestamp"`
	Namespace string            `json:"namespace"`
	Data      interface{}       `json:"data"`
	Unit      string            `json:"unit"`
	Tags      map[string]string `json:"tags"`
	Version   int64             `json:"version"`
}

//New returns an instance of filePublisher
func New() *filePublisher {
	return &filePublisher{}
}

func (f *filePublisher) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	logger := log.New()
	logger.Debug("Publishing started")

	destination, err := cfg.GetString("file")
	if err != nil {
		logger.Error("Unable to ")
		return fmt.Errorf("%s: %s", err, "file")
	}

	logger.Infof("publishing %v metrics to %s", len(mts), destination)
	file, err := os.OpenFile(destination, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	defer file.Close()
	if err != nil {
		logger.Printf("Error: %v", err)
		return err
	}
	w := bufio.NewWriter(file)

	// format metrics types to metrics to be published
	metrics := formatMetricTypes(mts)
	jsonOut, err := json.Marshal(metrics)
	if err != nil {
		return fmt.Errorf("Error while marshalling metrics to JSON: %v", err)
	}

	w.Write(jsonOut)
	w.WriteString("\n")
	w.Flush()

	return nil
}

func (f *filePublisher) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()
	policy.AddNewStringRule([]string{""}, "file", true)
	return *policy, nil
}

// formatMetricTypes returns metrics in format to be publish as a JSON based on incoming metrics types;
// i.a. namespace is formatted as a single string
func formatMetricTypes(mts []plugin.Metric) []MetricToPublish {
	var metrics []MetricToPublish
	for _, mt := range mts {
		metrics = append(metrics, MetricToPublish{
			Timestamp: mt.Timestamp,
			Namespace: mt.Namespace.String(),
			Data:      mt.Data,
			Unit:      mt.Unit,
			Tags:      mt.Tags,
			Version:   mt.Version,
		})
	}
	return metrics
}
<commit_msg>Update error handling<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2016 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package file

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
)

const (
	Name    = "file"
	Version = 3
)

type filePublisher struct {
}

type MetricToPublish struct {
	// The timestamp from when the metric was created.
	Timestamp time.Time         `json:"timestamp"`
	Namespace string            `json:"namespace"`
	Data      interface{}       `json:"data"`
	Unit      string            `json:"unit"`
	Tags      map[string]string `json:"tags"`
	Version   int64             `json:"version"`
}

//New returns an instance of filePublisher
func New() *filePublisher {
	return &filePublisher{}
}

func (f *filePublisher) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	logger := log.New()
	logger.Debug("Publishing started")

	destination, err := cfg.GetString("file")
	if err != nil {
		return fmt.Errorf("%s: %s", err, "file")
	}

	logger.Infof("Publishing %v metrics to %s", len(mts), destination)
	file, err := os.OpenFile(destination, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("Error opening file: %v", err)
	}
	w := bufio.NewWriter(file)

	// format metrics types to metrics to be published
	metrics := formatMetricTypes(mts)
	jsonOut, err := json.Marshal(metrics)
	if err != nil {
		return fmt.Errorf("Error while marshalling metrics to JSON: %v", err)
	}

	w.Write(jsonOut)
	w.WriteString("\n")
	w.Flush()

	return nil
}

func (f *filePublisher) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()
	policy.AddNewStringRule([]string{""}, "file", true)
	return *policy, nil
}

// formatMetricTypes returns metrics in format to be publish as a JSON based on incoming metrics types;
// i.a. namespace is formatted as a single string
func formatMetricTypes(mts []plugin.Metric) []MetricToPublish {
	var metrics []MetricToPublish
	for _, mt := range mts {
		metrics = append(metrics, MetricToPublish{
			Timestamp: mt.Timestamp,
			Namespace: mt.Namespace.String(),
			Data:      mt.Data,
			Unit:      mt.Unit,
			Tags:      mt.Tags,
			Version:   mt.Version,
		})
	}
	return metrics
}
<|endoftext|>


<commit_before>package b2

import (
	"encoding/json"
	"fmt"
	"testing"
	"time"
)

func Test_Bucket_ListFileNames_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionUpload, ActionUpload}
	setupFiles := ""
	for i := range fileAction {
		setupFiles += makeTestFileJson(i, fileAction[i])
		if i != len(fileAction)-1 {
			setupFiles += ","
		}
	}
	s := setupRequest(200, fmt.Sprintf(`{"files":[%s],"nextFileName":"name%d"}`, setupFiles, len(fileAction)))
	defer s.Close()

	response, err := bucket.ListFileNames("", 3)
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if len(response.Files) != 3 {
		t.Fatalf("Expected two files, instead got %d", len(response.Files))
	}
	if response.NextFileName != fmt.Sprintf("name%d", len(fileAction)) {
		t.Errorf("Expected next file name to be name%d, instead got %s", len(fileAction), response.NextFileName)
	}
	if response.NextFileID != "" {
		t.Errorf("Expected no next file id, instead got %s", response.NextFileID)
	}
	for i, file := range response.Files {
		if file.Action != ActionUpload {
			t.Errorf("Expected action to be upload, instead got %v", file.Action)
		}
		if file.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fmt.Sprintf("id%d", i))
		}
		if file.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fmt.Sprintf("name%d", i))
		}
		if file.Size != int64(10+i) {
			t.Errorf("Expected size to be %d, instead got %d", 10+i, file.Size)
		}
		if file.UploadTimestamp != int64(100+i) {
			t.Errorf("Expected upload timestamp to be %d, instead got %d", 10+i, file.UploadTimestamp)
		}
		if file.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", file.Bucket)
		}
	}
}

func Test_Bucket_ListFileNames_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.ListFileNames("", 0)
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}
}

func Test_Bucket_ListFileVersions_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionHide, ActionStart}
	setupFiles := ""
	for i := range fileAction {
		setupFiles += makeTestFileJson(i, fileAction[i])
		if i != len(fileAction)-1 {
			setupFiles += ","
		}
	}
	s := setupRequest(200, fmt.Sprintf(`{"files":[%s],"nextFileId":"id%d","nextFileName":"name%d"}`,
		setupFiles, len(fileAction), len(fileAction)))
	defer s.Close()

	response, err := bucket.ListFileVersions("", "", 3)
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if len(response.Files) != 3 {
		t.Fatalf("Expected three files, instead got %d", len(response.Files))
	}
	if response.NextFileName != "name3" {
		t.Errorf("Expected next file name to be name3, instead got %s", response.NextFileName)
	}
	if response.NextFileID != "id3" {
		t.Errorf("Expected next file id to be id3, instead got %s", response.NextFileID)
	}
	for i, file := range response.Files {
		if file.Action != fileAction[i] {
			t.Errorf("Expected action to be %v, instead got %v", fileAction[i], file.Action)
		}
		if file.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fmt.Sprintf("id%d", i))
		}
		if file.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fmt.Sprintf("name%d", i))
		}
		if file.Size != int64(10+i) {
			t.Errorf("Expected size to be %d, instead got %d", 10+i, file.Size)
		}
		if file.UploadTimestamp != int64(100+i) {
			t.Errorf("Expected upload timestamp to be %d, instead got %d", 10+i, file.UploadTimestamp)
		}
		if file.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", file.Bucket)
		}
	}
}

func Test_Bucket_ListFileVersions_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.ListFileVersions("", "", 0)
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}
}

func Test_Bucket_GetFileInfo_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionHide, ActionStart}

	for i := range fileAction {
		s := setupRequest(200, makeTestFileJson(i, fileAction[i]))

		fileID := fmt.Sprintf("id%d", i)
		file, err := bucket.GetFileInfo(fileID)
		if err != nil {
			t.Fatalf("Expected no error, instead got %s", err)
		}

		if file.Action != fileAction[i] {
			t.Errorf("Expected action to be %v, instead got %v", fileAction[i], file.Action)
		}
		if file.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fmt.Sprintf("id%d", i))
		}
		if file.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fmt.Sprintf("name%d", i))
		}
		if file.ContentLength != int64(10+i) {
			t.Errorf("Expected content length to be %d, instead got %d", 10+i, file.Size)
		}
		if file.ContentSha1 != "sha1" {
			t.Errorf(`Expected content sha1 to be "sha1", instead got %s`, file.ContentSha1)
		}
		if file.ContentType != "text" {
			t.Errorf("Expected content type to be text, instead got %s", file.ContentType)
		}
		if file.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", file.Bucket)
		}
		for k, v := range file.FileInfo {
			t.Errorf("Expected fileInfo to be blank, instead got %s, %s", k, v)
		}

		s.Close()
	}
}

func Test_Bucket_GetFileInfo_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.GetFileInfo(fmt.Sprintf("id%d", i))
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}

	// test no provided ID error
	response, err := bucket.GetFileInfo("")
	if err.Error() != "No fileID provided" {
		t.Errorf(`Expected "No fileID provided", instead got %s`, err)
	}
	if response != nil {
		t.Errorf("Expected response to be empty, instead got %+v", response)
	}
}

func Test_Bucket_GetUploadUrl_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	uploadUrl := "https://eg.backblaze.com/b2api/v1/b2_upload_file?cvt=eg&bucket=id"

	s := setupRequest(200, fmt.Sprintf(`{"bucketId":"id","uploadUrl":"%s","authorizationToken":"token"}`, uploadUrl))
	defer s.Close()

	response, err := bucket.GetUploadUrl()
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if response.Expiration.IsZero() {
		t.Error("Expected time to be now + 24h, instead got zero time")
	}
	if response.AuthorizationToken != "token" {
		t.Errorf(`Expected response token to be "token", instead got %s`, response.AuthorizationToken)
	}
	if response.Url != uploadUrl {
		t.Errorf("Expected response url to be uploadUrl, instead got %s", response.Url)
	}

	if len(bucket.UploadUrls) != 1 {
		t.Fatalf("Expected length of bucket upload urls to be 1, insetad was %d", len(bucket.UploadUrls))
	}
	if bucket.UploadUrls[0] != response {
		t.Error("Expected bucket's uploadUrls to be response, instead was", bucket.UploadUrls[0])
	}
}

func Test_Bucket_GetUploadUrl_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.GetUploadUrl()
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}
		if len(bucket.UploadUrls) != 0 {
			t.Errorf("Expected no upload urls, instead got %d", bucket.UploadUrls)
		}

		s.Close()
	}
}

func Test_Bucket_cleanUploadUrls(t *testing.T) {
	t.Skip()
}

func makeTestFileJson(num int, action Action) string {
	file := FileMeta{
		ID:              fmt.Sprintf("id%d", num),
		Name:            fmt.Sprintf("name%d", num),
		Size:            int64(10 + num),
		ContentLength:   int64(10 + num),
		ContentSha1:     "sha1", // TODO make valid SHA1
		ContentType:     "text",
		Action:          action,
		FileInfo:        map[string]string{},
		UploadTimestamp: int64(100 + num),
	}
	fileJson, _ := json.Marshal(file)
	return string(fileJson)
}
<commit_msg>Fix broken test checks, rename var<commit_after>package b2

import (
	"encoding/json"
	"fmt"
	"testing"
	"time"
)

func Test_Bucket_ListFileNames_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionUpload, ActionUpload}
	setupFiles := ""
	for i := range fileAction {
		setupFiles += makeTestFileJson(i, fileAction[i])
		if i != len(fileAction)-1 {
			setupFiles += ","
		}
	}
	s := setupRequest(200, fmt.Sprintf(`{"files":[%s],"nextFileName":"name%d"}`, setupFiles, len(fileAction)))
	defer s.Close()

	response, err := bucket.ListFileNames("", 3)
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if len(response.Files) != 3 {
		t.Fatalf("Expected two files, instead got %d", len(response.Files))
	}
	if response.NextFileName != fmt.Sprintf("name%d", len(fileAction)) {
		t.Errorf("Expected next file name to be name%d, instead got %s", len(fileAction), response.NextFileName)
	}
	if response.NextFileID != "" {
		t.Errorf("Expected no next file id, instead got %s", response.NextFileID)
	}
	for i, file := range response.Files {
		if file.Action != ActionUpload {
			t.Errorf("Expected action to be upload, instead got %v", file.Action)
		}
		if file.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fmt.Sprintf("id%d", i))
		}
		if file.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fmt.Sprintf("name%d", i))
		}
		if file.Size != int64(10+i) {
			t.Errorf("Expected size to be %d, instead got %d", 10+i, file.Size)
		}
		if file.UploadTimestamp != int64(100+i) {
			t.Errorf("Expected upload timestamp to be %d, instead got %d", 10+i, file.UploadTimestamp)
		}
		if file.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", file.Bucket)
		}
	}
}

func Test_Bucket_ListFileNames_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.ListFileNames("", 0)
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}
}

func Test_Bucket_ListFileVersions_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionHide, ActionStart}
	setupFiles := ""
	for i := range fileAction {
		setupFiles += makeTestFileJson(i, fileAction[i])
		if i != len(fileAction)-1 {
			setupFiles += ","
		}
	}
	s := setupRequest(200, fmt.Sprintf(`{"files":[%s],"nextFileId":"id%d","nextFileName":"name%d"}`,
		setupFiles, len(fileAction), len(fileAction)))
	defer s.Close()

	response, err := bucket.ListFileVersions("", "", 3)
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if len(response.Files) != 3 {
		t.Fatalf("Expected three files, instead got %d", len(response.Files))
	}
	if response.NextFileName != "name3" {
		t.Errorf("Expected next file name to be name3, instead got %s", response.NextFileName)
	}
	if response.NextFileID != "id3" {
		t.Errorf("Expected next file id to be id3, instead got %s", response.NextFileID)
	}
	for i, file := range response.Files {
		if file.Action != fileAction[i] {
			t.Errorf("Expected action to be %v, instead got %v", fileAction[i], file.Action)
		}
		if file.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fmt.Sprintf("id%d", i))
		}
		if file.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fmt.Sprintf("name%d", i))
		}
		if file.Size != int64(10+i) {
			t.Errorf("Expected size to be %d, instead got %d", 10+i, file.Size)
		}
		if file.UploadTimestamp != int64(100+i) {
			t.Errorf("Expected upload timestamp to be %d, instead got %d", 10+i, file.UploadTimestamp)
		}
		if file.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", file.Bucket)
		}
	}
}

func Test_Bucket_ListFileVersions_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.ListFileVersions("", "", 0)
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}
}

func Test_Bucket_GetFileInfo_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	fileAction := []Action{ActionUpload, ActionHide, ActionStart}

	for i := range fileAction {
		s := setupRequest(200, makeTestFileJson(i, fileAction[i]))

		fileID := fmt.Sprintf("id%d", i)
		fileMeta, err := bucket.GetFileInfo(fileID)
		if err != nil {
			t.Fatalf("Expected no error, instead got %s", err)
		}

		if fileMeta.Action != fileAction[i] {
			t.Errorf("Expected action to be %v, instead got %v", fileAction[i], fileMeta.Action)
		}
		if fileMeta.ID != fmt.Sprintf("id%d", i) {
			t.Errorf("Expected file ID to be id%d, instead got %s", i, fileMeta.ID)
		}
		if fileMeta.Name != fmt.Sprintf("name%d", i) {
			t.Errorf("Expected file name to be name%d, instead got %s", i, fileMeta.Name)
		}
		if fileMeta.ContentLength != int64(10+i) {
			t.Errorf("Expected content length to be %d, instead got %d", 10+i, fileMeta.ContentLength)
		}
		if fileMeta.ContentSha1 != "sha1" {
			t.Errorf(`Expected content sha1 to be "sha1", instead got %s`, fileMeta.ContentSha1)
		}
		if fileMeta.ContentType != "text" {
			t.Errorf("Expected content type to be text, instead got %s", fileMeta.ContentType)
		}
		if fileMeta.Bucket != bucket {
			t.Errorf("Expected file bucket to be bucket, instead got %+v", fileMeta.Bucket)
		}
		for k, v := range fileMeta.FileInfo {
			t.Errorf("Expected fileInfo to be blank, instead got %s, %s", k, v)
		}

		s.Close()
	}
}

func Test_Bucket_GetFileInfo_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.GetFileInfo(fmt.Sprintf("id%d", i))
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}

		s.Close()
	}

	// test no provided ID error
	response, err := bucket.GetFileInfo("")
	if err.Error() != "No fileID provided" {
		t.Errorf(`Expected "No fileID provided", instead got %s`, err)
	}
	if response != nil {
		t.Errorf("Expected response to be empty, instead got %+v", response)
	}
}

func Test_Bucket_GetUploadUrl_Success(t *testing.T) {
	b := makeTestB2()
	bucket := makeTestBucket(b)

	uploadUrl := "https://eg.backblaze.com/b2api/v1/b2_upload_file?cvt=eg&bucket=id"

	s := setupRequest(200, fmt.Sprintf(`{"bucketId":"id","uploadUrl":"%s","authorizationToken":"token"}`, uploadUrl))
	defer s.Close()

	response, err := bucket.GetUploadUrl()
	if err != nil {
		t.Fatalf("Expected no error, instead got %s", err)
	}

	if response.Expiration.IsZero() {
		t.Error("Expected time to be now + 24h, instead got zero time")
	}
	if response.AuthorizationToken != "token" {
		t.Errorf(`Expected response token to be "token", instead got %s`, response.AuthorizationToken)
	}
	if response.Url != uploadUrl {
		t.Errorf("Expected response url to be uploadUrl, instead got %s", response.Url)
	}

	if len(bucket.UploadUrls) != 1 {
		t.Fatalf("Expected length of bucket upload urls to be 1, insetad was %d", len(bucket.UploadUrls))
	}
	if bucket.UploadUrls[0] != response {
		t.Error("Expected bucket's uploadUrls to be response, instead was", bucket.UploadUrls[0])
	}
}

func Test_Bucket_GetUploadUrl_Errors(t *testing.T) {
	codes, bodies := errorResponses()
	b := makeTestB2()
	bucket := makeTestBucket(b)

	for i := range codes {
		s := setupRequest(codes[i], bodies[i])

		response, err := bucket.GetUploadUrl()
		testErrorResponse(err, codes[i], t)
		if response != nil {
			t.Errorf("Expected response to be empty, instead got %+v", response)
		}
		if len(bucket.UploadUrls) != 0 {
			t.Errorf("Expected no upload urls, instead got %d", bucket.UploadUrls)
		}

		s.Close()
	}
}

func Test_Bucket_cleanUploadUrls(t *testing.T) {
	t.Skip()
}

func makeTestFileJson(num int, action Action) string {
	file := FileMeta{
		ID:              fmt.Sprintf("id%d", num),
		Name:            fmt.Sprintf("name%d", num),
		Size:            int64(10 + num),
		ContentLength:   int64(10 + num),
		ContentSha1:     "sha1", // TODO make valid SHA1
		ContentType:     "text",
		Action:          action,
		FileInfo:        map[string]string{},
		UploadTimestamp: int64(100 + num),
	}
	fileJson, _ := json.Marshal(file)
	return string(fileJson)
}
<|endoftext|>