<commit_before>package router

import (
	"errors"
	"html"
	"net/http"
	"strconv"
	"strings"
	"time"

	"sort"

	"github.com/NyaaPantsu/nyaa/config"
	"github.com/NyaaPantsu/nyaa/model"
	userService "github.com/NyaaPantsu/nyaa/service/user"
	"github.com/NyaaPantsu/nyaa/util/categories"
	"github.com/NyaaPantsu/nyaa/util/feeds"
	"github.com/NyaaPantsu/nyaa/util/publicSettings"
	"github.com/NyaaPantsu/nyaa/util/search"
	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
)

// RSSHandler : Controller for displaying rss feed, accepting common search arguments
func RSSHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	// We only get the basic variable for rss based on search param
	torrents, createdAsTime, title, err := getTorrentList(r)

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed := &nyaafeeds.RssFeed{
		Title:   title,
		Link:    config.WebAddress() + "/",
		PubDate: createdAsTime.String(),
	}
	feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

	for i, torrent := range torrents {
		torrentJSON := torrent.ToJSON()
		feed.Items[i] = &nyaafeeds.RssItem{
			Title:       torrentJSON.Name,
			Link:        config.WebAddress() + "/download/" + torrentJSON.Hash,
			Description: string(torrentJSON.Description),
			PubDate:     torrent.Date.Format(time.RFC822),
			GUID:        config.WebAddress() + "view" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			Enclosure: &nyaafeeds.RssEnclosure{
				URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
				Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				Type:   "application/x-bittorrent",
			},
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")
	rss, rssErr := feeds.ToXML(feed)
	if rssErr != nil {
		http.Error(w, rssErr.Error(), http.StatusInternalServerError)
	}

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

// RSSEztvHandler : Controller for displaying rss feed, accepting common search arguments
func RSSEztvHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	// We only get the basic variable for rss based on search param
	torrents, createdAsTime, title, err := getTorrentList(r)

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed := &nyaafeeds.RssFeed{
		Title:   title,
		Link:    config.WebAddress() + "/",
		PubDate: createdAsTime.String(),
	}
	feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

	for i, torrent := range torrents {
		torrentJSON := torrent.ToJSON()
		feed.Items[i] = &nyaafeeds.RssItem{
			Title: torrentJSON.Name,
			Link:  config.WebAddress() + "/download/" + torrentJSON.Hash,
			Category: &nyaafeeds.RssCategory{
				Domain: config.WebAddress() + "/search?c=" + torrentJSON.Category + "_" + torrentJSON.SubCategory,
			},
			Description: string(torrentJSON.Description),
			Comments:    config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			PubDate:     torrent.Date.Format(time.RFC822),
			GUID:        config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			Enclosure: &nyaafeeds.RssEnclosure{
				URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
				Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				Type:   "application/x-bittorrent",
			},
			Torrent: &nyaafeeds.RssTorrent{
				Xmlns:         "http://xmlns.ezrss.it/0.1/",
				FileName:      torrentJSON.Name + ".torrent",
				ContentLength: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				InfoHash:      torrentJSON.Hash,
				MagnetURI:     string(torrentJSON.Magnet),
			},
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")
	rss, rssErr := feeds.ToXML(feed)
	if rssErr != nil {
		http.Error(w, rssErr.Error(), http.StatusInternalServerError)
	}

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

// RSSTorznabHandler : Controller for displaying rss feed, accepting common search arguments
func RSSTorznabHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	t := r.URL.Query().Get("t")
	rss := ""
	title := "Nyaa Pantsu"
	if config.IsSukebei() {
		title = "Sukebei Pantsu"
	}
	if t == "caps" {
		T := publicSettings.GetTfuncFromRequest(r)
		cat := categories.GetCategoriesSelect(true, true)
		var categories []*nyaafeeds.RssCategoryTorznab
		categories = append(categories, &nyaafeeds.RssCategoryTorznab{
			ID:          "5070",
			Name:        "Anime",
			Description: "Anime",
		})
		var keys []string
		for name := range cat {
			keys = append(keys, name)
		}
		sort.Strings(keys)
		last := 0
		for _, key := range keys {
			if len(cat[key]) <= 2 {
				categories = append(categories, &nyaafeeds.RssCategoryTorznab{
					ID:   nyaafeeds.ConvertFromCat(cat[key]),
					Name: string(T(key)),
				})
				last++
			} else {
				categories[last].Subcat = append(categories[last].Subcat, &nyaafeeds.RssSubCat{
					ID:   nyaafeeds.ConvertFromCat(cat[key]),
					Name: string(T(key)),
				})
			}
		}
		feed := &nyaafeeds.RssCaps{
			Server: &nyaafeeds.RssServer{
				Version:   "1.0",
				Title:     title,
				Strapline: "...",
				Email:     config.Conf.Email.From,
				URL:       config.WebAddress(),
				Image:     config.WebAddress() + "/img/logo.png",
			},
			Limits: &nyaafeeds.RssLimits{
				Max:     "300",
				Default: "50",
			},
			Registration: &nyaafeeds.RssRegistration{
				Available: "yes",
				Open:      "yes",
			},
			Searching: &nyaafeeds.RssSearching{
				Search: &nyaafeeds.RssSearch{
					Available:       "yes",
					SupportedParams: "q",
				},
				TvSearch: &nyaafeeds.RssSearch{
					Available: "no",
				},
				MovieSearch: &nyaafeeds.RssSearch{
					Available: "no",
				},
			},
			Categories: &nyaafeeds.RssCategories{
				Category: categories,
			},
		}
		var rssErr error
		rss, rssErr = feeds.ToXML(feed)
		if rssErr != nil {
			http.Error(w, rssErr.Error(), http.StatusInternalServerError)
		}
	} else {
		// We only get the basic variable for rss based on search param
		torrents, createdAsTime, title, err := getTorrentList(r)

		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		feed := &nyaafeeds.RssFeed{
			Title:   title,
			Xmlns:   "http://torznab.com/schemas/2015/feed",
			Link:    config.WebAddress() + "/",
			PubDate: createdAsTime.String(),
		}
		feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

		for i, torrent := range torrents {

			torrentJSON := torrent.ToJSON()
			filesNumber := ""
			if len(torrentJSON.FileList) > 0 {
				filesNumber = strconv.Itoa(len(torrentJSON.FileList))
			}
			seeders := ""
			if torrentJSON.Seeders > 0 {
				seeders = strconv.Itoa(int(torrentJSON.Seeders))
			}
			leechers := ""
			if torrentJSON.Leechers > 0 {
				leechers = strconv.Itoa(int(torrentJSON.Leechers))
			}
			feed.Items[i] = &nyaafeeds.RssItem{
				Title: torrentJSON.Name,
				Link:  config.WebAddress() + "/download/" + torrentJSON.Hash,
				Category: &nyaafeeds.RssCategory{
					Domain: config.WebAddress() + "/search?c=" + torrentJSON.Category + "_" + torrentJSON.SubCategory,
				},
				Description: string(torrentJSON.Description),
				Comments:    config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
				PubDate:     torrent.Date.Format(time.RFC822),
				GUID:        config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
				Enclosure: &nyaafeeds.RssEnclosure{
					URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
					Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
					Type:   "application/x-bittorrent",
				},
				Torznab: []*nyaafeeds.RssTorznab{
					&nyaafeeds.RssTorznab{
						Name:  "size",
						Value: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
					},
					&nyaafeeds.RssTorznab{
						Name:  "files",
						Value: filesNumber,
					},
					&nyaafeeds.RssTorznab{
						Name:  "grabs",
						Value: strconv.Itoa(torrentJSON.Downloads),
					},
					&nyaafeeds.RssTorznab{
						Name:  "seeders",
						Value: seeders,
					},
					&nyaafeeds.RssTorznab{
						Name:  "leechers",
						Value: leechers,
					},
					&nyaafeeds.RssTorznab{
						Name:  "infohash",
						Value: torrentJSON.Hash,
					},
					&nyaafeeds.RssTorznab{
						Name:  "magneturl",
						Value: string(torrentJSON.Magnet),
					},
				},
			}
		}
		var rssErr error
		rss, rssErr = feeds.ToXML(feed)
		if rssErr != nil {
			http.Error(w, rssErr.Error(), http.StatusInternalServerError)
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

func getTorrentList(r *http.Request) (torrents []model.Torrent, createdAsTime time.Time, title string, err error) {
	vars := mux.Vars(r)
	page := vars["page"]
	userID := vars["id"]
	cat := r.URL.Query().Get("cat")
	offset := 0
	if r.URL.Query().Get("offset") != "" {
		offset, err = strconv.Atoi(html.EscapeString(r.URL.Query().Get("offset")))
		if err != nil {
			return
		}
	}

	createdAsTime = time.Now()

	if len(torrents) > 0 {
		createdAsTime = torrents[0].Date
	}

	title = "Nyaa Pantsu"
	if config.IsSukebei() {
		title = "Sukebei Pantsu"
	}

	pagenum := 1
	if page == "" && offset > 0 { // first page for offset is 0
		pagenum = offset + 1
	} else if page != "" {
		pagenum, err = strconv.Atoi(html.EscapeString(page))
		if err != nil {
			return
		}
	}
	if pagenum <= 0 {
		err = errors.New("Page number is invalid")
		return
	}

	if userID != "" {
		userIDnum := 0
		userIDnum, err = strconv.Atoi(html.EscapeString(userID))
		// Should we have a feed for anonymous uploads?
		if err != nil || userIDnum == 0 {
			return
		}

		_, _, err = userService.RetrieveUserForAdmin(userID)
		if err != nil {
			return
		}
		// Set the user ID on the request, so that SearchByQuery finds it.
		query := r.URL.Query()
		query.Set("userID", userID)
		r.URL.RawQuery = query.Encode()
	}

	if cat != "" {
		query := r.URL.Query()
		c := nyaafeeds.ConvertToCat(cat)
		if c == "" {
			return
		}
		query.Set("c", c)
		r.URL.RawQuery = query.Encode()
	}

	_, torrents, err = search.SearchByQueryNoCount(r, pagenum)

	return
}
<commit_msg>Fixing Torznab bug<commit_after>package router

import (
	"errors"
	"html"
	"net/http"
	"strconv"
	"strings"
	"time"

	"sort"

	"github.com/NyaaPantsu/nyaa/config"
	"github.com/NyaaPantsu/nyaa/model"
	userService "github.com/NyaaPantsu/nyaa/service/user"
	"github.com/NyaaPantsu/nyaa/util/categories"
	"github.com/NyaaPantsu/nyaa/util/feeds"
	"github.com/NyaaPantsu/nyaa/util/publicSettings"
	"github.com/NyaaPantsu/nyaa/util/search"
	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
)

// RSSHandler : Controller for displaying rss feed, accepting common search arguments
func RSSHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	// We only get the basic variable for rss based on search param
	torrents, createdAsTime, title, err := getTorrentList(r)

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed := &nyaafeeds.RssFeed{
		Title:   title,
		Link:    config.WebAddress() + "/",
		PubDate: createdAsTime.String(),
	}
	feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

	for i, torrent := range torrents {
		torrentJSON := torrent.ToJSON()
		feed.Items[i] = &nyaafeeds.RssItem{
			Title:       torrentJSON.Name,
			Link:        config.WebAddress() + "/download/" + torrentJSON.Hash,
			Description: string(torrentJSON.Description),
			PubDate:     torrent.Date.Format(time.RFC822),
			GUID:        config.WebAddress() + "view" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			Enclosure: &nyaafeeds.RssEnclosure{
				URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
				Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				Type:   "application/x-bittorrent",
			},
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")
	rss, rssErr := feeds.ToXML(feed)
	if rssErr != nil {
		http.Error(w, rssErr.Error(), http.StatusInternalServerError)
	}

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

// RSSEztvHandler : Controller for displaying rss feed, accepting common search arguments
func RSSEztvHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	// We only get the basic variable for rss based on search param
	torrents, createdAsTime, title, err := getTorrentList(r)

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed := &nyaafeeds.RssFeed{
		Title:   title,
		Link:    config.WebAddress() + "/",
		PubDate: createdAsTime.String(),
	}
	feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

	for i, torrent := range torrents {
		torrentJSON := torrent.ToJSON()
		feed.Items[i] = &nyaafeeds.RssItem{
			Title: torrentJSON.Name,
			Link:  config.WebAddress() + "/download/" + torrentJSON.Hash,
			Category: &nyaafeeds.RssCategory{
				Domain: config.WebAddress() + "/search?c=" + torrentJSON.Category + "_" + torrentJSON.SubCategory,
			},
			Description: string(torrentJSON.Description),
			Comments:    config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			PubDate:     torrent.Date.Format(time.RFC822),
			GUID:        config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
			Enclosure: &nyaafeeds.RssEnclosure{
				URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
				Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				Type:   "application/x-bittorrent",
			},
			Torrent: &nyaafeeds.RssTorrent{
				Xmlns:         "http://xmlns.ezrss.it/0.1/",
				FileName:      torrentJSON.Name + ".torrent",
				ContentLength: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				InfoHash:      torrentJSON.Hash,
				MagnetURI:     string(torrentJSON.Magnet),
			},
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")
	rss, rssErr := feeds.ToXML(feed)
	if rssErr != nil {
		http.Error(w, rssErr.Error(), http.StatusInternalServerError)
	}

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

// RSSTorznabHandler : Controller for displaying rss feed, accepting common search arguments
func RSSTorznabHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	t := r.URL.Query().Get("t")
	rss := ""
	title := "Nyaa Pantsu"
	if config.IsSukebei() {
		title = "Sukebei Pantsu"
	}
	if t == "caps" {
		T := publicSettings.GetTfuncFromRequest(r)
		cat := categories.GetCategoriesSelect(true, true)
		var categories []*nyaafeeds.RssCategoryTorznab
		categories = append(categories, &nyaafeeds.RssCategoryTorznab{
			ID:          "5070",
			Name:        "Anime",
			Description: "Anime",
		})
		var keys []string
		for name := range cat {
			keys = append(keys, name)
		}
		sort.Strings(keys)
		last := 0
		for _, key := range keys {
			if len(cat[key]) <= 2 {
				categories = append(categories, &nyaafeeds.RssCategoryTorznab{
					ID:   nyaafeeds.ConvertFromCat(cat[key]),
					Name: string(T(key)),
				})
				last++
			} else {
				categories[last].Subcat = append(categories[last].Subcat, &nyaafeeds.RssSubCat{
					ID:   nyaafeeds.ConvertFromCat(cat[key]),
					Name: string(T(key)),
				})
			}
		}
		feed := &nyaafeeds.RssCaps{
			Server: &nyaafeeds.RssServer{
				Version:   "1.0",
				Title:     title,
				Strapline: "...",
				Email:     config.Conf.Email.From,
				URL:       config.WebAddress(),
				Image:     config.WebAddress() + "/img/logo.png",
			},
			Limits: &nyaafeeds.RssLimits{
				Max:     "300",
				Default: "50",
			},
			Registration: &nyaafeeds.RssRegistration{
				Available: "yes",
				Open:      "yes",
			},
			Searching: &nyaafeeds.RssSearching{
				Search: &nyaafeeds.RssSearch{
					Available:       "yes",
					SupportedParams: "q",
				},
				TvSearch: &nyaafeeds.RssSearch{
					Available: "no",
				},
				MovieSearch: &nyaafeeds.RssSearch{
					Available: "no",
				},
			},
			Categories: &nyaafeeds.RssCategories{
				Category: categories,
			},
		}
		var rssErr error
		rss, rssErr = feeds.ToXML(feed)
		if rssErr != nil {
			http.Error(w, rssErr.Error(), http.StatusInternalServerError)
		}
	} else {
		// We only get the basic variable for rss based on search param
		torrents, createdAsTime, title, err := getTorrentList(r)

		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		feed := &nyaafeeds.RssFeed{
			Title:   title,
			Xmlns:   "http://torznab.com/schemas/2015/feed",
			Link:    config.WebAddress() + "/",
			PubDate: createdAsTime.String(),
		}
		feed.Items = make([]*nyaafeeds.RssItem, len(torrents))

		for i, torrent := range torrents {

			torrentJSON := torrent.ToJSON()
			filesNumber := ""
			if len(torrentJSON.FileList) > 0 {
				filesNumber = strconv.Itoa(len(torrentJSON.FileList))
			}
			seeders := ""
			if torrentJSON.Seeders > 0 {
				seeders = strconv.Itoa(int(torrentJSON.Seeders))
			}
			leechers := ""
			if torrentJSON.Leechers > 0 {
				leechers = strconv.Itoa(int(torrentJSON.Leechers))
			}
			feed.Items[i] = &nyaafeeds.RssItem{
				Title: torrentJSON.Name,
				Link:  config.WebAddress() + "/download/" + torrentJSON.Hash,
				Category: &nyaafeeds.RssCategory{
					Domain: config.WebAddress() + "/search?c=" + torrentJSON.Category + "_" + torrentJSON.SubCategory,
				},
				Description: string(torrentJSON.Description),
				Comments:    config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
				PubDate:     torrent.Date.Format(time.RFC822),
				GUID:        config.WebAddress() + "/view/" + strconv.FormatUint(uint64(torrentJSON.ID), 10),
				Enclosure: &nyaafeeds.RssEnclosure{
					URL:    config.WebAddress() + "/download/" + strings.TrimSpace(torrentJSON.Hash),
					Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
					Type:   "application/x-bittorrent",
				},
			}
			torznab := []*nyaafeeds.RssTorznab{}
			if torrentJSON.Filesize > 0 {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "size",
					Value: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
				})
			}
			if filesNumber != "" {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "files",
					Value: filesNumber,
				})
			}
			torznab = append(torznab, &nyaafeeds.RssTorznab{
				Name:  "grabs",
				Value: strconv.Itoa(torrentJSON.Downloads),
			})
			if seeders != "" {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "seeders",
					Value: seeders,
				})
			}
			if leechers != "" {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "leechers",
					Value: leechers,
				})
			}
			if torrentJSON.Hash != "" {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "infohash",
					Value: torrentJSON.Hash,
				})
			}
			if torrentJSON.Magnet != "" {
				torznab = append(torznab, &nyaafeeds.RssTorznab{
					Name:  "magneturl",
					Value: string(torrentJSON.Magnet),
				})
			}
			if len(torznab) > 0 {
				feed.Items[i].Torznab = torznab
			}
		}
		var rssErr error
		rss, rssErr = feeds.ToXML(feed)
		if rssErr != nil {
			http.Error(w, rssErr.Error(), http.StatusInternalServerError)
		}
	}
	// allow cross domain AJAX requests
	w.Header().Set("Access-Control-Allow-Origin", "*")

	_, writeErr := w.Write([]byte(rss))
	if writeErr != nil {
		http.Error(w, writeErr.Error(), http.StatusInternalServerError)
	}
}

func getTorrentList(r *http.Request) (torrents []model.Torrent, createdAsTime time.Time, title string, err error) {
	vars := mux.Vars(r)
	page := vars["page"]
	userID := vars["id"]
	cat := r.URL.Query().Get("cat")
	offset := 0
	if r.URL.Query().Get("offset") != "" {
		offset, err = strconv.Atoi(html.EscapeString(r.URL.Query().Get("offset")))
		if err != nil {
			return
		}
	}

	createdAsTime = time.Now()

	if len(torrents) > 0 {
		createdAsTime = torrents[0].Date
	}

	title = "Nyaa Pantsu"
	if config.IsSukebei() {
		title = "Sukebei Pantsu"
	}

	pagenum := 1
	if page == "" && offset > 0 { // first page for offset is 0
		pagenum = offset + 1
	} else if page != "" {
		pagenum, err = strconv.Atoi(html.EscapeString(page))
		if err != nil {
			return
		}
	}
	if pagenum <= 0 {
		err = errors.New("Page number is invalid")
		return
	}

	if userID != "" {
		userIDnum := 0
		userIDnum, err = strconv.Atoi(html.EscapeString(userID))
		// Should we have a feed for anonymous uploads?
		if err != nil || userIDnum == 0 {
			return
		}

		_, _, err = userService.RetrieveUserForAdmin(userID)
		if err != nil {
			return
		}
		// Set the user ID on the request, so that SearchByQuery finds it.
		query := r.URL.Query()
		query.Set("userID", userID)
		r.URL.RawQuery = query.Encode()
	}

	if cat != "" {
		query := r.URL.Query()
		c := nyaafeeds.ConvertToCat(cat)
		if c == "" {
			return
		}
		query.Set("c", c)
		r.URL.RawQuery = query.Encode()
	}

	_, torrents, err = search.SearchByQueryNoCount(r, pagenum)

	return
}
<|endoftext|>


<commit_before>package app

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"os/exec"
	"syscall"
	"unsafe"

	"github.com/gorilla/websocket"
	"github.com/yudai/utf8reader"
)

type clientContext struct {
	app        *App
	request    *http.Request
	connection *websocket.Conn
	command    *exec.Cmd
	pty        *os.File
}

const (
	Input          = '0'
	ResizeTerminal = '1'
)

type argResizeTerminal struct {
	Columns float64
	Rows    float64
}

func (context *clientContext) goHandleClient() {
	exit := make(chan bool, 2)

	go func() {
		defer func() { exit <- true }()

		context.processSend()
	}()

	go func() {
		defer func() { exit <- true }()

		context.processReceive()
	}()

	go func() {
		<-exit
		context.command.Wait()
		context.connection.Close()
		log.Printf("Connection closed: %s", context.request.RemoteAddr)
	}()
}

func (context *clientContext) processSend() {
	buf := make([]byte, 1024)
	utf8f := utf8reader.New(context.pty)

	for {
		size, err := utf8f.Read(buf)
		if err != nil {
			log.Printf("Command exited for: %s", context.request.RemoteAddr)
			return
		}

		writer, err := context.connection.NextWriter(websocket.TextMessage)
		if err != nil {
			return
		}

		writer.Write(buf[:size])
		writer.Close()
	}
}

func (context *clientContext) processReceive() {
	for {
		_, data, err := context.connection.ReadMessage()
		if err != nil {
			return
		}

		switch data[0] {
		case Input:
			if !context.app.options.PermitWrite {
				break
			}

			_, err := context.pty.Write(data[1:])
			if err != nil {
				return
			}

		case ResizeTerminal:
			var args argResizeTerminal
			err = json.Unmarshal(data[1:], &args)
			if err != nil {
				log.Print("Malformed remote command")
				return
			}

			window := struct {
				row uint16
				col uint16
				x   uint16
				y   uint16
			}{
				uint16(args.Rows),
				uint16(args.Columns),
				0,
				0,
			}
			syscall.Syscall(
				syscall.SYS_IOCTL,
				context.pty.Fd(),
				syscall.TIOCSWINSZ,
				uintptr(unsafe.Pointer(&window)),
			)

		default:
			log.Print("Unknown message type")
			return
		}
	}
}
<commit_msg>Close pty<commit_after>package app

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"os/exec"
	"syscall"
	"unsafe"

	"github.com/gorilla/websocket"
	"github.com/yudai/utf8reader"
)

type clientContext struct {
	app        *App
	request    *http.Request
	connection *websocket.Conn
	command    *exec.Cmd
	pty        *os.File
}

const (
	Input          = '0'
	ResizeTerminal = '1'
)

type argResizeTerminal struct {
	Columns float64
	Rows    float64
}

func (context *clientContext) goHandleClient() {
	exit := make(chan bool, 2)

	go func() {
		defer func() { exit <- true }()

		context.processSend()
	}()

	go func() {
		defer func() { exit <- true }()

		context.processReceive()
	}()

	go func() {
		<-exit
		context.pty.Close()
		context.command.Wait()
		context.connection.Close()
		log.Printf("Connection closed: %s", context.request.RemoteAddr)
	}()
}

func (context *clientContext) processSend() {
	buf := make([]byte, 1024)
	utf8f := utf8reader.New(context.pty)

	for {
		size, err := utf8f.Read(buf)
		if err != nil {
			log.Printf("Command exited for: %s", context.request.RemoteAddr)
			return
		}

		writer, err := context.connection.NextWriter(websocket.TextMessage)
		if err != nil {
			return
		}

		writer.Write(buf[:size])
		writer.Close()
	}
}

func (context *clientContext) processReceive() {
	for {
		_, data, err := context.connection.ReadMessage()
		if err != nil {
			return
		}

		switch data[0] {
		case Input:
			if !context.app.options.PermitWrite {
				break
			}

			_, err := context.pty.Write(data[1:])
			if err != nil {
				return
			}

		case ResizeTerminal:
			var args argResizeTerminal
			err = json.Unmarshal(data[1:], &args)
			if err != nil {
				log.Print("Malformed remote command")
				return
			}

			window := struct {
				row uint16
				col uint16
				x   uint16
				y   uint16
			}{
				uint16(args.Rows),
				uint16(args.Columns),
				0,
				0,
			}
			syscall.Syscall(
				syscall.SYS_IOCTL,
				context.pty.Fd(),
				syscall.TIOCSWINSZ,
				uintptr(unsafe.Pointer(&window)),
			)

		default:
			log.Print("Unknown message type")
			return
		}
	}
}
<|endoftext|>


<commit_before>package common

import (
    "encoding"
    "time"

    "github.com/swiftkick-io/xbinary"
)

// Snapshot captures a specific state of the log. It consists of the time the snapshot was taken, the number of items in the log, and a XXH64 hash of all the log entries.
type Snapshot interface {
    Time() time.Time
    Size() int64
    Hash() uint64
    encoding.BinaryMarshaler
}

// BasicSnapshot represents the simplest snapshot which fulfills the Snapshot interface. The timestamp is stored as nanoseconds since epoch. Both size and hash are stored as 64-bit integers.
type BasicSnapshot struct {
    nanos int64
    size  int64
    hash  uint64
}

// Time converts the nanoseconds since epoch into a `time.Time` instance.
func (b BasicSnapshot) Time() time.Time {
    return time.Unix(0, b.nanos)
}

// Size returns the number of records in the log at the time the snapshot was taken.
func (b BasicSnapshot) Size() int64 {
    return b.size
}

// Hash returns the XXH64 hash of the log file.
func (b BasicSnapshot) Hash() uint64 {
    return b.hash
}

// MarshalBinary converts the snapshot into a byte array.
// The byte array is formatted like so:
//
// ```
// 8-byte int64  time
// 8-byte uint64 size
// 8-byte uint64 hash
//
// 0        8        16       24
// +--------+--------+--------+
// |  time  |  size  |  hash  |
// +--------+--------+--------+
// ```
func (b BasicSnapshot) MarshalBinary() ([]byte, error) {
    buffer := make([]byte, 24)
    xbinary.LittleEndian.PutInt64(buffer, 0, b.nanos)
    xbinary.LittleEndian.PutInt64(buffer, 8, b.size)
    xbinary.LittleEndian.PutUint64(buffer, 16, b.hash)
    return buffer, nil
}

// #### **UnmarshalShapshot**

// UnmarshalSnapshot is a utility function which converts a byte array into a snapshot. If the byte array is too small, a `ErrInvalidSnapshot` is returned.
func UnmarshalShapshot(data []byte) (Snapshot, error) {
    if len(data) != 24 {
        return nil, ErrInvalidSnapshot
    }

    var snapshot BasicSnapshot
    nanos, _ := xbinary.LittleEndian.Int64(data, 0)
    snapshot.nanos = nanos

    size, _ := xbinary.LittleEndian.Int64(data, 8)
    snapshot.size = size

    hash, _ := xbinary.LittleEndian.Uint64(data, 16)
    snapshot.hash = hash

    return snapshot, nil
}
<commit_msg>Add NewSnapshot method<commit_after>package common

import (
    "encoding"
    "time"

    "github.com/swiftkick-io/xbinary"
)

// Snapshot captures a specific state of the log. It consists of the time the snapshot was taken, the number of items in the log, and a XXH64 hash of all the log entries.
type Snapshot interface {
    Time() time.Time
    Size() int64
    Hash() uint64
    encoding.BinaryMarshaler
}

func NewSnapshot(nanos, size int64, hash uint64) Snapshot {
    return BasicSnapshot{nanos, size, hash}
}

// BasicSnapshot represents the simplest snapshot which fulfills the Snapshot interface. The timestamp is stored as nanoseconds since epoch. Both size and hash are stored as 64-bit integers.
type BasicSnapshot struct {
    nanos int64
    size  int64
    hash  uint64
}

// Time converts the nanoseconds since epoch into a `time.Time` instance.
func (b BasicSnapshot) Time() time.Time {
    return time.Unix(0, b.nanos)
}

// Size returns the number of records in the log at the time the snapshot was taken.
func (b BasicSnapshot) Size() int64 {
    return b.size
}

// Hash returns the XXH64 hash of the log file.
func (b BasicSnapshot) Hash() uint64 {
    return b.hash
}

// MarshalBinary converts the snapshot into a byte array.
// The byte array is formatted like so:
//
// ```
// 8-byte int64  time
// 8-byte uint64 size
// 8-byte uint64 hash
//
// 0        8        16       24
// +--------+--------+--------+
// |  time  |  size  |  hash  |
// +--------+--------+--------+
// ```
func (b BasicSnapshot) MarshalBinary() ([]byte, error) {
    buffer := make([]byte, 24)
    xbinary.LittleEndian.PutInt64(buffer, 0, b.nanos)
    xbinary.LittleEndian.PutInt64(buffer, 8, b.size)
    xbinary.LittleEndian.PutUint64(buffer, 16, b.hash)
    return buffer, nil
}

// #### **UnmarshalShapshot**

// UnmarshalSnapshot is a utility function which converts a byte array into a snapshot. If the byte array is too small, a `ErrInvalidSnapshot` is returned.
func UnmarshalShapshot(data []byte) (Snapshot, error) {
    if len(data) != 24 {
        return nil, ErrInvalidSnapshot
    }

    var snapshot BasicSnapshot
    nanos, _ := xbinary.LittleEndian.Int64(data, 0)
    snapshot.nanos = nanos

    size, _ := xbinary.LittleEndian.Int64(data, 8)
    snapshot.size = size

    hash, _ := xbinary.LittleEndian.Uint64(data, 16)
    snapshot.hash = hash

    return snapshot, nil
}
<|endoftext|>


<commit_before>package lrserver_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"testing"
	"time"

	"github.com/gorilla/websocket"
	"github.com/jaschaephraim/lrserver"
	. "github.com/smartystreets/goconvey/convey"
)

const localhost string = "://127.0.0.1"

var clientHello = struct {
	Command   string   `json:"command"`
	Protocols []string `json:"protocols"`
}{
	"hello",
	[]string{
		"http://livereload.com/protocols/official-7",
		"http://livereload.com/protocols/official-8",
		"http://livereload.com/protocols/2.x-origin-version-negotiation",
	},
}

var randomMessage = struct {
	Command string `json:"command"`
}{
	"random",
}

type serverHello struct {
	Command    string   `json:"command"`
	Protocols  []string `json:"protocols"`
	ServerName string   `json:"serverName"`
}

type serverReload struct {
	Command string `json:"command"`
	Path    string `json:"path"`
	LiveCSS bool   `json:"liveCSS"`
}

type serverAlert struct {
	Command string `json:"command"`
	Message string `json:"message"`
}

func Test(t *testing.T) {
	Convey("Given a new server", t, func() {
		srv, err := lrserver.New(lrserver.DefaultName, 0)
		if err != nil {
			t.Fatal(err)
		}

		Convey("StatusLog() and ErrorLog() should return loggers", func() {
			logger := log.New(nil, "", 0)
			So(srv.StatusLog(), ShouldHaveSameTypeAs, logger)
			So(srv.ErrorLog(), ShouldHaveSameTypeAs, logger)
		})

		srv.SetStatusLog(nil)
		srv.SetErrorLog(nil)

		// Start server
		Convey("that is running", func() {
			go srv.ListenAndServe()

			time.Sleep(time.Nanosecond)

			Convey("a dynamically assigned port should be updated", func() {
				So(srv.Port(), ShouldNotEqual, 0)
			})

			// Test JS
			Convey("JS should be served successfully", func() {
				client := new(http.Client)
				resp, err := client.Get(
					fmt.Sprintf("http%s:%d/livereload.js", localhost, srv.Port()),
				)
				if err != nil {
					t.Fatal(err)
				}
				defer resp.Body.Close()

				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					t.Fatal(err)
				}

				bodyString := string(body)
				So(bodyString, ShouldStartWith, "(function e(t,n,r)")
				So(bodyString, ShouldEndWith, "},{}]},{},[8]);")
			})

			// Connect WebSocket
			Convey("and a connected websocket", func() {
				dialer := new(websocket.Dialer)
				conn, resp, err := dialer.Dial(
					fmt.Sprintf("ws%s:%d/livereload", localhost, srv.Port()),
					http.Header{},
				)
				if err != nil {
					t.Fatal(err)
				}

				So(resp.StatusCode, ShouldEqual, 101)

				// Receive hello
				hello := new(serverHello)
				err = conn.ReadJSON(hello)
				if err != nil {
					t.Fatal(err)
				}

				So(*hello, ShouldResemble, serverHello{
					"hello",
					[]string{
						"http://livereload.com/protocols/official-7",
						"http://livereload.com/protocols/official-8",
						"http://livereload.com/protocols/official-9",
						"http://livereload.com/protocols/2.x-origin-version-negotiation",
						"http://livereload.com/protocols/2.x-remote-control",
					},
					srv.Name(),
				})

				// Test bad handshake
				Convey("an invalid handshake should close the connection", func() {
					err = conn.WriteJSON(randomMessage)
					if err != nil {
						t.Fatal(err)
					}

					_, _, err := conn.NextReader()
					So(reflect.TypeOf(err).String(), ShouldEqual, "*websocket.closeError")
				})

				// Send valid handshake
				Convey("and a successful handshake", func() {
					err = conn.WriteJSON(clientHello)
					if err != nil {
						t.Fatal(err)
					}

					time.Sleep(time.Millisecond)

					Convey("a valid client message should be tolerated", func() {
						err = conn.WriteJSON(randomMessage)
						if err != nil {
							t.Fatal(err)
						}

						errChan := make(chan error)
						failed := false
						go func() {
							_, _, err := conn.NextReader()
							errChan <- err
						}()
						go func() {
							<-errChan
							failed = true
						}()

						time.Sleep(time.Millisecond)

						So(failed, ShouldBeFalse)
					})

					// Test reload
					Convey("reload should work", func() {
						file := "file"
						srv.Reload(file)

						sr := new(serverReload)
						err = conn.ReadJSON(sr)
						if err != nil {
							t.Fatal(err)
						}

						So(*sr, ShouldResemble, serverReload{
							"reload",
							file,
							srv.LiveCSS(),
						})
					})

					// Test alert
					Convey("alert should work", func() {
						msg := "alert"
						srv.Alert(msg)

						sa := new(serverAlert)
						err = conn.ReadJSON(sa)
						if err != nil {
							t.Fatal(err)
						}

						So(*sa, ShouldResemble, serverAlert{
							"alert",
							msg,
						})
					})
				})
			})
		})
	})
}
<commit_msg>Update tests<commit_after>package lrserver_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"testing"
	"time"

	"github.com/gorilla/websocket"
	"github.com/jaschaephraim/lrserver"
	. "github.com/smartystreets/goconvey/convey"
)

const localhost string = "://127.0.0.1"

var clientHello = struct {
	Command   string   `json:"command"`
	Protocols []string `json:"protocols"`
}{
	"hello",
	[]string{
		"http://livereload.com/protocols/official-7",
		"http://livereload.com/protocols/official-8",
		"http://livereload.com/protocols/2.x-origin-version-negotiation",
	},
}

var randomMessage = struct {
	Command string `json:"command"`
}{
	"random",
}

type serverHello struct {
	Command    string   `json:"command"`
	Protocols  []string `json:"protocols"`
	ServerName string   `json:"serverName"`
}

type serverReload struct {
	Command string `json:"command"`
	Path    string `json:"path"`
	LiveCSS bool   `json:"liveCSS"`
}

type serverAlert struct {
	Command string `json:"command"`
	Message string `json:"message"`
}

func Test(t *testing.T) {
	Convey("Given a new server", t, func() {
		srv, err := lrserver.New(lrserver.DefaultName, 0)
		if err != nil {
			t.Fatal(err)
		}

		Convey("StatusLog() and ErrorLog() should return loggers", func() {
			logger := log.New(nil, "", 0)
			So(srv.StatusLog(), ShouldHaveSameTypeAs, logger)
			So(srv.ErrorLog(), ShouldHaveSameTypeAs, logger)
		})

		srv.SetStatusLog(nil)
		srv.SetErrorLog(nil)

		// Start server
		Convey("that is running", func() {
			go srv.ListenAndServe()

			time.Sleep(time.Millisecond)

			Convey("a dynamically assigned port should be updated", func() {
				So(srv.Port(), ShouldNotEqual, 0)
			})

			// Test JS
			Convey("JS should be served successfully", func() {
				client := new(http.Client)
				resp, err := client.Get(
					fmt.Sprintf("http%s:%d/livereload.js", localhost, srv.Port()),
				)
				if err != nil {
					t.Fatal(err)
				}
				defer resp.Body.Close()

				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					t.Fatal(err)
				}

				bodyString := string(body)
				So(bodyString, ShouldStartWith, "(function e(t,n,r)")
				So(bodyString, ShouldEndWith, "},{}]},{},[8]);")
			})

			// Connect WebSocket
			Convey("and a connected websocket", func() {
				dialer := new(websocket.Dialer)
				conn, resp, err := dialer.Dial(
					fmt.Sprintf("ws%s:%d/livereload", localhost, srv.Port()),
					http.Header{},
				)
				if err != nil {
					t.Fatal(err)
				}

				So(resp.StatusCode, ShouldEqual, 101)

				// Receive hello
				hello := new(serverHello)
				err = conn.ReadJSON(hello)
				if err != nil {
					t.Fatal(err)
				}

				So(*hello, ShouldResemble, serverHello{
					"hello",
					[]string{
						"http://livereload.com/protocols/official-7",
						"http://livereload.com/protocols/official-8",
						"http://livereload.com/protocols/official-9",
						"http://livereload.com/protocols/2.x-origin-version-negotiation",
						"http://livereload.com/protocols/2.x-remote-control",
					},
					srv.Name(),
				})

				// Test bad handshake
				Convey("an invalid handshake should close the connection", func() {
					err = conn.WriteJSON(randomMessage)
					if err != nil {
						t.Fatal(err)
					}

					_, _, err := conn.NextReader()
					So(reflect.TypeOf(err).String(), ShouldEqual, "*websocket.CloseError")
				})

				// Send valid handshake
				Convey("and a successful handshake", func() {
					err = conn.WriteJSON(clientHello)
					if err != nil {
						t.Fatal(err)
					}

					time.Sleep(time.Millisecond)

					Convey("a valid client message should be tolerated", func() {
						err = conn.WriteJSON(randomMessage)
						if err != nil {
							t.Fatal(err)
						}

						errChan := make(chan error)
						failed := false
						go func() {
							_, _, err := conn.NextReader()
							errChan <- err
						}()
						go func() {
							<-errChan
							failed = true
						}()

						time.Sleep(time.Millisecond)

						So(failed, ShouldBeFalse)
					})

					// Test reload
					Convey("reload should work", func() {
						file := "file"
						srv.Reload(file)

						sr := new(serverReload)
						err = conn.ReadJSON(sr)
						if err != nil {
							t.Fatal(err)
						}

						So(*sr, ShouldResemble, serverReload{
							"reload",
							file,
							srv.LiveCSS(),
						})
					})

					// Test alert
					Convey("alert should work", func() {
						msg := "alert"
						srv.Alert(msg)

						sa := new(serverAlert)
						err = conn.ReadJSON(sa)
						if err != nil {
							t.Fatal(err)
						}

						So(*sa, ShouldResemble, serverAlert{
							"alert",
							msg,
						})
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/web/apps"
	"github.com/cozy/cozy-stack/web/auth"
	"github.com/cozy/cozy-stack/web/data"
	"github.com/cozy/cozy-stack/web/files"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/status"
	"github.com/cozy/cozy-stack/web/version"
	"github.com/gin-gonic/gin"
)

// SetupRoutes sets the routing for HTTP endpoints to the Go methods
func SetupRoutes(router *gin.Engine, assetsPath string) {
	router.Use(middlewares.ParseHost())
	router.Use(middlewares.ServeApp(apps.Serve))
	router.Use(middlewares.ErrorHandler())

	// NOTE: CORS middleware is a "global" one because of the way gin works: to
	// handle preflight requests with OPTIONS method, every route would have to
	// have an empty OPTIONS handler in order to not get a 404 and actually
	// entering the middleware.
	router.Use(corsMiddleware("/status", "/apps", "/data", "/files"))

	router.LoadHTMLGlob(assetsPath + "/templates/*.html")
	router.Static("/assets", assetsPath)

	auth.Routes(router)
	apps.Routes(router.Group("/apps", middlewares.NeedInstance()))
	data.Routes(router.Group("/data", middlewares.NeedInstance()))
	files.Routes(router.Group("/files", middlewares.NeedInstance()))
	status.Routes(router.Group("/status"))
	version.Routes(router.Group("/version"))
}

func corsMiddleware(routes ...string) gin.HandlerFunc {
	normalHeaders := make(http.Header)
	normalHeaders.Set("Access-Control-Allow-Credentials", "true")

	preflightHeaders := make(http.Header)
	preflightHeaders.Set("Access-Control-Allow-Credentials", "true")
	preflightHeaders.Set("Access-Control-Allow-Methods", "*")
	preflightHeaders.Set("Access-Control-Max-Age", strconv.FormatInt(int64(12*time.Hour/time.Second), 10))

	return func(c *gin.Context) {
		origin := c.Request.Header.Get("Origin")

		// TODO(oauth): check that the Origin header matches a subdomain of the
		// instance or a registered domain for an OAuth2 app/device.
		if origin == "" {
			return
		}

		path := c.Request.URL.Path
		valid := false
		for _, route := range routes {
			if strings.HasPrefix(path, route) {
				valid = true
				break
			}
		}

		if !valid {
			c.AbortWithStatus(http.StatusForbidden)
			return
		}

		isOptions := c.Request.Method == "OPTIONS"

		var headers http.Header
		if isOptions {
			headers = preflightHeaders
		} else {
			headers = normalHeaders
		}

		header := c.Writer.Header()
		for key, value := range headers {
			header[key] = value
		}

		c.Header("Access-Control-Allow-Origin", origin)

		if isOptions {
			c.Header("Access-Control-Allow-Headers", c.Request.Header.Get("Access-Control-Request-Headers"))
			c.AbortWithStatus(http.StatusNoContent)
		}
	}
}
<commit_msg>CORS: explicit allowed methods instead of wildcard<commit_after>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/web/apps"
	"github.com/cozy/cozy-stack/web/auth"
	"github.com/cozy/cozy-stack/web/data"
	"github.com/cozy/cozy-stack/web/files"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/status"
	"github.com/cozy/cozy-stack/web/version"
	"github.com/gin-gonic/gin"
)

// SetupRoutes sets the routing for HTTP endpoints to the Go methods
func SetupRoutes(router *gin.Engine, assetsPath string) {
	router.Use(middlewares.ParseHost())
	router.Use(middlewares.ServeApp(apps.Serve))
	router.Use(middlewares.ErrorHandler())

	// NOTE: CORS middleware is a "global" one because of the way gin works: to
	// handle preflight requests with OPTIONS method, every route would have to
	// have an empty OPTIONS handler in order to not get a 404 and actually
	// entering the middleware.
	router.Use(corsMiddleware("/status", "/apps", "/data", "/files"))

	router.LoadHTMLGlob(assetsPath + "/templates/*.html")
	router.Static("/assets", assetsPath)

	auth.Routes(router)
	apps.Routes(router.Group("/apps", middlewares.NeedInstance()))
	data.Routes(router.Group("/data", middlewares.NeedInstance()))
	files.Routes(router.Group("/files", middlewares.NeedInstance()))
	status.Routes(router.Group("/status"))
	version.Routes(router.Group("/version"))
}

func corsMiddleware(routes ...string) gin.HandlerFunc {
	normalHeaders := make(http.Header)
	normalHeaders.Set("Access-Control-Allow-Credentials", "true")

	preflightHeaders := make(http.Header)
	preflightHeaders.Set("Access-Control-Allow-Credentials", "true")
	preflightHeaders.Set("Access-Control-Allow-Methods", "GET,HEAD,PUT,PATCH,POST,DELETE")
	preflightHeaders.Set("Access-Control-Max-Age", strconv.FormatInt(int64(12*time.Hour/time.Second), 10))

	return func(c *gin.Context) {
		origin := c.Request.Header.Get("Origin")

		// TODO(oauth): check that the Origin header matches a subdomain of the
		// instance or a registered domain for an OAuth2 app/device.
		if origin == "" {
			return
		}

		path := c.Request.URL.Path
		valid := false
		for _, route := range routes {
			if strings.HasPrefix(path, route) {
				valid = true
				break
			}
		}

		if !valid {
			c.AbortWithStatus(http.StatusForbidden)
			return
		}

		isOptions := c.Request.Method == "OPTIONS"

		var headers http.Header
		if isOptions {
			headers = preflightHeaders
		} else {
			headers = normalHeaders
		}

		header := c.Writer.Header()
		for key, value := range headers {
			header[key] = value
		}

		c.Header("Access-Control-Allow-Origin", origin)

		if isOptions {
			c.Header("Access-Control-Allow-Headers", c.Request.Header.Get("Access-Control-Request-Headers"))
			c.AbortWithStatus(http.StatusNoContent)
		}
	}
}
<|endoftext|>


<commit_before>package web

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/daohoangson/go-sitemirror/cacher"
)

type server struct {
	cacher cacher.Cacher
	logger *logrus.Logger

	onCacheIssue *func(CacheIssue)

	listeners map[string]net.Listener
}

func NewServer(cacher cacher.Cacher, logger *logrus.Logger) Server {
	s := &server{}
	s.init(cacher, logger)
	return s
}

func (s *server) init(httpCacher cacher.Cacher, logger *logrus.Logger) {
	if httpCacher == nil {
		httpCacher = cacher.NewHttpCacher(nil)
	}
	s.cacher = httpCacher

	if logger == nil {
		logger = logrus.New()
	}
	s.logger = logger

	s.listeners = make(map[string]net.Listener)
}

func (s *server) GetCacher() cacher.Cacher {
	return s.cacher
}

func (s *server) SetOnCacheIssue(f func(CacheIssue)) {
	s.onCacheIssue = &f
}

func (s *server) ListenAndServe(host string, port int) (io.Closer, error) {
	if port < 0 {
		return nil, errors.New("Invalid port")
	}

	if existing, existingFound := s.listeners[host]; existingFound {
		return existing, errors.New("Existing listener has been found for this host")
	}

	loggerContext := s.logger.WithFields(logrus.Fields{
		"host": host,
		"port": port,
	})

	listener, listenError := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if listenError != nil {
		loggerContext.WithField("error", listenError).Errorf("Cannot listen")
		return nil, listenError
	}

	if port == 0 {
		loggerContext = loggerContext.WithField("addr", listener.Addr().String())
	}

	start := time.Now()
	go func() {
		var f http.HandlerFunc = func(w http.ResponseWriter, req *http.Request) {
			s.Serve(host, w, req)
		}

		loggerContext.Info("Serving")
		serveError := http.Serve(listener, f)
		if serveError != nil {
			elapsed := time.Since(start)
			errorContext := loggerContext.WithField("error", serveError)
			if elapsed > 100*time.Millisecond {
				// some time has passed, it's likely that it worked
				// but the listener has been asked to be closed
				errorContext.Debug("Listener has been closed")
			} else {
				errorContext.Errorf("Cannot serve")
			}
		}
	}()

	s.listeners[host] = listener
	return listener, nil
}

func (s *server) GetListeningPort(host string) (int, error) {
	listener, ok := s.listeners[host]
	if !ok {
		return 0, errors.New("Listener not found")
	}

	addr := listener.Addr().String()
	matches := regexp.MustCompile(`:(\d+)$`).FindStringSubmatch(addr)
	port, err := strconv.ParseInt(matches[1], 10, 64)

	return int(port), err
}

func (s *server) Serve(host string, w http.ResponseWriter, req *http.Request) {
	url, _ := url.Parse(req.URL.String())
	url.Host = host
	loggerContext := s.logger.WithField("url", url)

	cache, err := s.cacher.Open(url)
	if err != nil {
		loggerContext.Debug("Cache not found")

		infoOnError := &CacheInfo{
			ResponseWriter: w,
			Error:          err,
		}
		if !s.triggerOnCacheIssue(CacheNotFound, url, infoOnError) {
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	defer cache.Close()

	info := ServeHTTPCache(cache, w)
	if info.Error != nil {
		loggerContext.WithField("error", info.Error).Error("Cannot serve")
		s.triggerOnCacheIssue(CacheError, url, info)
		return
	}

	loggerContext.WithField("statusCode", info.StatusCode).Debug("Served")
	if info.Expires != nil && info.Expires.Before(time.Now()) {
		s.triggerOnCacheIssue(CacheExpired, url, info)
	}
}

func (s *server) StopListening(host string) error {
	listener, ok := s.listeners[host]
	if !ok {
		return errors.New("Listener not found")
	}

	err := listener.Close()

	loggerContext := s.logger.WithField("host", host)
	if err == nil {
		loggerContext.Info("Stopped listening")
	} else {
		loggerContext.WithField("error", err).Error("Cannot stop listening")
	}

	return err
}

func (s *server) StopAll() []string {
	hosts := make([]string, 0)

	for host, listener := range s.listeners {
		err := listener.Close()
		loggerContext := s.logger.WithField("host", host)

		if err == nil {
			loggerContext.Info("Stopped listening")
			hosts = append(hosts, host)
		} else {
			loggerContext.WithField("error", err).Error("Cannot stop listening")
		}
	}

	return hosts
}

func (s *server) triggerOnCacheIssue(t cacheIssueType, url *url.URL, info *CacheInfo) bool {
	if s.onCacheIssue == nil {
		return false
	}

	(*s.onCacheIssue)(CacheIssue{
		Type: t,
		URL:  url,
		Info: info,
	})

	return true
}
<commit_msg>Fix bug url generated by server may be incomplete<commit_after>package web

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/daohoangson/go-sitemirror/cacher"
)

type server struct {
	cacher cacher.Cacher
	logger *logrus.Logger

	onCacheIssue *func(CacheIssue)

	listeners map[string]net.Listener
}

func NewServer(cacher cacher.Cacher, logger *logrus.Logger) Server {
	s := &server{}
	s.init(cacher, logger)
	return s
}

func (s *server) init(httpCacher cacher.Cacher, logger *logrus.Logger) {
	if httpCacher == nil {
		httpCacher = cacher.NewHttpCacher(nil)
	}
	s.cacher = httpCacher

	if logger == nil {
		logger = logrus.New()
	}
	s.logger = logger

	s.listeners = make(map[string]net.Listener)
}

func (s *server) GetCacher() cacher.Cacher {
	return s.cacher
}

func (s *server) SetOnCacheIssue(f func(CacheIssue)) {
	s.onCacheIssue = &f
}

func (s *server) ListenAndServe(host string, port int) (io.Closer, error) {
	if port < 0 {
		return nil, errors.New("Invalid port")
	}

	if existing, existingFound := s.listeners[host]; existingFound {
		return existing, errors.New("Existing listener has been found for this host")
	}

	loggerContext := s.logger.WithFields(logrus.Fields{
		"host": host,
		"port": port,
	})

	listener, listenError := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if listenError != nil {
		loggerContext.WithField("error", listenError).Errorf("Cannot listen")
		return nil, listenError
	}

	if port == 0 {
		loggerContext = loggerContext.WithField("addr", listener.Addr().String())
	}

	start := time.Now()
	go func() {
		var f http.HandlerFunc = func(w http.ResponseWriter, req *http.Request) {
			s.Serve(host, w, req)
		}

		loggerContext.Info("Serving")
		serveError := http.Serve(listener, f)
		if serveError != nil {
			elapsed := time.Since(start)
			errorContext := loggerContext.WithField("error", serveError)
			if elapsed > 100*time.Millisecond {
				// some time has passed, it's likely that it worked
				// but the listener has been asked to be closed
				errorContext.Debug("Listener has been closed")
			} else {
				errorContext.Errorf("Cannot serve")
			}
		}
	}()

	s.listeners[host] = listener
	return listener, nil
}

func (s *server) GetListeningPort(host string) (int, error) {
	listener, ok := s.listeners[host]
	if !ok {
		return 0, errors.New("Listener not found")
	}

	addr := listener.Addr().String()
	matches := regexp.MustCompile(`:(\d+)$`).FindStringSubmatch(addr)
	port, err := strconv.ParseInt(matches[1], 10, 64)

	return int(port), err
}

func (s *server) Serve(host string, w http.ResponseWriter, req *http.Request) {
	url, _ := url.Parse(req.URL.String())
	url.Host = host
	if len(url.Scheme) == 0 {
		url.Scheme = "http"
	}
	loggerContext := s.logger.WithField("url", url)

	cache, err := s.cacher.Open(url)
	if err != nil {
		loggerContext.Debug("Cache not found")

		infoOnError := &CacheInfo{
			ResponseWriter: w,
			Error:          err,
		}
		if !s.triggerOnCacheIssue(CacheNotFound, url, infoOnError) {
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	defer cache.Close()

	info := ServeHTTPCache(cache, w)
	if info.Error != nil {
		loggerContext.WithField("error", info.Error).Error("Cannot serve")
		s.triggerOnCacheIssue(CacheError, url, info)
		return
	}

	loggerContext.WithField("statusCode", info.StatusCode).Debug("Served")
	if info.Expires != nil && info.Expires.Before(time.Now()) {
		s.triggerOnCacheIssue(CacheExpired, url, info)
	}
}

func (s *server) StopListening(host string) error {
	listener, ok := s.listeners[host]
	if !ok {
		return errors.New("Listener not found")
	}

	err := listener.Close()

	loggerContext := s.logger.WithField("host", host)
	if err == nil {
		loggerContext.Info("Stopped listening")
	} else {
		loggerContext.WithField("error", err).Error("Cannot stop listening")
	}

	return err
}

func (s *server) StopAll() []string {
	hosts := make([]string, 0)

	for host, listener := range s.listeners {
		err := listener.Close()
		loggerContext := s.logger.WithField("host", host)

		if err == nil {
			loggerContext.Info("Stopped listening")
			hosts = append(hosts, host)
		} else {
			loggerContext.WithField("error", err).Error("Cannot stop listening")
		}
	}

	return hosts
}

func (s *server) triggerOnCacheIssue(t cacheIssueType, url *url.URL, info *CacheInfo) bool {
	if s.onCacheIssue == nil {
		return false
	}

	(*s.onCacheIssue)(CacheIssue{
		Type: t,
		URL:  url,
		Info: info,
	})

	return true
}
<|endoftext|>


<commit_before>package bazooka

import "time"

type Project struct {
	ScmType    string            `bson:"scm_type" json:"scm_type" validate:"required"`
	ScmURI     string            `bson:"scm_uri" json:"scm_uri" validate:"required"`
	Name       string            `bson:"name" json:"name" validate:"required"`
	ID         string            `bson:"id" json:"id"`
	HookKey    string            `bson:"hook_key" json:"hook_key"`
	JobCounter int               `bson:"job_counter" json:"job_counter"`
	Config     map[string]string `bson:"config" json:"config"`
}

type ProjectWithStatus struct {
	*Project
	LastJob *Job `json:"last_job"`
}

type JobStatus string

const (
	JOB_SUCCESS JobStatus = "SUCCESS"
	JOB_FAILED            = "FAILED"
	JOB_ERRORED           = "ERRORED"
	JOB_RUNNING           = "RUNNING"
)

type Job struct {
	ID              string      `bson:"id" json:"id"`
	Number          int         `bson:"number" json:"number"`
	ProjectID       string      `bson:"project_id" json:"project_id"`
	OrchestrationID string      `bson:"orchestration_id" json:"orchestration_id"`
	Started         time.Time   `bson:"started" json:"started"`
	Completed       time.Time   `bson:"completed" json:"completed"`
	Status          JobStatus   `bson:"status" json:"status"`
	SCMMetadata     SCMMetadata `bson:"scm_metadata" json:"scm_metadata"`
	Parameters      []string    `bson:"parameters" json:"parameters"`
}

type Variant struct {
	Status     JobStatus     `bson:"status" json:"status"`
	Started    time.Time     `bson:"started" json:"started"`
	Completed  time.Time     `bson:"completed" json:"completed"`
	BuildImage string        `bson:"image" json:"image"`
	ProjectID  string        `bson:"project_id" json:"job_id"`
	JobID      string        `bson:"job_id" json:"job_id"`
	Number     int           `bson:"number" json:"number"`
	ID         string        `bson:"id" json:"id"`
	Metas      *VariantMetas `bson:"metas" json:"metas"`
	Artifacts  []string      `bson:"artifacts" json:"artifacts"`
}

type VariantMetas []*VariantMeta

type VariantMeta struct {
	Kind  MetaKind `bson:"kind" json:"kind"`
	Name  string   `bson:"name" json:"name"`
	Value string   `bson:"value" json:"value"`
}

type MetaKind string

const (
	META_ENV  MetaKind = "env"
	META_LANG          = "lang"
)

type StartJob struct {
	ScmReference string   `json:"reference"`
	Parameters   []string `json:"parameters"`
}

type LogEntry struct {
	ID        string    `bson:"id" json:"id"`
	Message   string    `bson:"msg" json:"msg"`
	Time      time.Time `bson:"time" json:"time"`
	Level     string    `bson:"level" json:"level"`
	Phase     string    `bson:"phase" json:"phase"`
	Command   string    `bson:"command" json:"command"`
	ProjectID string    `bson:"project_id" json:"project_id"`
	JobID     string    `bson:"job_id" json:"job_id"`
	VariantID string    `bson:"variant_id" json:"variant_id"`
	Image     string    `bson:"image" json:"image"`
}

type SCMMetadata struct {
	Origin    string   `bson:"origin" json:"origin" yaml:"origin"`
	Reference string   `bson:"reference" json:"reference" yaml:"reference"`
	CommitID  string   `bson:"commit_id" json:"commit_id" yaml:"commit_id"`
	Author    Person   `bson:"author" json:"author" yaml:"author"`
	Committer Person   `bson:"committer" json:"committer" yaml:"committer"`
	Date      YamlTime `bson:"time" json:"date" yaml:"date"`
	Message   string   `bson:"message" json:"message" yaml:"message"`
}

func (ms *VariantMetas) Append(m *VariantMeta) { *ms = append(*ms, m) }
func (ms *VariantMetas) Len() int              { return len(*ms) }
func (ms *VariantMetas) Swap(i, j int)         { (*ms)[i], (*ms)[j] = (*ms)[j], (*ms)[i] }
func (ms *VariantMetas) Less(i, j int) bool {
	a, b := (*ms)[i], (*ms)[j]
	if a.Kind == b.Kind {
		return a.Name < b.Name
	}
	return a.Kind == META_LANG
}
<commit_msg>Fix a JSON config bug in Variant struct<commit_after>package bazooka

import "time"

type Project struct {
	ScmType    string            `bson:"scm_type" json:"scm_type" validate:"required"`
	ScmURI     string            `bson:"scm_uri" json:"scm_uri" validate:"required"`
	Name       string            `bson:"name" json:"name" validate:"required"`
	ID         string            `bson:"id" json:"id"`
	HookKey    string            `bson:"hook_key" json:"hook_key"`
	JobCounter int               `bson:"job_counter" json:"job_counter"`
	Config     map[string]string `bson:"config" json:"config"`
}

type ProjectWithStatus struct {
	*Project
	LastJob *Job `json:"last_job"`
}

type JobStatus string

const (
	JOB_SUCCESS JobStatus = "SUCCESS"
	JOB_FAILED            = "FAILED"
	JOB_ERRORED           = "ERRORED"
	JOB_RUNNING           = "RUNNING"
)

type Job struct {
	ID              string      `bson:"id" json:"id"`
	Number          int         `bson:"number" json:"number"`
	ProjectID       string      `bson:"project_id" json:"project_id"`
	OrchestrationID string      `bson:"orchestration_id" json:"orchestration_id"`
	Started         time.Time   `bson:"started" json:"started"`
	Completed       time.Time   `bson:"completed" json:"completed"`
	Status          JobStatus   `bson:"status" json:"status"`
	SCMMetadata     SCMMetadata `bson:"scm_metadata" json:"scm_metadata"`
	Parameters      []string    `bson:"parameters" json:"parameters"`
}

type Variant struct {
	Status     JobStatus     `bson:"status" json:"status"`
	Started    time.Time     `bson:"started" json:"started"`
	Completed  time.Time     `bson:"completed" json:"completed"`
	BuildImage string        `bson:"image" json:"image"`
	ProjectID  string        `bson:"project_id" json:"project_id"`
	JobID      string        `bson:"job_id" json:"job_id"`
	Number     int           `bson:"number" json:"number"`
	ID         string        `bson:"id" json:"id"`
	Metas      *VariantMetas `bson:"metas" json:"metas"`
	Artifacts  []string      `bson:"artifacts" json:"artifacts"`
}

type VariantMetas []*VariantMeta

type VariantMeta struct {
	Kind  MetaKind `bson:"kind" json:"kind"`
	Name  string   `bson:"name" json:"name"`
	Value string   `bson:"value" json:"value"`
}

type MetaKind string

const (
	META_ENV  MetaKind = "env"
	META_LANG          = "lang"
)

type StartJob struct {
	ScmReference string   `json:"reference"`
	Parameters   []string `json:"parameters"`
}

type LogEntry struct {
	ID        string    `bson:"id" json:"id"`
	Message   string    `bson:"msg" json:"msg"`
	Time      time.Time `bson:"time" json:"time"`
	Level     string    `bson:"level" json:"level"`
	Phase     string    `bson:"phase" json:"phase"`
	Command   string    `bson:"command" json:"command"`
	ProjectID string    `bson:"project_id" json:"project_id"`
	JobID     string    `bson:"job_id" json:"job_id"`
	VariantID string    `bson:"variant_id" json:"variant_id"`
	Image     string    `bson:"image" json:"image"`
}

type SCMMetadata struct {
	Origin    string   `bson:"origin" json:"origin" yaml:"origin"`
	Reference string   `bson:"reference" json:"reference" yaml:"reference"`
	CommitID  string   `bson:"commit_id" json:"commit_id" yaml:"commit_id"`
	Author    Person   `bson:"author" json:"author" yaml:"author"`
	Committer Person   `bson:"committer" json:"committer" yaml:"committer"`
	Date      YamlTime `bson:"time" json:"date" yaml:"date"`
	Message   string   `bson:"message" json:"message" yaml:"message"`
}

func (ms *VariantMetas) Append(m *VariantMeta) { *ms = append(*ms, m) }
func (ms *VariantMetas) Len() int              { return len(*ms) }
func (ms *VariantMetas) Swap(i, j int)         { (*ms)[i], (*ms)[j] = (*ms)[j], (*ms)[i] }
func (ms *VariantMetas) Less(i, j int) bool {
	a, b := (*ms)[i], (*ms)[j]
	if a.Kind == b.Kind {
		return a.Name < b.Name
	}
	return a.Kind == META_LANG
}
<|endoftext|>


<commit_before>package aws

import (
	"encoding/json"
	"log"
	"os/exec"
	"strings"

	"github.com/cockroachdb/roachprod/vm"
	"github.com/pkg/errors"
)

// Both M5 and I3 machines expose their EBS or local SSD volumes as NVMe block devices, but
// the actual device numbers vary a bit between the two types.
// This user-data script will create a filesystem, mount the data volume, and chmod 777.
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html
const awsStartupScript = `#!/usr/bin/env bash
set -x
disknum=0
for d in $(ls /dev/nvme?n1); do
  if ! mount | grep ${d}; then
    let "disknum++"
    echo "Disk ${d} not mounted, creating..."
    mountpoint="/mnt/data${disknum}"
    mkdir -p "${mountpoint}"
    mkfs.ext4 ${d}
    mount -o discard,defaults ${d} ${mountpoint}
    chmod 777 ${mountpoint}
    echo "${d} ${mountpoint} ext4 discard,defaults 1 1" | tee -a /etc/fstab
  else
    echo "Disk ${disknum}: ${d} already mounted, skipping..."
  fi
done
if [ "${disknum}" -eq "0" ]; then
  echo "No disks mounted, creating /mnt/data1"
  mkdir -p /mnt/data1
  chmod 777 /mnt/data1
fi

sudo apt-get update
sudo apt-get install -qy chrony
echo -e "\nserver 169.254.169.123 prefer iburst" | sudo tee -a /etc/chrony/chrony.conf
echo -e "\nmakestep 0.1 3" | sudo tee -a /etc/chrony/chrony.conf
sudo /etc/init.d/chrony restart
sudo chronyc -a waitsync 30 0.01 | sudo tee -a /root/chrony.log

sudo touch /mnt/data1/.roachprod-initialized
`

// runCommand is used to invoke an AWS command for which no output is expected.
func runCommand(args []string) error {
	cmd := exec.Command("aws", args...)

	_, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}
	return nil
}

// runJSONCommand invokes an aws command and parses the json output.
func runJSONCommand(args []string, parsed interface{}) error {
	// force json output in case the user has overridden the default behavior
	args = append(args[:len(args):len(args)], "--output", "json")
	cmd := exec.Command("aws", args...)

	rawJSON, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}

	if err := json.Unmarshal(rawJSON, &parsed); err != nil {
		return errors.Wrapf(err, "failed to parse json %s", rawJSON)
	}

	return nil
}

// splitMap splits a list of `key:value` pairs into a map.
func splitMap(data []string) (map[string]string, error) {
	ret := make(map[string]string, len(data))
	for _, part := range data {
		parts := strings.Split(part, ":")
		if len(parts) != 2 {
			return nil, errors.Errorf("Could not split Region:AMI: %s", part)
		}
		ret[parts[0]] = parts[1]
	}
	return ret, nil
}

// regionMap collates VM instances by their region.
func regionMap(vms vm.List) (map[string]vm.List, error) {
	// Fan out the work by region
	byRegion := make(map[string]vm.List)
	for _, m := range vms {
		region, err := zoneToRegion(m.Zone)
		if err != nil {
			return nil, err
		}
		byRegion[region] = append(byRegion[region], m)
	}
	return byRegion, nil
}

// zoneToRegion converts an availability zone like us-east-2a to the zone name us-east-2
func zoneToRegion(zone string) (string, error) {
	return zone[0 : len(zone)-1], nil
}
<commit_msg>Bump file descriptor limit on AWS<commit_after>package aws

import (
	"encoding/json"
	"log"
	"os/exec"
	"strings"

	"github.com/cockroachdb/roachprod/vm"
	"github.com/pkg/errors"
)

// Both M5 and I3 machines expose their EBS or local SSD volumes as NVMe block devices, but
// the actual device numbers vary a bit between the two types.
// This user-data script will create a filesystem, mount the data volume, and chmod 777.
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html
const awsStartupScript = `#!/usr/bin/env bash
set -x
disknum=0
for d in $(ls /dev/nvme?n1); do
  if ! mount | grep ${d}; then
    let "disknum++"
    echo "Disk ${d} not mounted, creating..."
    mountpoint="/mnt/data${disknum}"
    mkdir -p "${mountpoint}"
    mkfs.ext4 ${d}
    mount -o discard,defaults ${d} ${mountpoint}
    chmod 777 ${mountpoint}
    echo "${d} ${mountpoint} ext4 discard,defaults 1 1" | tee -a /etc/fstab
  else
    echo "Disk ${disknum}: ${d} already mounted, skipping..."
  fi
done
if [ "${disknum}" -eq "0" ]; then
  echo "No disks mounted, creating /mnt/data1"
  mkdir -p /mnt/data1
  chmod 777 /mnt/data1
fi

sudo apt-get update
sudo apt-get install -qy chrony
echo -e "\nserver 169.254.169.123 prefer iburst" | sudo tee -a /etc/chrony/chrony.conf
echo -e "\nmakestep 0.1 3" | sudo tee -a /etc/chrony/chrony.conf
sudo /etc/init.d/chrony restart
sudo chronyc -a waitsync 30 0.01 | sudo tee -a /root/chrony.log

# increase the default maximum number of open file descriptors for
# root and non-root users. Load generators running a lot of concurrent
# workers bump into this often.
sudo sh -c 'echo "root - nofile 65536\n* - nofile 65536" > /etc/security/limits.d/10-roachprod-nofiles.conf'
sudo touch /mnt/data1/.roachprod-initialized
`

// runCommand is used to invoke an AWS command for which no output is expected.
func runCommand(args []string) error {
	cmd := exec.Command("aws", args...)

	_, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}
	return nil
}

// runJSONCommand invokes an aws command and parses the json output.
func runJSONCommand(args []string, parsed interface{}) error {
	// force json output in case the user has overridden the default behavior
	args = append(args[:len(args):len(args)], "--output", "json")
	cmd := exec.Command("aws", args...)

	rawJSON, err := cmd.Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			log.Println(string(exitErr.Stderr))
		}
		return errors.Wrapf(err, "failed to run: aws %s", strings.Join(args, " "))
	}

	if err := json.Unmarshal(rawJSON, &parsed); err != nil {
		return errors.Wrapf(err, "failed to parse json %s", rawJSON)
	}

	return nil
}

// splitMap splits a list of `key:value` pairs into a map.
func splitMap(data []string) (map[string]string, error) {
	ret := make(map[string]string, len(data))
	for _, part := range data {
		parts := strings.Split(part, ":")
		if len(parts) != 2 {
			return nil, errors.Errorf("Could not split Region:AMI: %s", part)
		}
		ret[parts[0]] = parts[1]
	}
	return ret, nil
}

// regionMap collates VM instances by their region.
func regionMap(vms vm.List) (map[string]vm.List, error) {
	// Fan out the work by region
	byRegion := make(map[string]vm.List)
	for _, m := range vms {
		region, err := zoneToRegion(m.Zone)
		if err != nil {
			return nil, err
		}
		byRegion[region] = append(byRegion[region], m)
	}
	return byRegion, nil
}

// zoneToRegion converts an availability zone like us-east-2a to the zone name us-east-2
func zoneToRegion(zone string) (string, error) {
	return zone[0 : len(zone)-1], nil
}
<|endoftext|>


<commit_before><commit_msg>region: fix TestProcessRPCs/2 test<commit_after><|endoftext|>


<commit_before>// Package test includes all integration tests (typically that talk to Queue
// service). Unit tests go in package they are testing.
package test

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/taskcluster/httpbackoff"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
	_ "github.com/taskcluster/taskcluster-worker/plugins/artifacts"
	_ "github.com/taskcluster/taskcluster-worker/plugins/livelog"
)

func TestUpload(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("Currently not supported on Windows")
	}

	expires := tcclient.Time(time.Now().Add(time.Minute * 30))

	task, workerType := NewTestTask("TestUpload")
	payload := TaskPayload{
		Command: helloGoodbye(),
		Artifacts: []struct {
			Expires tcclient.Time `json:"expires,omitempty"`
			Name    string        `json:"name"`
			Path    string        `json:"path"`
			Type    string        `json:"type"`
		}{
			{
				Expires: expires,
				Name:    "SampleArtifacts/_/X.txt",
				Path:    filepath.Join(testdata, "SampleArtifacts/_/X.txt"),
				Type:    "file",
			},
		},
	}
	taskID, q := SubmitTask(t, task, payload)
	t.Logf("Task ID: %v", taskID)
	RunTestWorker(workerType)

	// now check results
	// some required substrings - not all, just a selection
	expectedArtifacts := map[string]struct {
		extracts        []string
		contentEncoding string
		expires         tcclient.Time
	}{
		"public/logs/live_backing.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
			},
			contentEncoding: "gzip",
			expires:         task.Expires,
		},
		"public/logs/live.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
			},
			contentEncoding: "gzip",
			expires:         task.Expires,
		},
		"SampleArtifacts/_/X.txt": {
			extracts: []string{
				"test artifact",
			},
			contentEncoding: "",
			expires:         payload.Artifacts[0].Expires,
		},
	}

	artifacts, err := q.ListArtifacts(taskID, "0", "", "")

	if err != nil {
		t.Fatalf("Error listing artifacts: %v", err)
	}

	actualArtifacts := make(map[string]struct {
		ContentType string        `json:"contentType"`
		Expires     tcclient.Time `json:"expires"`
		Name        string        `json:"name"`
		StorageType string        `json:"storageType"`
	}, len(artifacts.Artifacts))

	for _, actualArtifact := range artifacts.Artifacts {
		actualArtifacts[actualArtifact.Name] = actualArtifact
	}

	for artifact := range expectedArtifacts {
		if a, ok := actualArtifacts[artifact]; ok {
			if a.ContentType != "text/plain; charset=utf-8" {
				t.Errorf("Artifact %s should have mime type 'text/plain; charset=utf-8' but has '%s'", artifact, a.ContentType)
			}
			if a.Expires.String() != expectedArtifacts[artifact].expires.String() {
				t.Errorf("Artifact %s should have expiry '%s' but has '%s'", artifact, expires, a.Expires)
			}
		} else {
			t.Errorf("Artifact '%s' not created", artifact)
		}
	}

	// now check content was uploaded to Amazon, and is correct

	for artifact, content := range expectedArtifacts {
		var url *url.URL
		url, err = q.GetLatestArtifact_SignedURL(taskID, artifact, 10*time.Minute)
		if err != nil {
			t.Fatalf("Error trying to fetch artifacts from Amazon...\n%s", err)
		}
		// need to do this so Content-Encoding header isn't swallowed by Go for test later on
		tr := &http.Transport{
			DisableCompression: true,
		}
		client := &http.Client{Transport: tr}
		var rawResp *http.Response
		rawResp, _, err = httpbackoff.ClientGet(client, url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch decompressed artifact from signed URL %s ...\n%s", url.String(), err)
		}
		var resp *http.Response
		resp, _, err = httpbackoff.Get(url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch artifact from signed URL %s ...\n%s", url.String(), err)
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatalf("Error trying to read response body of artifact from signed URL %s ...\n%s", url.String(), err)
		}
		for _, requiredSubstring := range content.extracts {
			if !strings.Contains(string(b), requiredSubstring) {
				t.Errorf("Artifact '%s': Could not find substring %q in '%s'", artifact, requiredSubstring, string(b))
			}
		}
		if actualContentEncoding := rawResp.Header.Get("Content-Encoding"); actualContentEncoding != content.contentEncoding {
			t.Fatalf("Expected Content-Encoding %q but got Content-Encoding %q for artifact %q from url %v", content.contentEncoding, actualContentEncoding, artifact, url)
		}
		if actualContentType := resp.Header.Get("Content-Type"); actualContentType != "text/plain; charset=utf-8" {
			t.Fatalf("Content-Type in signed URL %v response should be '%v' but is '%v'", url, "text/plain; charset=utf-8", actualContentType)
		}
	}

	var status *queue.TaskStatusResponse
	status, err = q.Status(taskID)
	if err != nil {
		t.Fatal("Error retrieving status from queue")
	}
	if status.Status.State != "completed" {
		t.Fatalf("Expected state 'completed' but got state '%v'", status.Status.State)
	}
}
<commit_msg>Disabled broken test, we should not put a file in testdata/\n\nWe should create the file from the command in the payload...<commit_after>// Package test includes all integration tests (typically that talk to Queue
// service). Unit tests go in package they are testing.
package test

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/taskcluster/httpbackoff"
	tcclient "github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
	_ "github.com/taskcluster/taskcluster-worker/plugins/artifacts"
	_ "github.com/taskcluster/taskcluster-worker/plugins/livelog"
)

func TestUpload(t *testing.T) {
	t.Skip(`Test case is broken because native engine runs with $HOME as cwd,
		and extracts artifacts relative to cwd. In fact it doesn't even allow
		export of artifacts outside of cwd == $HOME.`)
	if runtime.GOOS == "windows" {
		t.Skip("Currently not supported on Windows")
	}

	expires := tcclient.Time(time.Now().Add(time.Minute * 30))

	task, workerType := NewTestTask("TestUpload")
	payload := TaskPayload{
		Command: helloGoodbye(),
		Artifacts: []struct {
			Expires tcclient.Time `json:"expires,omitempty"`
			Name    string        `json:"name"`
			Path    string        `json:"path"`
			Type    string        `json:"type"`
		}{
			{
				Expires: expires,
				Name:    "SampleArtifacts/_/X.txt",
				Path:    filepath.Join(testdata, "SampleArtifacts/_/X.txt"),
				Type:    "file",
			},
		},
	}
	taskID, q := SubmitTask(t, task, payload)
	t.Logf("Task ID: %v", taskID)
	RunTestWorker(workerType)

	// now check results
	// some required substrings - not all, just a selection
	expectedArtifacts := map[string]struct {
		extracts        []string
		contentEncoding string
		expires         tcclient.Time
	}{
		"public/logs/live_backing.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
			},
			contentEncoding: "gzip",
			expires:         task.Expires,
		},
		"public/logs/live.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
			},
			contentEncoding: "gzip",
			expires:         task.Expires,
		},
		"SampleArtifacts/_/X.txt": {
			extracts: []string{
				"test artifact",
			},
			contentEncoding: "",
			expires:         payload.Artifacts[0].Expires,
		},
	}

	artifacts, err := q.ListArtifacts(taskID, "0", "", "")

	if err != nil {
		t.Fatalf("Error listing artifacts: %v", err)
	}

	actualArtifacts := make(map[string]struct {
		ContentType string        `json:"contentType"`
		Expires     tcclient.Time `json:"expires"`
		Name        string        `json:"name"`
		StorageType string        `json:"storageType"`
	}, len(artifacts.Artifacts))

	for _, actualArtifact := range artifacts.Artifacts {
		actualArtifacts[actualArtifact.Name] = actualArtifact
	}

	for artifact := range expectedArtifacts {
		if a, ok := actualArtifacts[artifact]; ok {
			if a.ContentType != "text/plain; charset=utf-8" {
				t.Errorf("Artifact %s should have mime type 'text/plain; charset=utf-8' but has '%s'", artifact, a.ContentType)
			}
			if a.Expires.String() != expectedArtifacts[artifact].expires.String() {
				t.Errorf("Artifact %s should have expiry '%s' but has '%s'", artifact, expires, a.Expires)
			}
		} else {
			t.Errorf("Artifact '%s' not created", artifact)
		}
	}

	// now check content was uploaded to Amazon, and is correct

	for artifact, content := range expectedArtifacts {
		var url *url.URL
		url, err = q.GetLatestArtifact_SignedURL(taskID, artifact, 10*time.Minute)
		if err != nil {
			t.Fatalf("Error trying to fetch artifacts from Amazon...\n%s", err)
		}
		// need to do this so Content-Encoding header isn't swallowed by Go for test later on
		tr := &http.Transport{
			DisableCompression: true,
		}
		client := &http.Client{Transport: tr}
		var rawResp *http.Response
		rawResp, _, err = httpbackoff.ClientGet(client, url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch decompressed artifact from signed URL %s ...\n%s", url.String(), err)
		}
		var resp *http.Response
		resp, _, err = httpbackoff.Get(url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch artifact from signed URL %s ...\n%s", url.String(), err)
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatalf("Error trying to read response body of artifact from signed URL %s ...\n%s", url.String(), err)
		}
		for _, requiredSubstring := range content.extracts {
			if !strings.Contains(string(b), requiredSubstring) {
				t.Errorf("Artifact '%s': Could not find substring %q in '%s'", artifact, requiredSubstring, string(b))
			}
		}
		if actualContentEncoding := rawResp.Header.Get("Content-Encoding"); actualContentEncoding != content.contentEncoding {
			t.Fatalf("Expected Content-Encoding %q but got Content-Encoding %q for artifact %q from url %v", content.contentEncoding, actualContentEncoding, artifact, url)
		}
		if actualContentType := resp.Header.Get("Content-Type"); actualContentType != "text/plain; charset=utf-8" {
			t.Fatalf("Content-Type in signed URL %v response should be '%v' but is '%v'", url, "text/plain; charset=utf-8", actualContentType)
		}
	}

	var status *queue.TaskStatusResponse
	status, err = q.Status(taskID)
	if err != nil {
		t.Fatal("Error retrieving status from queue")
	}
	if status.Status.State != "completed" {
		t.Fatalf("Expected state 'completed' but got state '%v'", status.Status.State)
	}
}
<|endoftext|>


<commit_before>package vsphere

import (
	"context"
	"net/url"
	"time"

	"github.com/pkg/errors"
	vspherejanitor "github.com/travis-ci/vsphere-janitor"
	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
)

type Client struct {
	client *govmomi.Client
}

func NewClient(ctx context.Context, u *url.URL, insecure bool) (*Client, error) {
	vClient, err := govmomi.NewClient(ctx, u, insecure)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't create govmomi client")
	}

	return &Client{
		client: vClient,
	}, nil
}

func (c *Client) ListVMs(ctx context.Context, path string) ([]vspherejanitor.VirtualMachine, error) {
	folder, err := c.folder(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "error finding folder")
	}

	rawVMs, err := folder.Children(ctx)
	if err != nil {
		return nil, errors.Wrap(err, "error listing contents of VM folder")
	}

	vms := make([]vspherejanitor.VirtualMachine, 0, len(rawVMs))

	for _, rawVM := range rawVMs {
		ovm, ok := rawVM.(*object.VirtualMachine)
		if !ok {
			continue
		}

		mvm := &mo.VirtualMachine{}

		err = ovm.Properties(ctx, ovm.Reference(), []string{"config", "summary"}, mvm)
		if err != nil {
			return nil, errors.Wrap(err, "couldn't get properties for VM")
		}

		vm := &VirtualMachine{
			vm:  ovm,
			mvm: mvm,
		}

		vms = append(vms, vm)
	}

	return vms, nil
}

func (c *Client) folder(ctx context.Context, path string) (*object.Folder, error) {
	searchIndex := object.NewSearchIndex(c.client.Client)

	folderRef, err := searchIndex.FindByInventoryPath(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "error looking for VM folder")
	}

	if folderRef == nil {
		return nil, errors.New("couldn't find VM folder")
	}

	folder, ok := folderRef.(*object.Folder)
	if !ok {
		return nil, errors.Errorf("VM folder is not a folder but a %T", folderRef)
	}

	return folder, nil
}

type VirtualMachine struct {
	vm  *object.VirtualMachine
	mvm *mo.VirtualMachine
}

func (vm *VirtualMachine) Name() string {
	if vm.mvm.Config == nil {
		return "<unnamed>"
	}

	return vm.mvm.Config.Name
}

func (vm *VirtualMachine) Uptime() time.Duration {
	return time.Duration(vm.mvm.Summary.QuickStats.UptimeSeconds) * time.Second
}

func (vm *VirtualMachine) BootTime() *time.Time {
	return vm.mvm.Summary.Runtime.BootTime
}

func (vm *VirtualMachine) PoweredOn() bool {
	return vm.mvm.Summary.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOn
}

func (vm *VirtualMachine) PowerOff(ctx context.Context) error {
	task, err := vm.vm.PowerOff(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't create power off task")
	}

	err = task.Wait(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't power off instance")
	}

	return nil
}

func (vm *VirtualMachine) Destroy(ctx context.Context) error {
	task, err := vm.vm.Destroy(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't create destroy task")
	}

	err = task.Wait(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't destroy instance")
	}

	return nil
}
<commit_msg>vsphere/list: don't stop if we're getting an error on a single vm<commit_after>package vsphere

import (
	"context"
	"net/url"
	"time"

	"github.com/pkg/errors"
	vspherejanitor "github.com/travis-ci/vsphere-janitor"
	"github.com/travis-ci/vsphere-janitor/log"
	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
)

type Client struct {
	client *govmomi.Client
}

func NewClient(ctx context.Context, u *url.URL, insecure bool) (*Client, error) {
	vClient, err := govmomi.NewClient(ctx, u, insecure)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't create govmomi client")
	}

	return &Client{
		client: vClient,
	}, nil
}

func (c *Client) ListVMs(ctx context.Context, path string) ([]vspherejanitor.VirtualMachine, error) {
	folder, err := c.folder(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "error finding folder")
	}

	rawVMs, err := folder.Children(ctx)
	if err != nil {
		return nil, errors.Wrap(err, "error listing contents of VM folder")
	}

	vms := make([]vspherejanitor.VirtualMachine, 0, len(rawVMs))

	for _, rawVM := range rawVMs {
		ovm, ok := rawVM.(*object.VirtualMachine)
		if !ok {
			continue
		}

		mvm := &mo.VirtualMachine{}

		err = ovm.Properties(ctx, ovm.Reference(), []string{"config", "summary"}, mvm)
		if err != nil {
			log.WithContext(ctx).WithError(err).Info("couldn't get properties for VM")
			continue
		}

		vm := &VirtualMachine{
			vm:  ovm,
			mvm: mvm,
		}

		vms = append(vms, vm)
	}

	return vms, nil
}

func (c *Client) folder(ctx context.Context, path string) (*object.Folder, error) {
	searchIndex := object.NewSearchIndex(c.client.Client)

	folderRef, err := searchIndex.FindByInventoryPath(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "error looking for VM folder")
	}

	if folderRef == nil {
		return nil, errors.New("couldn't find VM folder")
	}

	folder, ok := folderRef.(*object.Folder)
	if !ok {
		return nil, errors.Errorf("VM folder is not a folder but a %T", folderRef)
	}

	return folder, nil
}

type VirtualMachine struct {
	vm  *object.VirtualMachine
	mvm *mo.VirtualMachine
}

func (vm *VirtualMachine) Name() string {
	if vm.mvm.Config == nil {
		return "<unnamed>"
	}

	return vm.mvm.Config.Name
}

func (vm *VirtualMachine) Uptime() time.Duration {
	return time.Duration(vm.mvm.Summary.QuickStats.UptimeSeconds) * time.Second
}

func (vm *VirtualMachine) BootTime() *time.Time {
	return vm.mvm.Summary.Runtime.BootTime
}

func (vm *VirtualMachine) PoweredOn() bool {
	return vm.mvm.Summary.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOn
}

func (vm *VirtualMachine) PowerOff(ctx context.Context) error {
	task, err := vm.vm.PowerOff(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't create power off task")
	}

	err = task.Wait(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't power off instance")
	}

	return nil
}

func (vm *VirtualMachine) Destroy(ctx context.Context) error {
	task, err := vm.vm.Destroy(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't create destroy task")
	}

	err = task.Wait(ctx)
	if err != nil {
		return errors.Wrap(err, "couldn't destroy instance")
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>fixed error message<commit_after><|endoftext|>