<commit_before>package proxy

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/downloader/surfer"
	"github.com/henrylee2cn/pholcus/common/ping"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
)

type Proxy struct {
	ipRegexp           *regexp.Regexp
	proxyIPTypeRegexp  *regexp.Regexp
	proxyUrlTypeRegexp *regexp.Regexp
	allIps             map[string]string
	all                map[string]bool
	online             int64
	usable             map[string]*ProxyForHost
	ticker             *time.Ticker
	tickMinute         int64
	threadPool         chan bool
	surf               surfer.Surfer
	sync.Mutex
}

const (
	CONN_TIMEOUT = 4 //4s
	DAIL_TIMEOUT = 4 //4s
	TRY_TIMES    = 3
	// IP测速的最大并发量
	MAX_THREAD_NUM = 1000
)

func New() *Proxy {
	p := &Proxy{
		ipRegexp:           regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`),
		proxyIPTypeRegexp:  regexp.MustCompile(`https?://([\w]*:[\w]*@)?[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`),
		proxyUrlTypeRegexp: regexp.MustCompile(`((https?|ftp):\/\/)?(([^:\n\r]+):([^@\n\r]+)@)?((www\.)?([^/\n\r:]+)):?([0-9]{1,5})?\/?([^?\n\r]+)?\??([^#\n\r]*)?#?([^\n\r]*)`),
		allIps:             map[string]string{},
		all:                map[string]bool{},
		usable:             make(map[string]*ProxyForHost),
		threadPool:         make(chan bool, MAX_THREAD_NUM),
		surf:               surfer.New(),
	}
	go p.Update()
	return p
}

// 代理IP数量
func (self *Proxy) Count() int64 {
	return self.online
}

// 更新代理IP列表
func (self *Proxy) Update() *Proxy {
	f, err := os.Open(config.PROXY)
	if err != nil {
		// logs.Log.Error("Error: %v\n", err)
		return self
	}
	b, _ := ioutil.ReadAll(f)
	f.Close()

	proxysIPType := self.proxyIPTypeRegexp.FindAllString(string(b), -1)
	for _, proxy := range proxysIPType {
		self.allIps[proxy] = self.ipRegexp.FindString(proxy)
		self.all[proxy] = false
	}

	proxysUrlType := self.proxyUrlTypeRegexp.FindAllString(string(b), -1)
	for _, proxy := range proxysUrlType {
		gvalue := self.proxyUrlTypeRegexp.FindStringSubmatch(proxy)
		self.allIps[proxy] = gvalue[6]
		self.all[proxy] = false
	}

	log.Printf(" *     读取代理IP: %v 条\n", len(self.all))

	self.findOnline()

	return self
}

// 筛选在线的代理IP
func (self *Proxy) findOnline() *Proxy {
	log.Printf(" *     正在筛选在线的代理IP……")
	self.online = 0
	for proxy := range self.all {
		self.threadPool <- true
		go func(proxy string) {
			alive, _, _ := ping.Ping(self.allIps[proxy], CONN_TIMEOUT)
			self.Lock()
			self.all[proxy] = alive
			self.Unlock()
			if alive {
				atomic.AddInt64(&self.online, 1)
			}
			<-self.threadPool
		}(proxy)
	}
	for len(self.threadPool) > 0 {
		time.Sleep(0.2e9)
	}
	self.online = atomic.LoadInt64(&self.online)
	log.Printf(" *     在线代理IP筛选完成，共计：%v 个\n", self.online)

	return self
}

// 更新继时器
func (self *Proxy) UpdateTicker(tickMinute int64) {
	self.tickMinute = tickMinute
	self.ticker = time.NewTicker(time.Duration(self.tickMinute) * time.Minute)
	for _, proxyForHost := range self.usable {
		proxyForHost.curIndex++
		proxyForHost.isEcho = true
	}
}

// 获取本次循环中未使用的代理IP及其响应时长
func (self *Proxy) GetOne(u string) (curProxy string) {
	if self.online == 0 {
		return
	}
	u2, _ := url.Parse(u)
	if u2.Host == "" {
		logs.Log.Informational(" *     [%v]设置代理IP失败，目标url不正确\n", u)
		return
	}
	var key = u2.Host
	if strings.Count(key, ".") > 1 {
		key = key[strings.Index(key, ".")+1:]
	}

	self.Lock()
	defer self.Unlock()

	var ok = true
	var proxyForHost = self.usable[key]

	select {
	case <-self.ticker.C:
		proxyForHost.curIndex++
		if proxyForHost.curIndex >= proxyForHost.Len() {
			_, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
		}
		proxyForHost.isEcho = true

	default:
		if proxyForHost == nil {
			self.usable[key] = &ProxyForHost{
				proxys:    []string{},
				timedelay: []time.Duration{},
				isEcho:    true,
			}
			proxyForHost, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
		} else if l := proxyForHost.Len(); l == 0 {
			ok = false
		} else if proxyForHost.curIndex >= l {
			_, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
			proxyForHost.isEcho = true
		}
	}
	if !ok {
		logs.Log.Informational(" *     [%v]设置代理IP失败，没有可用的代理IP\n", key)
		return
	}
	curProxy = proxyForHost.proxys[proxyForHost.curIndex]
	if proxyForHost.isEcho {
		logs.Log.Informational(" *     设置代理IP为 [%v](%v)\n",
			curProxy,
			proxyForHost.timedelay[proxyForHost.curIndex],
		)
		proxyForHost.isEcho = false
	}
	return
}

// 测试并排序
func (self *Proxy) testAndSort(key string, testHost string) (*ProxyForHost, bool) {
	logs.Log.Informational(" *     [%v]正在测试与排序代理IP……", key)
	proxyForHost := self.usable[key]
	proxyForHost.proxys = []string{}
	proxyForHost.timedelay = []time.Duration{}
	proxyForHost.curIndex = 0
	for proxy, online := range self.all {
		if !online {
			continue
		}
		self.threadPool <- true
		go func(proxy string) {
			//alive, timedelay := self.findUsable(proxy, testHost)
			alive, timedelay := self.findUsable(proxy, testHost)
			if alive {
				proxyForHost.Mutex.Lock()
				proxyForHost.proxys = append(proxyForHost.proxys, proxy)
				proxyForHost.timedelay = append(proxyForHost.timedelay, timedelay)
				proxyForHost.Mutex.Unlock()
			}
			<-self.threadPool
		}(proxy)
	}
	for len(self.threadPool) > 0 {
		time.Sleep(0.2e9)
	}
	if proxyForHost.Len() > 0 {
		sort.Sort(proxyForHost)
		logs.Log.Informational(" *     [%v]测试与排序代理IP完成，可用：%v 个\n", key, proxyForHost.Len())
		return proxyForHost, true
	}
	logs.Log.Informational(" *     [%v]测试与排序代理IP完成，没有可用的代理IP\n", key)
	return proxyForHost, false
}

// 测试代理ip可用性
func (self *Proxy) findUsable(proxy string, testHost string) (alive bool, timedelay time.Duration) {
	t0 := time.Now()
	req := &request.Request{
		Url:         testHost,
		Method:      "HEAD",
		Header:      make(http.Header),
		DialTimeout: time.Second * time.Duration(DAIL_TIMEOUT),
		ConnTimeout: time.Second * time.Duration(CONN_TIMEOUT),
		TryTimes:    TRY_TIMES,
	}
	req.SetProxy(proxy)
	resp, err := self.surf.Download(req)
	if err != nil {
		return false, 0
	}

	if resp.StatusCode != http.StatusOK {
		return false, 0
	}

	return true, time.Since(t0)
}

func (self *Proxy) findUsable2(proxy string) (alive bool, timedelay time.Duration) {
	t0 := time.Now()
	req := &request.Request{
		Url:         "http://httpbin.org/",
		Method:      "GET",
		DialTimeout: time.Second * time.Duration(DAIL_TIMEOUT),
		ConnTimeout: time.Second * time.Duration(CONN_TIMEOUT),
		TryTimes:    TRY_TIMES,
	}
	req.SetProxy(proxy)
	_, err := self.surf.Download(req)
	return err == nil, time.Since(t0)
}
<commit_msg>proxy<commit_after>package proxy

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/downloader/surfer"
	"github.com/henrylee2cn/pholcus/common/ping"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
)

type Proxy struct {
	ipRegexp           *regexp.Regexp
	proxyIPTypeRegexp  *regexp.Regexp
	proxyUrlTypeRegexp *regexp.Regexp
	allIps             map[string]string
	all                map[string]bool
	online             int64
	usable             map[string]*ProxyForHost
	ticker             *time.Ticker
	tickMinute         int64
	threadPool         chan bool
	surf               surfer.Surfer
	sync.Mutex
}

const (
	CONN_TIMEOUT = 4 //4s
	DAIL_TIMEOUT = 4 //4s
	TRY_TIMES    = 3
	// IP测速的最大并发量
	MAX_THREAD_NUM = 1000
)

func New() *Proxy {
	p := &Proxy{
		ipRegexp:           regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`),
		proxyIPTypeRegexp:  regexp.MustCompile(`https?://([\w]*:[\w]*@)?[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`),
		proxyUrlTypeRegexp: regexp.MustCompile(`((https?|ftp):\/\/)?(([^:\n\r]+):([^@\n\r]+)@)?((www\.)?([^/\n\r:]+)):?([0-9]{1,5})?\/?([^?\n\r]+)?\??([^#\n\r]*)?#?([^\n\r]*)`),
		allIps:             map[string]string{},
		all:                map[string]bool{},
		usable:             make(map[string]*ProxyForHost),
		threadPool:         make(chan bool, MAX_THREAD_NUM),
		surf:               surfer.New(),
	}
	go p.Update()
	return p
}

// 代理IP数量
func (self *Proxy) Count() int64 {
	return self.online
}

// 更新代理IP列表
func (self *Proxy) Update() *Proxy {
	f, err := os.Open(config.PROXY)
	if err != nil {
		// logs.Log.Error("Error: %v\n", err)
		return self
	}
	b, _ := ioutil.ReadAll(f)
	f.Close()

	proxysIPType := self.proxyIPTypeRegexp.FindAllString(string(b), -1)
	for _, proxy := range proxysIPType {
		self.allIps[proxy] = self.ipRegexp.FindString(proxy)
		self.all[proxy] = false
	}

	proxysUrlType := self.proxyUrlTypeRegexp.FindAllString(string(b), -1)
	for _, proxy := range proxysUrlType {
		gvalue := self.proxyUrlTypeRegexp.FindStringSubmatch(proxy)
		self.allIps[proxy] = gvalue[6]
		self.all[proxy] = false
	}

	log.Printf(" *     读取代理IP: %v 条\n", len(self.all))

	self.findOnline()

	return self
}

// 筛选在线的代理IP
func (self *Proxy) findOnline() *Proxy {
	log.Printf(" *     正在筛选在线的代理IP……")
	self.online = 0
	for proxy := range self.all {
		self.threadPool <- true
		go func(proxy string) {
			alive, _, _ := ping.Ping(self.allIps[proxy], CONN_TIMEOUT)
			self.Lock()
			self.all[proxy] = alive
			self.Unlock()
			if alive {
				atomic.AddInt64(&self.online, 1)
			}
			<-self.threadPool
		}(proxy)
	}
	for len(self.threadPool) > 0 {
		time.Sleep(0.2e9)
	}
	self.online = atomic.LoadInt64(&self.online)
	log.Printf(" *     在线代理IP筛选完成，共计：%v 个\n", self.online)

	return self
}

// 更新继时器
func (self *Proxy) UpdateTicker(tickMinute int64) {
	self.tickMinute = tickMinute
	self.ticker = time.NewTicker(time.Duration(self.tickMinute) * time.Minute)
	for _, proxyForHost := range self.usable {
		proxyForHost.curIndex++
		proxyForHost.isEcho = true
	}
}

// 获取本次循环中未使用的代理IP及其响应时长
func (self *Proxy) GetOne(u string) (curProxy string) {
	if self.online == 0 {
		return
	}
	u2, _ := url.Parse(u)
	if u2.Host == "" {
		logs.Log.Informational(" *     [%v]设置代理IP失败，目标url不正确\n", u)
		return
	}
	var key = u2.Host
	if strings.Count(key, ".") > 1 {
		key = key[strings.Index(key, ".")+1:]
	}

	self.Lock()
	defer self.Unlock()

	var ok = true
	var proxyForHost = self.usable[key]

	select {
	case <-self.ticker.C:
		proxyForHost.curIndex++
		if proxyForHost.curIndex >= proxyForHost.Len() {
			_, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
		}
		proxyForHost.isEcho = true

	default:
		if proxyForHost == nil {
			self.usable[key] = &ProxyForHost{
				proxys:    []string{},
				timedelay: []time.Duration{},
				isEcho:    true,
			}
			proxyForHost, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
		} else if l := proxyForHost.Len(); l == 0 {
			ok = false
		} else if proxyForHost.curIndex >= l {
			_, ok = self.testAndSort(key, u2.Scheme+"://"+u2.Host)
			proxyForHost.isEcho = true
		}
	}
	if !ok {
		logs.Log.Informational(" *     [%v]设置代理IP失败，没有可用的代理IP\n", key)
		return
	}
	curProxy = proxyForHost.proxys[proxyForHost.curIndex]
	if proxyForHost.isEcho {
		logs.Log.Informational(" *     设置代理IP为 [%v](%v)\n",
			curProxy,
			proxyForHost.timedelay[proxyForHost.curIndex],
		)
		proxyForHost.isEcho = false
	}
	return
}

// 测试并排序
func (self *Proxy) testAndSort(key string, testHost string) (*ProxyForHost, bool) {
	logs.Log.Informational(" *     [%v]正在测试与排序代理IP……", key)
	proxyForHost := self.usable[key]
	proxyForHost.proxys = []string{}
	proxyForHost.timedelay = []time.Duration{}
	proxyForHost.curIndex = 0
	for proxy, online := range self.all {
		if !online {
			continue
		}
		self.threadPool <- true
		go func(proxy string) {
			alive, timedelay := self.findUsable(proxy, testHost)
			if alive {
				proxyForHost.Mutex.Lock()
				proxyForHost.proxys = append(proxyForHost.proxys, proxy)
				proxyForHost.timedelay = append(proxyForHost.timedelay, timedelay)
				proxyForHost.Mutex.Unlock()
			}
			<-self.threadPool
		}(proxy)
	}
	for len(self.threadPool) > 0 {
		time.Sleep(0.2e9)
	}
	if proxyForHost.Len() > 0 {
		sort.Sort(proxyForHost)
		logs.Log.Informational(" *     [%v]测试与排序代理IP完成，可用：%v 个\n", key, proxyForHost.Len())
		return proxyForHost, true
	}
	logs.Log.Informational(" *     [%v]测试与排序代理IP完成，没有可用的代理IP\n", key)
	return proxyForHost, false
}

// 测试代理ip可用性
func (self *Proxy) findUsable(proxy string, testHost string) (alive bool, timedelay time.Duration) {
	t0 := time.Now()
	req := &request.Request{
		Url:         testHost,
		Method:      "HEAD",
		Header:      make(http.Header),
		DialTimeout: time.Second * time.Duration(DAIL_TIMEOUT),
		ConnTimeout: time.Second * time.Duration(CONN_TIMEOUT),
		TryTimes:    TRY_TIMES,
	}
	req.SetProxy(proxy)
	resp, err := self.surf.Download(req)

	if resp.StatusCode != http.StatusOK {
		return false, 0
	}

	return err == nil, time.Since(t0)
}
<|endoftext|>


<commit_before>package controllers

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	"github.com/takonews/takonews-api/app/models"
	"github.com/takonews/takonews-api/db"
)

// ArticleIndex show articles
// Available Query Parameters
// * sort
// * fields
// * filter
func ArticleIndex(c *gin.Context) {
	// parameters
	var sort []string
	var fields []string
	var startDate time.Time
	var endDate time.Time

	sort = strings.Split(c.Query("sort"), ",")
	fields = strings.Split(c.Query("fields"), ",")
	loc, err := time.LoadLocation("Asia/Tokyo")
	if err != nil {
		panic(err)
	}
	now := time.Now().In(loc)
	if c.Query("start-date") == "" { // default: today:00:00:00
		startDate = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc)
	} else {
		startDate, err = time.Parse("2006-01-02", c.Query("start-date"))
		startDate = startDate.Add(-9 * time.Hour).In(loc)
	}
	if c.Query("end-date") == "" { // default: tomorrow:00:00:00
		endDate = time.Date(now.Year(), now.Month(), now.Day(), 24, 0, 0, 0, loc)
	} else {
		endDate, err = time.Parse("2006-01-02", c.Query("end-date"))
		endDate = endDate.Add((-9 + 24) * time.Hour).In(loc)
	}

	/*
		DB processing
	*/
	articles := []models.Article{}
	sql := db.DB

	// filter
	sql = sql.Where("published_at >= ?", startDate)
	sql = sql.Where("published_at <= ?", endDate)

	// sort
	sql, err = OrderArticles(sql, sort...)

	if err != nil {
		c.Status(http.StatusBadRequest)
		errorResp := ErrorResponse{Message: "wrong sort param"}
		encoder := json.NewEncoder(c.Writer)
		err = encoder.Encode(errorResp)
		if err != nil {
			panic(err)
		}
		return
	}

	// find
	sql.Find(&articles)

	/*
		select output field
	*/
	results := SelectArticles(&articles, fields...)

	// set header
	c.Writer.Header().Set("Link", "<page=3>; rel=\"next\", <page=1>; rel=\"prev\", <page=5>; rel=\"last\"")
	c.Status(http.StatusOK)

	// write response
	b, err := json.MarshalIndent(results, "", " ")
	if err != nil {
		panic(err)
	}
	_, err = c.Writer.WriteString(string(b))
	if err != nil {
		panic(err)
	}
}

// ArticleShow show article details
func ArticleShow(c *gin.Context) {
	// params
	// articlesID := c.Param("articles_id")
	articles := []models.Article{}
	db.DB.Find(&articles).Order("created_at desc")

	// set header
	c.Writer.Header().Set("Link", "<page=3>; rel=\"next\", <page=1>; rel=\"prev\", <page=5>; rel=\"last\"")
	c.Status(http.StatusOK)

	// write response
	encoder := json.NewEncoder(c.Writer)
	err := encoder.Encode(articles)
	if err != nil {
		c.Status(http.StatusBadRequest)
		errorResp := ErrorResponse{Message: "wrong sort param"}
		encoder := json.NewEncoder(c.Writer)
		err = encoder.Encode(errorResp)
		if err != nil {
			panic(err)
		}
	}
}

// SelectArticles extends gorm.DB.Select function
func SelectArticles(articles *[]models.Article, fields ...string) (results []map[string]interface{}) {
	for _, v := range *articles {
		if len(fields) > 1 || (len(fields) == 1 && fields[0] != "") {
			results = append(results, (&v).SelectFields(fields...))
		} else {
			results = append(results, (&v).SelectFields(
				"id",
				"title",
				"news_site_id",
				"published_at",
				"url",
			))
		}
	}

	return results
}

// OrderArticles extends gorm.DB.Order function
func OrderArticles(db *gorm.DB, sorts ...string) (*gorm.DB, error) {
	var dbRet = db
	var err error

	for i, v := range sorts {
		if v == "" {
			if i == 0 { // /articles /articles?sort= /articles?sort=,hoge
				dbRet = dbRet.Order("published_at desc")
			} else { // /articles?sort=hoge,
				return nil, err
			}
		} else if string(v[0]) == "-" { // sort=-hoge
			dbRet = dbRet.Order(v[1:] + " desc")
		} else { // sort=hoge
			dbRet = dbRet.Order(v + " asc")
		}
	}

	return dbRet, nil
}
<commit_msg>add title search<commit_after>package controllers

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	"github.com/takonews/takonews-api/app/models"
	"github.com/takonews/takonews-api/db"
)

// ArticleIndex show articles
// Available Query Parameters
// * sort
// * fields
// * filter
func ArticleIndex(c *gin.Context) {
	// parameters
	var sort []string
	var fields []string
	var startDate time.Time
	var endDate time.Time
	var title string

	sort = strings.Split(c.Query("sort"), ",")
	fields = strings.Split(c.Query("fields"), ",")
	loc, err := time.LoadLocation("Asia/Tokyo")
	if err != nil {
		panic(err)
	}
	now := time.Now().In(loc)
	if c.Query("start-date") == "" { // default: today:00:00:00
		startDate = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, loc)
	} else {
		startDate, err = time.Parse("2006-01-02", c.Query("start-date"))
		startDate = startDate.Add(-9 * time.Hour).In(loc)
	}
	if c.Query("end-date") == "" { // default: tomorrow:00:00:00
		endDate = time.Date(now.Year(), now.Month(), now.Day(), 24, 0, 0, 0, loc)
	} else {
		endDate, err = time.Parse("2006-01-02", c.Query("end-date"))
		endDate = endDate.Add((-9 + 24) * time.Hour).In(loc)
	}
	title = c.Query("title")

	/*
		DB processing
	*/
	articles := []models.Article{}
	sql := db.DB

	// filter
	sql = sql.Where("published_at >= ?", startDate)
	sql = sql.Where("published_at <= ?", endDate)
	sql = sql.Where("title LIKE ?", "%"+title+"%")

	// sort
	sql, err = OrderArticles(sql, sort...)

	if err != nil {
		c.Status(http.StatusBadRequest)
		errorResp := ErrorResponse{Message: "wrong sort param"}
		encoder := json.NewEncoder(c.Writer)
		err = encoder.Encode(errorResp)
		if err != nil {
			panic(err)
		}
		return
	}

	// find
	sql.Find(&articles)

	/*
		select output field
	*/
	results := SelectArticles(&articles, fields...)

	// set header
	c.Writer.Header().Set("Link", "<page=3>; rel=\"next\", <page=1>; rel=\"prev\", <page=5>; rel=\"last\"")
	c.Status(http.StatusOK)

	// write response
	b, err := json.MarshalIndent(results, "", " ")
	if err != nil {
		panic(err)
	}
	_, err = c.Writer.WriteString(string(b))
	if err != nil {
		panic(err)
	}
}

// ArticleShow show article details
func ArticleShow(c *gin.Context) {
	// params
	// articlesID := c.Param("articles_id")
	articles := []models.Article{}
	db.DB.Find(&articles).Order("created_at desc")

	// set header
	c.Writer.Header().Set("Link", "<page=3>; rel=\"next\", <page=1>; rel=\"prev\", <page=5>; rel=\"last\"")
	c.Status(http.StatusOK)

	// write response
	encoder := json.NewEncoder(c.Writer)
	err := encoder.Encode(articles)
	if err != nil {
		c.Status(http.StatusBadRequest)
		errorResp := ErrorResponse{Message: "wrong sort param"}
		encoder := json.NewEncoder(c.Writer)
		err = encoder.Encode(errorResp)
		if err != nil {
			panic(err)
		}
	}
}

// SelectArticles extends gorm.DB.Select function
func SelectArticles(articles *[]models.Article, fields ...string) (results []map[string]interface{}) {
	for _, v := range *articles {
		if len(fields) > 1 || (len(fields) == 1 && fields[0] != "") {
			results = append(results, (&v).SelectFields(fields...))
		} else {
			results = append(results, (&v).SelectFields(
				"id",
				"title",
				"news_site_id",
				"published_at",
				"url",
			))
		}
	}

	return results
}

// OrderArticles extends gorm.DB.Order function
func OrderArticles(db *gorm.DB, sorts ...string) (*gorm.DB, error) {
	var dbRet = db
	var err error

	for i, v := range sorts {
		if v == "" {
			if i == 0 { // /articles /articles?sort= /articles?sort=,hoge
				dbRet = dbRet.Order("published_at desc")
			} else { // /articles?sort=hoge,
				return nil, err
			}
		} else if string(v[0]) == "-" { // sort=-hoge
			dbRet = dbRet.Order(v[1:] + " desc")
		} else { // sort=hoge
			dbRet = dbRet.Order(v + " asc")
		}
	}

	return dbRet, nil
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/robfig/revel"
	"melange/app/models"
	"melange/app/routes"
	"strconv"
)

func GetUserId(session map[string]string) int {
	i, _ := strconv.Atoi(session["user"])
	return i
}

type App struct {
	GorpController
}

func (c App) Index() revel.Result {
	_, ok := c.Session["user"]
	if ok {
		return c.Redirect(routes.Dispatch.Dashboard())
	}
	return c.Render()
}

// This action controls Logout
func (c App) Logout() revel.Result {
	delete(c.Session, "user")
	c.Flash.Success("You have successfully logged out.")
	return c.Redirect(routes.App.Index())
}

// These Two Actions Control Displaying the Login View and Processing Logins
func (c App) Login() revel.Result {
	_, ok := c.Session["user"]
	if ok {
		return c.Redirect(routes.Dispatch.Dashboard())
	}
	return c.Render()
}

func (c App) ProcessLogin(username string, password string) revel.Result {
	user, err := models.AuthenticateUser(username, password, c.Txn)
	if err != nil {
		c.Flash.Error("Unable to authenticate user.")
		return c.Redirect(routes.App.Login())
	}

	// User is logged in, redirect to dashboard.
	c.Session["user"] = strconv.Itoa(user.UserId)
	c.Flash.Success("Welcome back, " + user.Name + ".")
	return c.Redirect(routes.Dispatch.Dashboard())
}

// These Two Actions Control Displaying the Registration View and Processing Registration
func (c App) Register() revel.Result {
	return c.Render()
}

func (c App) ProcessRegistration(
	name string,
	username string,
	password string,
	confirmPassword string) revel.Result {

	// Validations Time
	c.Validation.Required(name).Message("Full Name is required.")
	c.Validation.MaxSize(name, 100).Message("Full name cannot be longer than 100 characters.")
	c.Validation.Required(username).Message("Username is required.")
	c.Validation.MaxSize(username, 20).Message("Username cannot be longer than 20 characters")
	c.Validation.Required(password).Message("Password is required.")
	c.Validation.Required(confirmPassword).Message("Confirmation password is required.")
	c.Validation.Required(password == confirmPassword).Message("Passwords must match.")

	if c.Validation.HasErrors() {
		c.Validation.Keep()
		c.FlashParams()
		return c.Redirect(App.ProcessRegistration)
	}

	// Validation Passed, Let's Create the User
	u := models.CreateUser(username, password, name)
	u.Save(c.Txn)

	id, err := models.NewIdentityForUser(u)
	if err != nil {
		panic(err)
	}
	c.Txn.Insert(id)

	c.Session["user"] = strconv.Itoa(u.UserId)
	c.Flash.Success("Hi there, " + u.Name + " looks like it's your first time here. Why not try the tutorial?")

	return c.Redirect(routes.Dispatch.Dashboard())
}
<commit_msg>Tracker Registration on Sign Up<commit_after>package controllers

import (
	"github.com/robfig/revel"
	"melange/app/models"
	"melange/app/routes"
	"melange/mailserver"
	"strconv"
)

func GetUserId(session map[string]string) int {
	i, _ := strconv.Atoi(session["user"])
	return i
}

type App struct {
	GorpController
}

func (c App) Index() revel.Result {
	_, ok := c.Session["user"]
	if ok {
		return c.Redirect(routes.Dispatch.Dashboard())
	}
	return c.Render()
}

// This action controls Logout
func (c App) Logout() revel.Result {
	delete(c.Session, "user")
	c.Flash.Success("You have successfully logged out.")
	return c.Redirect(routes.App.Index())
}

// These Two Actions Control Displaying the Login View and Processing Logins
func (c App) Login() revel.Result {
	_, ok := c.Session["user"]
	if ok {
		return c.Redirect(routes.Dispatch.Dashboard())
	}
	return c.Render()
}

func (c App) ProcessLogin(username string, password string) revel.Result {
	user, err := models.AuthenticateUser(username, password, c.Txn)
	if err != nil {
		c.Flash.Error("Unable to authenticate user.")
		return c.Redirect(routes.App.Login())
	}

	// User is logged in, redirect to dashboard.
	c.Session["user"] = strconv.Itoa(user.UserId)
	c.Flash.Success("Welcome back, " + user.Name + ".")
	return c.Redirect(routes.Dispatch.Dashboard())
}

// These Two Actions Control Displaying the Registration View and Processing Registration
func (c App) Register() revel.Result {
	return c.Render()
}

func (c App) ProcessRegistration(
	name string,
	username string,
	password string,
	confirmPassword string) revel.Result {

	// Validations Time
	c.Validation.Required(name).Message("Full Name is required.")
	c.Validation.MaxSize(name, 100).Message("Full name cannot be longer than 100 characters.")
	c.Validation.Required(username).Message("Username is required.")
	c.Validation.MaxSize(username, 20).Message("Username cannot be longer than 20 characters")
	c.Validation.Required(password).Message("Password is required.")
	c.Validation.Required(confirmPassword).Message("Confirmation password is required.")
	c.Validation.Required(password == confirmPassword).Message("Passwords must match.")

	if c.Validation.HasErrors() {
		c.Validation.Keep()
		c.FlashParams()
		return c.Redirect(App.ProcessRegistration)
	}

	// Validation Passed, Let's Create the User
	u := models.CreateUser(username, password, name)
	u.Save(c.Txn)

	id, err := models.NewIdentityForUser(u, u.Username)
	if err != nil {
		panic(err)
	}
	c.Txn.Insert(id)

	mailserver.InitRouter()

	err = id.Register(mailserver.RegistrationRouter)
	if err != nil {
		panic(err)
	}

	c.Session["user"] = strconv.Itoa(u.UserId)
	c.Flash.Success("Hi there, " + u.Name + " looks like it's your first time here. Why not try the tutorial?")

	return c.Redirect(routes.Dispatch.Dashboard())
}
<|endoftext|>


<commit_before>package es

import (
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

// TODO change handling to e.g. Add command with implizit opening of channel
// to not expose the channel to the public
func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					indexer.indexBatch()
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	close(indexer.docsChannel)
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<commit_msg>log errors on batch indexing<commit_after>package es

import (
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

// TODO change handling to e.g. Add command with implizit opening of channel
// to not expose the channel to the public
func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR: %s", e)
					}
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	close(indexer.docsChannel)
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<|endoftext|>


<commit_before>package gopisysfs

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	// pollInterval is how long we should wait between naieve polling of files
	pollInterval = 20 * time.Millisecond
	// forever is about 100 years.
	forever = 100 * 365 * 24 * time.Hour
)

var rootpath = "/"

// setRoot is designed to be called by the test cases to exercise some hard-to-change things on an actual pi.
func setRoot(rt string) {
	rootpath = rt
}

// file gets a file path inside the /sys file system,
// but it can be hooked by the test cases to use a test filesystem instead of the real /sys
func file(paths ...string) string {
	path := filepath.Join(paths...)
	if !filepath.IsAbs(path) {
		path = filepath.Join(rootpath, path)
	}
	return path
}

// readFilePanic reads a file returning the contents as a string, and panics if it cannot be read
func readFilePanic(name string) string {
	data, err := readFile(name)
	if err != nil {
		log.Panicf("Unable to read file %v: %v", name, err)
	}
	return data
}

// awaitFileCreate establishes an asynchronous poll on a file location until it exists
// at which point the returned channel will return a nil on the channel. A non-nil indicates
// an error in the polling.
func awaitFileCreate(name string, timeout time.Duration) (<-chan error, error) {

	ret := make(chan error, 1)

	if checkFile(name) {
		ret <- nil
		return ret, nil
	}

	dir := filepath.Dir(name)
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		if err != nil {
			return nil, fmt.Errorf("Unable to poll for a file in a nonexistent folder %v: %v", dir, err)
		}
		return nil, fmt.Errorf("Unable to poll for a file in a non-folder %v: %v", dir, stat)
	}

	// set up notification and timeout
	tout := time.After(timeout)
	// intervals at every poll cycle
	interval := time.NewTicker(pollInterval).C
	// naieve polling system
	go func() {
		for {

			if checkFile(name) {
				// Found it!
				ret <- nil
				return
			}

			select {
			case <-tout:
				ret <- fmt.Errorf("Timed out waiting for %v after %v", name, timeout)
				return
			case <-interval:
				// ignore specific event, check actual file later
			}
		}
	}()

	return ret, nil

}

// awaitFileRemove establishes an asynchronous poll on a file location until it is removed
// at which point the returned channel will return a nil on the channel. A non-nil indicates
// an error in the polling.
func awaitFileRemove(name string, timeout time.Duration) (<-chan error, error) {

	ret := make(chan error, 1)

	// file is not there. Easy.
	if !checkFile(name) {
		ret <- nil
		return ret, nil
	}

	// set up notification and timeout
	tout := time.After(timeout)
	// intervals at every 20 milliseconds
	interval := time.NewTicker(pollInterval).C

	// naieve polling system
	go func() {
		for {

			if !checkFile(name) {
				// gone!
				ret <- nil
				return
			}

			select {
			case <-tout:
				ret <- fmt.Errorf("Timed out waiting for %v after %v", name, timeout)
				return
			case <-interval:
				// ignore specific event, check actual file later
			}
		}
	}()

	return ret, nil

}

func readStringFileAsInt(name string) (int, error) {
	data, err := readFile(name)
	if err != nil {
		return 0, err
	}
	val, err := strconv.Atoi(data)
	if err != nil {
		return 0, fmt.Errorf("Unable to convert value %v from %v to an int: %v", data, name, err)
	}
	return val, nil
}

//readFile reads the file and returns the contents as a string (trimmed)
func readFile(name string) (string, error) {
	data, err := ioutil.ReadFile(name)
	if err != nil {
		return "", err
	}
	str := string(data)
	str = strings.TrimSpace(str)
	return str, nil
}

// readBuffer reads a file in to a byte buffer
func readBytes(name string) ([]byte, error) {
	return ioutil.ReadFile(name)
}

// writeBuffer writes a buffer in to a file
func writeBuffer(name string, data []byte) error {
	info("Writing to %v: %v\n", name, data)
	return ioutil.WriteFile(name, data, 0444)
}

// writeFile will overwrite the specified file with the given string content
func writeFile(name, text string) error {
	info("Writing to %v: %v\n", name, text)
	data := []byte(text)
	return ioutil.WriteFile(name, data, 0444)
}

func checkWritable(name string) bool {
	if stat, err := os.Stat(name); err == nil {
		// exists, but is it writable?
		mode := os.O_RDWR
		desc := "writable file"
		if stat.IsDir() {
			mode = os.O_RDONLY
			desc = "readable folder"
		}
		// Note, you can open directories as well
		file, err := os.OpenFile(name, mode, 0)
		if err != nil {
			info("Existing file %v but it is not a %v: %v\n", name, desc, err)
			return false
		}
		file.Close()
		// already exists
		return true
	}
	return false
}

// checkFile retuns true if the specified file exists
func checkFile(name string) bool {
	if _, err := os.Stat(name); err == nil {
		// exists, but is it writable?
		// already exists
		return true
	}
	return false
}
<commit_msg>Remove dead code and overlogging<commit_after>package gopisysfs

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	// pollInterval is how long we should wait between naieve polling of files
	pollInterval = 20 * time.Millisecond
	// forever is about 100 years.
	forever = 100 * 365 * 24 * time.Hour
)

var rootpath = "/"

// setRoot is designed to be called by the test cases to exercise some hard-to-change things on an actual pi.
func setRoot(rt string) {
	rootpath = rt
}

// file gets a file path inside the /sys file system,
// but it can be hooked by the test cases to use a test filesystem instead of the real /sys
func file(paths ...string) string {
	path := filepath.Join(paths...)
	if !filepath.IsAbs(path) {
		path = filepath.Join(rootpath, path)
	}
	return path
}

// readFilePanic reads a file returning the contents as a string, and panics if it cannot be read
func readFilePanic(name string) string {
	data, err := readFile(name)
	if err != nil {
		log.Panicf("Unable to read file %v: %v", name, err)
	}
	return data
}

// awaitFileCreate establishes an asynchronous poll on a file location until it exists
// at which point the returned channel will return a nil on the channel. A non-nil indicates
// an error in the polling.
func awaitFileCreate(name string, timeout time.Duration) (<-chan error, error) {

	ret := make(chan error, 1)

	if checkFile(name) {
		ret <- nil
		return ret, nil
	}

	dir := filepath.Dir(name)
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		if err != nil {
			return nil, fmt.Errorf("Unable to poll for a file in a nonexistent folder %v: %v", dir, err)
		}
		return nil, fmt.Errorf("Unable to poll for a file in a non-folder %v: %v", dir, stat)
	}

	// set up notification and timeout
	tout := time.After(timeout)
	// intervals at every poll cycle
	interval := time.NewTicker(pollInterval).C
	// naieve polling system
	go func() {
		for {

			if checkFile(name) {
				// Found it!
				ret <- nil
				return
			}

			select {
			case <-tout:
				ret <- fmt.Errorf("Timed out waiting for %v after %v", name, timeout)
				return
			case <-interval:
				// ignore specific event, check actual file later
			}
		}
	}()

	return ret, nil

}

// awaitFileRemove establishes an asynchronous poll on a file location until it is removed
// at which point the returned channel will return a nil on the channel. A non-nil indicates
// an error in the polling.
func awaitFileRemove(name string, timeout time.Duration) (<-chan error, error) {

	ret := make(chan error, 1)

	// file is not there. Easy.
	if !checkFile(name) {
		ret <- nil
		return ret, nil
	}

	// set up notification and timeout
	tout := time.After(timeout)
	// intervals at every 20 milliseconds
	interval := time.NewTicker(pollInterval).C

	// naieve polling system
	go func() {
		for {

			if !checkFile(name) {
				// gone!
				ret <- nil
				return
			}

			select {
			case <-tout:
				ret <- fmt.Errorf("Timed out waiting for %v after %v", name, timeout)
				return
			case <-interval:
				// ignore specific event, check actual file later
			}
		}
	}()

	return ret, nil

}

func readStringFileAsInt(name string) (int, error) {
	data, err := readFile(name)
	if err != nil {
		return 0, err
	}
	val, err := strconv.Atoi(data)
	if err != nil {
		return 0, fmt.Errorf("Unable to convert value %v from %v to an int: %v", data, name, err)
	}
	return val, nil
}

//readFile reads the file and returns the contents as a string (trimmed)
func readFile(name string) (string, error) {
	data, err := ioutil.ReadFile(name)
	if err != nil {
		return "", err
	}
	str := string(data)
	str = strings.TrimSpace(str)
	return str, nil
}

// readBuffer reads a file in to a byte buffer
func readBytes(name string) ([]byte, error) {
	return ioutil.ReadFile(name)
}

// writeBuffer writes a buffer in to a file
func writeBuffer(name string, data []byte) error {
	//info("Writing to %v: %v\n", name, data)
	return ioutil.WriteFile(name, data, 0444)
}

// writeFile will overwrite the specified file with the given string content
func writeFile(name, text string) error {
	//info("Writing to %v: %v\n", name, text)
	data := []byte(text)
	return ioutil.WriteFile(name, data, 0444)
}

// checkFile retuns true if the specified file exists
func checkFile(name string) bool {
	if _, err := os.Stat(name); err == nil {
		// exists, but is it writable?
		// already exists
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// tail is responsible for tailing a log file and extracting new log lines to
// be passed into the virtual machines.

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

package tailer

import (
	"expvar"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/jaqx0r/afero"
)

var (
	log_count     = expvar.NewInt("log_count")
	log_errors    = expvar.NewMap("log_errors_total")
	log_rotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched      map[string]struct{}   // Names of logs being watched.
	watched_lock sync.RWMutex          // protects `watched'
	lines        chan<- string         // Logfile lines being emitted.
	files        map[string]afero.File // File handles for each pathname.
	files_lock   sync.Mutex            // protects `files'
	partials     map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// New returns a new Tailer.
func New(lines chan<- string, w watcher.Watcher, fs afero.Fs) *Tailer {
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t
}

func (t *Tailer) addWatched(path string) {
	t.watched_lock.Lock()
	defer t.watched_lock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watched_lock.RLock()
	defer t.watched_lock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		log_count.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	for {
		b := make([]byte, 4096)
		f, ok := t.files[pathname]
		if !ok {
			glog.Infof("No file found for %q", pathname)
			return
		}
		n, err := f.Read(b)
		if err != nil {
			if err == io.EOF && n == 0 {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return
		} else {
			for i, width := 0, 0; i < len(b) && i < n; i += width {
				var rune rune
				rune, width = utf8.DecodeRune(b[i:])
				switch {
				case rune != '\n':
					t.partials[pathname] += string(rune)
				default:
					// send off line for processing
					t.lines <- t.partials[pathname]
					// reset accumulator
					t.partials[pathname] = ""
				}
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.files_lock.Lock()
	fd, ok := t.files[pathname]
	t.files_lock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			log_rotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seek_to_start bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	fd, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		log_errors.Add(pathname, 1)
		return
	}
	t.files_lock.Lock()
	t.files[pathname] = fd

	if seek_to_start {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.files_lock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down Tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>Address golint comments in package tailer.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// tail is responsible for tailing a log file and extracting new log lines to
// be passed into the virtual machines.

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

package tailer

import (
	"expvar"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/jaqx0r/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// New returns a new Tailer.
func New(lines chan<- string, w watcher.Watcher, fs afero.Fs) *Tailer {
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t
}

func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	for {
		b := make([]byte, 4096)
		f, ok := t.files[pathname]
		if !ok {
			glog.Infof("No file found for %q", pathname)
			return
		}
		n, err := f.Read(b)
		if err != nil {
			if err == io.EOF && n == 0 {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return // TODO(jaq): handle this path better
		}
		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				t.partials[pathname] += string(rune)
			default:
				// send off line for processing
				t.lines <- t.partials[pathname]
				// reset accumulator
				t.partials[pathname] = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seekStart bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	fd, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		return
	}
	t.filesLock.Lock()
	t.files[pathname] = fd

	if seekStart {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.filesLock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down Tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<|endoftext|>


<commit_before>package constants

// THIS FILE IS GENERATED AUTOMATICALLY, NO TOUCHING!!!!!

const (
	BOSHURL         = "https://s3.amazonaws.com/bbl-precompiled-bosh-releases/release-bosh-260.4-on-ubuntu-trusty-stemcell-3312.15.tgz"
	BOSHSHA1        = "1ddc797d157d4c83b37d702019dc764d8f8715ce"
	BOSHAWSCPIURL   = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=62"
	BOSHAWSCPISHA1  = "f36967927ceae09e5663a41fdda199edfe649dc6"
	AWSStemcellURL  = "https://s3.amazonaws.com/bosh-aws-light-stemcells/light-bosh-stemcell-3312.15-aws-xen-hvm-ubuntu-trusty-go_agent.tgz"
	AWSStemcellSHA1 = "f5e78f61b8026377300db544e9d3ca0b89516af3"
	BOSHGCPCPIURL   = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-google-cpi-release?v=25.6.2"
	BOSHGCPCPISHA1  = "b4865397d867655fdcc112bc5a7f9a5025cdf311"
	GCPStemcellURL  = "https://s3.amazonaws.com/bosh-gce-light-stemcells/light-bosh-stemcell-3312.15-google-kvm-ubuntu-trusty-go_agent.tgz"
	GCPStemcellSHA1 = "3ac3ee83750f75bd74e8d3e3ad97808db23c30ba"
)
<commit_msg>Update constants<commit_after>package constants

// THIS FILE IS GENERATED AUTOMATICALLY, NO TOUCHING!!!!!

const (
	BOSHURL         = "https://s3.amazonaws.com/bbl-precompiled-bosh-releases/release-bosh-260.5-on-ubuntu-trusty-stemcell-3312.15.tgz"
	BOSHSHA1        = "2d6cdb684f1fc536d71ce076b90bc78a1ba190a4"
	BOSHAWSCPIURL   = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=62"
	BOSHAWSCPISHA1  = "f36967927ceae09e5663a41fdda199edfe649dc6"
	AWSStemcellURL  = "https://s3.amazonaws.com/bosh-aws-light-stemcells/light-bosh-stemcell-3312.15-aws-xen-hvm-ubuntu-trusty-go_agent.tgz"
	AWSStemcellSHA1 = "f5e78f61b8026377300db544e9d3ca0b89516af3"
	BOSHGCPCPIURL   = "https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-google-cpi-release?v=25.6.2"
	BOSHGCPCPISHA1  = "b4865397d867655fdcc112bc5a7f9a5025cdf311"
	GCPStemcellURL  = "https://s3.amazonaws.com/bosh-gce-light-stemcells/light-bosh-stemcell-3312.15-google-kvm-ubuntu-trusty-go_agent.tgz"
	GCPStemcellSHA1 = "3ac3ee83750f75bd74e8d3e3ad97808db23c30ba"
)
<|endoftext|>


<commit_before>package admin

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/qor"
	"github.com/qor/qor/media_library"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Meta struct {
	base          *Resource
	Name          string
	DBName        string
	Alias         string
	Label         string
	Type          string
	Valuer        func(interface{}, *qor.Context) interface{}
	Setter        func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context)
	Metas         []resource.Metaor
	Resource      resource.Resourcer
	Collection    interface{}
	GetCollection func(interface{}, *qor.Context) [][]string
	Permission    *roles.Permission
}

func (meta *Meta) GetName() string {
	return meta.Name
}

func (meta *Meta) GetFieldName() string {
	return meta.Alias
}

func (meta *Meta) GetMetas() []resource.Metaor {
	if len(meta.Metas) > 0 {
		return meta.Metas
	} else if meta.Resource == nil {
		return []resource.Metaor{}
	} else {
		return meta.Resource.GetMetas([]string{})
	}
}

func (meta *Meta) GetResource() resource.Resourcer {
	return meta.Resource
}

func (meta *Meta) GetValuer() func(interface{}, *qor.Context) interface{} {
	return meta.Valuer
}

func (meta *Meta) GetSetter() func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
	return meta.Setter
}

func (meta *Meta) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if meta.Permission == nil {
		return true
	}
	return meta.Permission.HasPermission(mode, context.Roles...)
}

func getField(fields map[string]*gorm.Field, name string) (*gorm.Field, bool) {
	for _, field := range fields {
		if field.Name == name || field.DBName == name {
			return field, true
		}
	}
	return nil, false
}

func (meta *Meta) updateMeta() {
	if meta.Name == "" {
		utils.ExitWithMsg("Meta should have name: %v", reflect.ValueOf(meta).Type())
	} else if meta.Alias == "" {
		meta.Alias = meta.Name
	}

	if meta.Label == "" {
		meta.Label = utils.HumanizeString(meta.Name)
	}

	var (
		scope       = &gorm.Scope{Value: meta.base.Value}
		nestedField = strings.Contains(meta.Alias, ".")
		field       *gorm.Field
		hasColumn   bool
	)

	if nestedField {
		subModel, name := utils.ParseNestedField(reflect.ValueOf(meta.base.Value), meta.Alias)
		subScope := &gorm.Scope{Value: subModel.Interface()}
		field, hasColumn = getField(subScope.Fields(), name)
	} else {
		if field, hasColumn = getField(scope.Fields(), meta.Alias); hasColumn {
			meta.Alias = field.Name
			if field.IsNormal {
				meta.DBName = field.DBName
			}
		}
	}

	var fieldType reflect.Type
	if hasColumn {
		fieldType = field.Field.Type()
		for fieldType.Kind() == reflect.Ptr {
			fieldType = fieldType.Elem()
		}
	}

	// Set Meta Type
	if meta.Type == "" && hasColumn {
		if relationship := field.Relationship; relationship != nil {
			if relationship.Kind == "has_one" {
				meta.Type = "single_edit"
			} else if relationship.Kind == "has_many" {
				meta.Type = "collection_edit"
			} else if relationship.Kind == "belongs_to" {
				meta.Type = "select_one"
			} else if relationship.Kind == "many_to_many" {
				meta.Type = "select_many"
			}
		} else {
			switch fieldType.Kind().String() {
			case "string":
				if size, ok := utils.ParseTagOption(field.Tag.Get("sql"))["SIZE"]; ok {
					if i, _ := strconv.Atoi(size); i > 255 {
						meta.Type = "text"
					} else {
						meta.Type = "string"
					}
				} else if text, ok := utils.ParseTagOption(field.Tag.Get("sql"))["TYPE"]; ok && text == "text" {
					meta.Type = "text"
				} else {
					meta.Type = "string"
				}
			case "bool":
				meta.Type = "checkbox"
			default:
				if regexp.MustCompile(`^(.*)?(u)?(int)(\d+)?`).MatchString(fieldType.Kind().String()) {
					meta.Type = "number"
				} else if regexp.MustCompile(`^(.*)?(float)(\d+)?`).MatchString(fieldType.Kind().String()) {
					meta.Type = "float"
				} else if _, ok := reflect.New(fieldType).Interface().(*time.Time); ok {
					meta.Type = "datetime"
				} else if _, ok := reflect.New(fieldType).Interface().(media_library.MediaLibrary); ok {
					meta.Type = "file"
				}
			}
		}
	}

	// Set Meta Resource
	if meta.Resource == nil {
		if hasColumn && (field.Relationship != nil) {
			var result interface{}
			if fieldType.Kind().String() == "struct" {
				result = reflect.New(field.Field.Type()).Interface()
			} else if fieldType.Kind().String() == "slice" {
				refelectType := field.Field.Type().Elem()
				for refelectType.Kind() == reflect.Ptr {
					refelectType = refelectType.Elem()
				}
				result = reflect.New(refelectType).Interface()
			}

			res := meta.base.GetAdmin().NewResource(result)
			res.compile()
			meta.Resource = res
		}
	}

	// Set Meta Valuer
	if meta.Valuer == nil {
		if hasColumn {
			meta.Valuer = func(value interface{}, context *qor.Context) interface{} {
				scope := context.GetDB().NewScope(value)
				alias := meta.Alias
				if nestedField {
					fields := strings.Split(alias, ".")
					alias = fields[len(fields)-1]
				}

				if f, ok := scope.FieldByName(alias); ok {
					if field.Relationship != nil {
						if f.Field.CanAddr() && !scope.PrimaryKeyZero() {
							context.GetDB().Model(value).Related(f.Field.Addr().Interface(), meta.Alias)
						}
					}
					if f.Field.CanAddr() {
						return f.Field.Addr().Interface()
					} else {
						return f.Field.Interface()
					}
				}

				return ""
			}
		} else {
			utils.ExitWithMsg("Unsupported meta name %v for resource %v", meta.Name, reflect.TypeOf(meta.base.Value))
		}
	}

	scopeField, _ := scope.FieldByName(meta.Alias)

	// Set Meta Collection
	if meta.Collection != nil {
		if maps, ok := meta.Collection.([]string); ok {
			meta.GetCollection = func(interface{}, *qor.Context) (results [][]string) {
				for _, value := range maps {
					results = append(results, []string{value, value})
				}
				return
			}
		} else if maps, ok := meta.Collection.([][]string); ok {
			meta.GetCollection = func(interface{}, *qor.Context) [][]string {
				return maps
			}
		} else if f, ok := meta.Collection.(func(interface{}, *qor.Context) [][]string); ok {
			meta.GetCollection = f
		} else {
			utils.ExitWithMsg("Unsupported Collection format for meta %v of resource %v", meta.Name, reflect.TypeOf(meta.base.Value))
		}
	} else if meta.Type == "select_one" || meta.Type == "select_many" {
		if scopeField.Relationship != nil {
			fieldType := scopeField.StructField.Struct.Type
			if fieldType.Kind() == reflect.Slice {
				fieldType = fieldType.Elem()
			}

			meta.GetCollection = func(value interface{}, context *qor.Context) (results [][]string) {
				values := reflect.New(reflect.SliceOf(fieldType)).Interface()
				context.GetDB().Find(values)
				reflectValues := reflect.Indirect(reflect.ValueOf(values))
				for i := 0; i < reflectValues.Len(); i++ {
					scope := scope.New(reflectValues.Index(i).Interface())
					primaryKey := fmt.Sprintf("%v", scope.PrimaryKeyValue())
					results = append(results, []string{primaryKey, utils.Stringify(reflectValues.Index(i).Interface())})
				}
				return
			}
		} else {
			utils.ExitWithMsg("%v meta type %v needs Collection", meta.Name, meta.Type)
		}
	}

	if meta.Setter == nil && hasColumn {
		if relationship := field.Relationship; relationship != nil {
			if meta.Type == "select_one" || meta.Type == "select_many" {
				meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
					scope := &gorm.Scope{Value: resource}
					field := reflect.Indirect(reflect.ValueOf(resource)).FieldByName(meta.Alias)

					if field.Kind() == reflect.Ptr && field.IsNil() {
						field.Set(utils.NewValue(field.Type()).Elem())
					}

					for field.Kind() == reflect.Ptr {
						field = field.Elem()
					}

					if primaryKeys := utils.ToArray(metaValue.Value); len(primaryKeys) > 0 {
						context.GetDB().Where(primaryKeys).Find(field.Addr().Interface())
					}

					// Replace many 2 many relations
					if relationship.Kind == "many_to_many" {
						if !scope.PrimaryKeyZero() {
							context.GetDB().Model(resource).Association(meta.Alias).Replace(field.Interface())
						}
					}
				}
			}
		} else {
			meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
				if metaValue == nil {
					return
				}

				value := metaValue.Value
				alias := meta.Alias
				if nestedField {
					fields := strings.Split(alias, ".")
					alias = fields[len(fields)-1]
				}

				field := reflect.Indirect(reflect.ValueOf(resource)).FieldByName(alias)
				if field.Kind() == reflect.Ptr && field.IsNil() {
					field.Set(utils.NewValue(field.Type()).Elem())
				}

				for field.Kind() == reflect.Ptr {
					field = field.Elem()
				}

				if field.IsValid() && field.CanAddr() {
					switch field.Kind() {
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
						field.SetInt(utils.ToInt(value))
					case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						field.SetUint(utils.ToUint(value))
					case reflect.Float32, reflect.Float64:
						field.SetFloat(utils.ToFloat(value))
					case reflect.Bool:
						// TODO: add test
						if utils.ToString(value) == "true" {
							field.SetBool(true)
						} else {
							field.SetBool(false)
						}
					default:
						if scanner, ok := field.Addr().Interface().(sql.Scanner); ok {
							if scanner.Scan(value) != nil {
								scanner.Scan(utils.ToString(value))
							}
						} else if reflect.TypeOf("").ConvertibleTo(field.Type()) {
							field.Set(reflect.ValueOf(utils.ToString(value)).Convert(field.Type()))
						} else if reflect.TypeOf([]string{}).ConvertibleTo(field.Type()) {
							field.Set(reflect.ValueOf(utils.ToArray(value)).Convert(field.Type()))
						} else if rvalue := reflect.ValueOf(value); reflect.TypeOf(rvalue.Type()).ConvertibleTo(field.Type()) {
							field.Set(rvalue.Convert(field.Type()))
						} else if _, ok := field.Addr().Interface().(*time.Time); ok {
							if str := utils.ToString(value); str != "" {
								if newTime, err := now.Parse(str); err == nil {
									field.Set(reflect.ValueOf(newTime))
								}
							}
						} else {
							var buf = bytes.NewBufferString("")
							json.NewEncoder(buf).Encode(value)
							if err := json.NewDecoder(strings.NewReader(buf.String())).Decode(field.Addr().Interface()); err != nil {
								utils.ExitWithMsg("Can't set value %v to %v [meta %v]", reflect.ValueOf(value).Type(), field.Type(), meta)
							}
						}
					}
				}
			}
		}
	}

	if nestedField {
		oldvalue := meta.Valuer
		meta.Valuer = func(value interface{}, context *qor.Context) interface{} {
			return oldvalue(utils.GetNestedModel(value, meta.Alias, context), context)
		}
		oldSetter := meta.Setter
		meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
			oldSetter(utils.GetNestedModel(resource, meta.Alias, context), metaValue, context)
		}
	}
}
<commit_msg>Use StructFields of Scope for Metas<commit_after>package admin

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/qor"
	"github.com/qor/qor/media_library"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Meta struct {
	base          *Resource
	Name          string
	DBName        string
	Alias         string
	Label         string
	Type          string
	Valuer        func(interface{}, *qor.Context) interface{}
	Setter        func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context)
	Metas         []resource.Metaor
	Resource      resource.Resourcer
	Collection    interface{}
	GetCollection func(interface{}, *qor.Context) [][]string
	Permission    *roles.Permission
}

func (meta *Meta) GetName() string {
	return meta.Name
}

func (meta *Meta) GetFieldName() string {
	return meta.Alias
}

func (meta *Meta) GetMetas() []resource.Metaor {
	if len(meta.Metas) > 0 {
		return meta.Metas
	} else if meta.Resource == nil {
		return []resource.Metaor{}
	} else {
		return meta.Resource.GetMetas([]string{})
	}
}

func (meta *Meta) GetResource() resource.Resourcer {
	return meta.Resource
}

func (meta *Meta) GetValuer() func(interface{}, *qor.Context) interface{} {
	return meta.Valuer
}

func (meta *Meta) GetSetter() func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
	return meta.Setter
}

func (meta *Meta) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if meta.Permission == nil {
		return true
	}
	return meta.Permission.HasPermission(mode, context.Roles...)
}

func getField(fields []*gorm.StructField, name string) (*gorm.StructField, bool) {
	for _, field := range fields {
		if field.Name == name || field.DBName == name {
			return field, true
		}
	}
	return nil, false
}

func (meta *Meta) updateMeta() {
	if meta.Name == "" {
		utils.ExitWithMsg("Meta should have name: %v", reflect.ValueOf(meta).Type())
	} else if meta.Alias == "" {
		meta.Alias = meta.Name
	}

	if meta.Label == "" {
		meta.Label = utils.HumanizeString(meta.Name)
	}

	var (
		scope       = &gorm.Scope{Value: meta.base.Value}
		nestedField = strings.Contains(meta.Alias, ".")
		field       *gorm.StructField
		hasColumn   bool
	)

	if nestedField {
		subModel, name := utils.ParseNestedField(reflect.ValueOf(meta.base.Value), meta.Alias)
		subScope := &gorm.Scope{Value: subModel.Interface()}
		field, hasColumn = getField(subScope.GetStructFields(), name)
	} else {
		if field, hasColumn = getField(scope.GetStructFields(), meta.Alias); hasColumn {
			meta.Alias = field.Name
			if field.IsNormal {
				meta.DBName = field.DBName
			}
		}
	}

	var fieldType reflect.Type
	if hasColumn {
		fieldType = field.Struct.Type
		for fieldType.Kind() == reflect.Ptr {
			fieldType = fieldType.Elem()
		}
	}

	// Set Meta Type
	if meta.Type == "" && hasColumn {
		if relationship := field.Relationship; relationship != nil {
			if relationship.Kind == "has_one" {
				meta.Type = "single_edit"
			} else if relationship.Kind == "has_many" {
				meta.Type = "collection_edit"
			} else if relationship.Kind == "belongs_to" {
				meta.Type = "select_one"
			} else if relationship.Kind == "many_to_many" {
				meta.Type = "select_many"
			}
		} else {
			switch fieldType.Kind().String() {
			case "string":
				if size, ok := utils.ParseTagOption(field.Tag.Get("sql"))["SIZE"]; ok {
					if i, _ := strconv.Atoi(size); i > 255 {
						meta.Type = "text"
					} else {
						meta.Type = "string"
					}
				} else if text, ok := utils.ParseTagOption(field.Tag.Get("sql"))["TYPE"]; ok && text == "text" {
					meta.Type = "text"
				} else {
					meta.Type = "string"
				}
			case "bool":
				meta.Type = "checkbox"
			default:
				if regexp.MustCompile(`^(.*)?(u)?(int)(\d+)?`).MatchString(fieldType.Kind().String()) {
					meta.Type = "number"
				} else if regexp.MustCompile(`^(.*)?(float)(\d+)?`).MatchString(fieldType.Kind().String()) {
					meta.Type = "float"
				} else if _, ok := reflect.New(fieldType).Interface().(*time.Time); ok {
					meta.Type = "datetime"
				} else if _, ok := reflect.New(fieldType).Interface().(media_library.MediaLibrary); ok {
					meta.Type = "file"
				}
			}
		}
	}

	// Set Meta Resource
	if meta.Resource == nil {
		if hasColumn && (field.Relationship != nil) {
			var result interface{}
			if fieldType.Kind().String() == "struct" {
				result = reflect.New(fieldType).Interface()
			} else if fieldType.Kind().String() == "slice" {
				refelectType := fieldType.Elem()
				for refelectType.Kind() == reflect.Ptr {
					refelectType = refelectType.Elem()
				}
				result = reflect.New(refelectType).Interface()
			}

			res := meta.base.GetAdmin().NewResource(result)
			res.compile()
			meta.Resource = res
		}
	}

	// Set Meta Valuer
	if meta.Valuer == nil {
		if hasColumn {
			meta.Valuer = func(value interface{}, context *qor.Context) interface{} {
				scope := context.GetDB().NewScope(value)
				alias := meta.Alias
				if nestedField {
					fields := strings.Split(alias, ".")
					alias = fields[len(fields)-1]
				}

				if f, ok := scope.FieldByName(alias); ok {
					if field.Relationship != nil {
						if f.Field.CanAddr() && !scope.PrimaryKeyZero() {
							context.GetDB().Model(value).Related(f.Field.Addr().Interface(), meta.Alias)
						}
					}
					if f.Field.CanAddr() {
						return f.Field.Addr().Interface()
					} else {
						return f.Field.Interface()
					}
				}

				return ""
			}
		} else {
			utils.ExitWithMsg("Unsupported meta name %v for resource %v", meta.Name, reflect.TypeOf(meta.base.Value))
		}
	}

	scopeField, _ := scope.FieldByName(meta.Alias)

	// Set Meta Collection
	if meta.Collection != nil {
		if maps, ok := meta.Collection.([]string); ok {
			meta.GetCollection = func(interface{}, *qor.Context) (results [][]string) {
				for _, value := range maps {
					results = append(results, []string{value, value})
				}
				return
			}
		} else if maps, ok := meta.Collection.([][]string); ok {
			meta.GetCollection = func(interface{}, *qor.Context) [][]string {
				return maps
			}
		} else if f, ok := meta.Collection.(func(interface{}, *qor.Context) [][]string); ok {
			meta.GetCollection = f
		} else {
			utils.ExitWithMsg("Unsupported Collection format for meta %v of resource %v", meta.Name, reflect.TypeOf(meta.base.Value))
		}
	} else if meta.Type == "select_one" || meta.Type == "select_many" {
		if scopeField.Relationship != nil {
			fieldType := scopeField.StructField.Struct.Type
			if fieldType.Kind() == reflect.Slice {
				fieldType = fieldType.Elem()
			}

			meta.GetCollection = func(value interface{}, context *qor.Context) (results [][]string) {
				values := reflect.New(reflect.SliceOf(fieldType)).Interface()
				context.GetDB().Find(values)
				reflectValues := reflect.Indirect(reflect.ValueOf(values))
				for i := 0; i < reflectValues.Len(); i++ {
					scope := scope.New(reflectValues.Index(i).Interface())
					primaryKey := fmt.Sprintf("%v", scope.PrimaryKeyValue())
					results = append(results, []string{primaryKey, utils.Stringify(reflectValues.Index(i).Interface())})
				}
				return
			}
		} else {
			utils.ExitWithMsg("%v meta type %v needs Collection", meta.Name, meta.Type)
		}
	}

	if meta.Setter == nil && hasColumn {
		if relationship := field.Relationship; relationship != nil {
			if meta.Type == "select_one" || meta.Type == "select_many" {
				meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
					scope := &gorm.Scope{Value: resource}
					field := reflect.Indirect(reflect.ValueOf(resource)).FieldByName(meta.Alias)

					if field.Kind() == reflect.Ptr && field.IsNil() {
						field.Set(utils.NewValue(field.Type()).Elem())
					}

					for field.Kind() == reflect.Ptr {
						field = field.Elem()
					}

					if primaryKeys := utils.ToArray(metaValue.Value); len(primaryKeys) > 0 {
						context.GetDB().Where(primaryKeys).Find(field.Addr().Interface())
					}

					// Replace many 2 many relations
					if relationship.Kind == "many_to_many" {
						if !scope.PrimaryKeyZero() {
							context.GetDB().Model(resource).Association(meta.Alias).Replace(field.Interface())
						}
					}
				}
			}
		} else {
			meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
				if metaValue == nil {
					return
				}

				value := metaValue.Value
				alias := meta.Alias
				if nestedField {
					fields := strings.Split(alias, ".")
					alias = fields[len(fields)-1]
				}

				field := reflect.Indirect(reflect.ValueOf(resource)).FieldByName(alias)
				if field.Kind() == reflect.Ptr && field.IsNil() {
					field.Set(utils.NewValue(field.Type()).Elem())
				}

				for field.Kind() == reflect.Ptr {
					field = field.Elem()
				}

				if field.IsValid() && field.CanAddr() {
					switch field.Kind() {
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
						field.SetInt(utils.ToInt(value))
					case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						field.SetUint(utils.ToUint(value))
					case reflect.Float32, reflect.Float64:
						field.SetFloat(utils.ToFloat(value))
					case reflect.Bool:
						// TODO: add test
						if utils.ToString(value) == "true" {
							field.SetBool(true)
						} else {
							field.SetBool(false)
						}
					default:
						if scanner, ok := field.Addr().Interface().(sql.Scanner); ok {
							if scanner.Scan(value) != nil {
								scanner.Scan(utils.ToString(value))
							}
						} else if reflect.TypeOf("").ConvertibleTo(field.Type()) {
							field.Set(reflect.ValueOf(utils.ToString(value)).Convert(field.Type()))
						} else if reflect.TypeOf([]string{}).ConvertibleTo(field.Type()) {
							field.Set(reflect.ValueOf(utils.ToArray(value)).Convert(field.Type()))
						} else if rvalue := reflect.ValueOf(value); reflect.TypeOf(rvalue.Type()).ConvertibleTo(field.Type()) {
							field.Set(rvalue.Convert(field.Type()))
						} else if _, ok := field.Addr().Interface().(*time.Time); ok {
							if str := utils.ToString(value); str != "" {
								if newTime, err := now.Parse(str); err == nil {
									field.Set(reflect.ValueOf(newTime))
								}
							}
						} else {
							var buf = bytes.NewBufferString("")
							json.NewEncoder(buf).Encode(value)
							if err := json.NewDecoder(strings.NewReader(buf.String())).Decode(field.Addr().Interface()); err != nil {
								utils.ExitWithMsg("Can't set value %v to %v [meta %v]", reflect.ValueOf(value).Type(), field.Type(), meta)
							}
						}
					}
				}
			}
		}
	}

	if nestedField {
		oldvalue := meta.Valuer
		meta.Valuer = func(value interface{}, context *qor.Context) interface{} {
			return oldvalue(utils.GetNestedModel(value, meta.Alias, context), context)
		}
		oldSetter := meta.Setter
		meta.Setter = func(resource interface{}, metaValue *resource.MetaValue, context *qor.Context) {
			oldSetter(utils.GetNestedModel(resource, meta.Alias, context), metaValue, context)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestGetAllRecords(t *testing.T) {
	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()
	m := getBoneRouter(*dbClient)

	req, err := http.NewRequest("GET", "/records", nil)
	expect(t, err, nil)

	//The response recorder used to record HTTP responses
	respRec := httptest.NewRecorder()

	m.ServeHTTP(respRec, req)

	expect(t, respRec.Code, http.StatusOK)

	body, err := ioutil.ReadAll(respRec.Body)

	rr := recordedRequests{}
	err = json.Unmarshal(body, &rr)

	expect(t, len(rr.Data), 0)
}

func TestGetAllRecordsWRecords(t *testing.T) {
	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	// inserting some payloads
	for i := 0; i < 5; i++ {
		req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com/q=%d", i), nil)
		expect(t, err, nil)
		dbClient.captureRequest(req)
	}
	// performing query
	m := getBoneRouter(*dbClient)

	req, err := http.NewRequest("GET", "/records", nil)
	expect(t, err, nil)

	//The response recorder used to record HTTP responses
	respRec := httptest.NewRecorder()

	m.ServeHTTP(respRec, req)

	expect(t, respRec.Code, http.StatusOK)

	body, err := ioutil.ReadAll(respRec.Body)

	rr := recordedRequests{}
	err = json.Unmarshal(body, &rr)

	expect(t, len(rr.Data), 5)
}
<commit_msg>testing admin /count handler<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestGetAllRecords(t *testing.T) {
	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()
	m := getBoneRouter(*dbClient)

	req, err := http.NewRequest("GET", "/records", nil)
	expect(t, err, nil)

	//The response recorder used to record HTTP responses
	respRec := httptest.NewRecorder()

	m.ServeHTTP(respRec, req)

	expect(t, respRec.Code, http.StatusOK)

	body, err := ioutil.ReadAll(respRec.Body)

	rr := recordedRequests{}
	err = json.Unmarshal(body, &rr)

	expect(t, len(rr.Data), 0)
}

func TestGetAllRecordsWRecords(t *testing.T) {
	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	// inserting some payloads
	for i := 0; i < 5; i++ {
		req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com/q=%d", i), nil)
		expect(t, err, nil)
		dbClient.captureRequest(req)
	}
	// performing query
	m := getBoneRouter(*dbClient)

	req, err := http.NewRequest("GET", "/records", nil)
	expect(t, err, nil)

	//The response recorder used to record HTTP responses
	respRec := httptest.NewRecorder()

	m.ServeHTTP(respRec, req)

	expect(t, respRec.Code, http.StatusOK)

	body, err := ioutil.ReadAll(respRec.Body)

	rr := recordedRequests{}
	err = json.Unmarshal(body, &rr)

	expect(t, len(rr.Data), 5)
}

func TestGetRecordsCount(t *testing.T) {
	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()
	m := getBoneRouter(*dbClient)

	req, err := http.NewRequest("GET", "/count", nil)
	expect(t, err, nil)

	//The response recorder used to record HTTP responses
	respRec := httptest.NewRecorder()

	m.ServeHTTP(respRec, req)

	expect(t, respRec.Code, http.StatusOK)

	body, err := ioutil.ReadAll(respRec.Body)

	rc := recordsCount{}
	err = json.Unmarshal(body, &rc)

	expect(t, rc.Count, 0)
}
<|endoftext|>


<commit_before>package agent

import (
	"context"
	"github.com/opennetworkinglab/onos-warden/warden"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
	"io"
)

type Handler interface {
	Handle(req *warden.ClusterRequest)
}

type WardenClient interface {
	PublishUpdate(ad *warden.ClusterAdvertisement)
	Teardown()
}

type wardenClient struct {
	conn   *grpc.ClientConn
	stream warden.ClusterAgentService_AgentClustersClient
	alive  bool
}

func NewWardenClient(target string, handler Handler, creds credentials.TransportCredentials) (*wardenClient, error) {
	var wc wardenClient
	var err error

	var opts grpc.DialOption
	if creds == nil {
		opts = grpc.WithInsecure()
	} else {
		opts = grpc.WithTransportCredentials(creds)
	}
	wc.conn, err = grpc.Dial(target, opts)
	if err != nil {
		grpclog.Fatalf("fail to dial: %v", err)
		return nil, err
	}
	client := warden.NewClusterAgentServiceClient(wc.conn)
	wc.stream, err = client.AgentClusters(context.Background())
	if err != nil {
		grpclog.Fatalf("%v.ServerClusters(_) = _, %v", client, err)
		wc.conn.Close()
		return nil, err
	}

	go func() {
		for {
			in, err := wc.stream.Recv()
			if err == io.EOF {
				// TODO server has closed their side of the connection, when do we stop sending updates?
				// let's stop now!
				wc.Teardown()
				return
			}
			if err != nil {
				grpclog.Fatalf("Failed to receive: %v", err)
			}
			grpclog.Printf("Got message: %v", in)
			if handler != nil {
				handler.Handle(in)
			}
		}
	}()
	wc.alive = true
	return &wc, nil
}

func (c *wardenClient) PublishUpdate(ad *warden.ClusterAdvertisement) {
	if c.alive == false {
		//TODO keep calm, don't panic
		panic("Trying to publish update to dead connection.")
	}
	err := c.stream.Send(ad)
	if err != nil {
		panic(err)
	}
}

func (c *wardenClient) Teardown() {
	c.alive = false
	if c.stream != nil {
		c.stream.CloseSend()
		c.stream = nil
	}
	if c.conn != nil {
		c.conn.Close()
		c.conn = nil
	}
}
<commit_msg>Factoring out grpc connect and retrying<commit_after>package agent

import (
	"context"
	"github.com/opennetworkinglab/onos-warden/warden"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
	"io"
	"time"
	"fmt"
)

type Handler interface {
	Handle(req *warden.ClusterRequest)
}

type WardenClient interface {
	PublishUpdate(ad *warden.ClusterAdvertisement) error
	Teardown()
}

type wardenClient struct {
	conn   *grpc.ClientConn
	stream warden.ClusterAgentService_AgentClustersClient
	alive  bool
	pub    chan *warden.ClusterAdvertisement
}

func NewWardenClient(target string, handler Handler, creds credentials.TransportCredentials) (*wardenClient, error) {
	var wc wardenClient
	var err error
	var opts grpc.DialOption

	if creds == nil {
		opts = grpc.WithInsecure()
	} else {
		opts = grpc.WithTransportCredentials(creds)
	}

	err = wc.connect(target, opts)
	if err != nil {
		return nil, err
	}

	go func() {
		for {
			err := wc.receive(handler)
			fmt.Println("Receive error", err)
			// try to reconnect on receive error
			for i := 0; ; i++ {
				err = wc.connect(target, opts)
				if err == nil {
					break
				}
				fmt.Println("Connect error error; retrying...", err)
				time.Sleep(time.Duration(i) * 100 * time.Millisecond)
			}
		}
	}()

	return &wc, nil
}

func (c *wardenClient) connect(target string, opts grpc.DialOption) (err error) {
	c.conn, err = grpc.Dial(target, opts)
	if err != nil {
		grpclog.Printf("fail to dial: %v", err)
		return
	}
	client := warden.NewClusterAgentServiceClient(c.conn)
	c.stream, err = client.AgentClusters(context.Background())
	if err != nil {
		grpclog.Printf("failed to start stream: %v", err)
		c.conn.Close()
		return
	}
	return nil
}

func (c *wardenClient) receive(handler Handler) error {
	for {
		in, err := c.stream.Recv()
		if err == io.EOF {
			// TODO server has closed their side of the connection, when do we stop sending updates?
			// let's stop now!
			grpclog.Printf("Received EOF from server; tearing down...")
			c.Teardown()
			return err
		}
		if err != nil {
			grpclog.Printf("Failed to receive: %v", err)
			return err
		}
		grpclog.Printf("Got message: %v", in)
		if handler != nil {
			handler.Handle(in)
		}
	}
}

func (c *wardenClient) PublishUpdate(ad *warden.ClusterAdvertisement) (err error) {
	// TODO this is pretty rudimentary
	// retry 3 times with back-off
	for i := 1; i <= 3; i++ {
		err = c.stream.Send(ad)
		if err == nil {
			return
		}
		time.Sleep(time.Duration(i) * 100 * time.Millisecond)
	}
	return
}

func (c *wardenClient) Teardown() {
	c.alive = false
	if c.stream != nil {
		c.stream.CloseSend()
		c.stream = nil
	}
	if c.conn != nil {
		c.conn.Close()
		c.conn = nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/owulveryck/flue"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	//	"os"
	//	    "syscall"
	//	"os/signal"
)

func cleanup() {
	log.Println("cleanup")
}

func main() {

	/*
		// Catching the interrupt
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		signal.Notify(c, syscall.SIGTERM)
		go func() {
		    <-c
		    cleanup()
		    os.Exit(1)
		}()
	*/
	//Parsing the dot
	dotFile := flag.String("dot", "", "The dot file")

	flag.Parse()
	if *dotFile == "" {
		log.Println("Server mode")
		proto := "tcp"
		socket := "localhost:4546"
		flue.Rserver(&proto, &socket)
	} else {
		log.Println("Client mode")

		var topologyDot []byte
		topologyDot, err := ioutil.ReadFile(*dotFile)
		if err != nil {
			log.Panic("Cannot read file")
		}

		log.Println("Parsing...")
		taskStructure := flue.ParseTasks(topologyDot)

		// TEST
		sigmaStructure := flue.GetSigmaStructure(taskStructure)
		jsonOutput, _ := json.Marshal(sigmaStructure)
		log.Println(string(jsonOutput))
		// End TEST
		var wg sync.WaitGroup
		doneChan := make(chan *flue.Task)

		// END DEBUG

		// For each task, if it can run, place true in its communication channel
		for taskIndex, _ := range taskStructure.Tasks {
			log.Printf("taskIndex: %v", taskIndex)
			go flue.Runner(taskStructure.Tasks[taskIndex], doneChan, &wg)
			wg.Add(1)
		}
		go flue.Advertize(taskStructure, doneChan)

		router := flue.NewRouter(taskStructure)

		go log.Fatal(http.ListenAndServe(":8080", router))
		wg.Wait()
	}
}
<commit_msg>change the cli arguments: - no more -dot flag - you can parse as many dot file as you want<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/owulveryck/flue"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	//	"os"
	//	    "syscall"
	//	"os/signal"
)

func cleanup() {
	log.Println("cleanup")
}

func main() {

	/*
		// Catching the interrupt
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		signal.Notify(c, syscall.SIGTERM)
		go func() {
		    <-c
		    cleanup()
		    os.Exit(1)
		}()
	*/
	//Parsing the dot
	var dotFiles []string
	flag.Parse()
	dotFiles = flag.Args()
	log.Printf("tail: %v", dotFiles)
	if dotFiles == []string(nil) {
		log.Println("Server mode")
		proto := "tcp"
		socket := "localhost:4546"
		flue.Rserver(&proto, &socket)
	} else {
		log.Println("Client mode")
		for _, dotFile := range dotFiles {

			var topologyDot []byte
			topologyDot, err := ioutil.ReadFile(dotFile)
			if err != nil {
				log.Panic("Cannot read file")
			}

			log.Println("Parsing...")
			taskStructure := flue.ParseTasks(topologyDot)

			var wg sync.WaitGroup
			doneChan := make(chan *flue.Task)

			// For each task, if it can run, place true in its communication channel
			for taskIndex, _ := range taskStructure.Tasks {
				log.Printf("taskIndex: %v", taskIndex)
				go flue.Runner(taskStructure.Tasks[taskIndex], doneChan, &wg)
				wg.Add(1)
			}
			go flue.Advertize(taskStructure, doneChan)

			router := flue.NewRouter(taskStructure)

			go log.Fatal(http.ListenAndServe(":8080", router))
			wg.Wait()
		}
	}
}
<|endoftext|>


<commit_before>package ir_test

import (
	"fmt"
	"testing"

	"github.com/rthornton128/calc/ast"
	"github.com/rthornton128/calc/ir"
	"github.com/rthornton128/calc/parse"
	"github.com/rthornton128/calc/token"
)

type Test struct {
	src  string
	pass bool
}

func TestSimple(t *testing.T) {
	tests := []Test{
		{src: "42", pass: true},
		{src: "true", pass: true},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestDeclaration(t *testing.T) {
	tests := []Test{
		{src: "(decl fn(a b int) int true)", pass: false},
		{src: "(decl fn(a b int) bool a)", pass: false},
		{src: "(decl fn bool 24)", pass: false},
		{src: "(decl fn(a b int) int (+ a b))(decl main int (fn 2 3))",
			pass: true},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestBinary(t *testing.T) {
	tests := []Test{
		{src: "(+ 2 3)", pass: true},
		{src: "(* 2 3 4 5 6)", pass: true},
		{src: "(/ (* 2 3) (% 4 5) (- 8 6))", pass: true},
		{src: "(decl main bool (!= 2 3))", pass: true},
		{src: "(decl main int (+ main 1))", pass: false},
		{src: "(decl main(a b int) bool (< a b))", pass: true},
		{src: "(decl main(a bool) bool (== a true))", pass: true},
		{src: "(decl main(a bool) int (== a true))", pass: false},
		{src: "(decl main int (+ main 1))", pass: false},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestIf(t *testing.T) {
	tests := []Test{
		{src: "(if (== 1 1) int 1 0)", pass: true},
		{src: "(if (!= 1 1) int 1 true)", pass: false},
		{src: "(if (< 1 1) int false 1)", pass: false},
		{src: "(decl main (a b int) int (if (<= a b) int 0 1))", pass: true},
		{src: "(decl main (a b int) int (if (> a b) int 0 1))", pass: true},
		{src: "(decl main (a b int) int (if (>= a b) int 0 1))", pass: true},
		{src: "(decl main (a bool) int (if (== a false) int 0 1))", pass: true},
		{src: "(decl main (a int) int (if (== a false) int 0 1))", pass: false},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestUnary(t *testing.T) {
	tests := []Test{
		{src: "-24", pass: true},
		{src: "+(- 3 5)", pass: true},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestVar(t *testing.T) {
	tests := []Test{
		{src: "(decl main int ((var a int) a))", pass: true},
		{src: "(decl main int ((var a bool) a))", pass: false},
		{src: "(decl main int ((var (= a 42)) a))", pass: true},
		{src: "(decl main int ((var (= main 3)) a))", pass: false},
		{src: "(decl main bool ((var (= a true)) a))", pass: true},
		{src: "(decl main int ((var (= a true)) a))", pass: false},
		{src: "(decl main int ((var (= a 24) bool) a))", pass: false},
		{src: "(decl main int ((var a int)(= a 42) a))", pass: true},
	}
	for i, test := range tests {
		test_handler(t, fmt.Sprintf("example%d", i), test)
	}
}

func test_handler(t *testing.T, name string, test Test) {
	expr, err := parse.ParseExpression(name, test.src)
	if err != nil {
		t.Fatal(err)
	}

	var o ir.Object
	if decl, ok := expr.(*ast.DeclExpr); ok {
		o = ir.MakeDeclaration(decl, ir.NewScope(nil))
	} else {
		o = ir.MakeExpr(expr, ir.NewScope(nil))
	}
	t.Log(o)
	fset := token.NewFileSet()
	fset.Add(name, test.src)
	if err := ir.TypeCheck(o, fset); (err == nil) != test.pass {
		t.Logf("expected %v got %v", test.pass, err == nil)
		if err != nil {
			t.Log(err)
		}
		t.Fail()
	}
	t.Log(ir.FoldConstants(o))
	ir.Tag(o)

}
<commit_msg>additional ir testing<commit_after>package ir_test

import (
	"fmt"
	"testing"

	"github.com/rthornton128/calc/ast"
	"github.com/rthornton128/calc/ir"
	"github.com/rthornton128/calc/parse"
	"github.com/rthornton128/calc/token"
)

type Test struct {
	src  string
	pass bool
}

func TestSimple(t *testing.T) {
	tests := []Test{
		{src: "42", pass: true},
		{src: "true", pass: true},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestDeclaration(t *testing.T) {
	tests := []Test{
		{src: "(decl fn(a b int) int true)", pass: false},
		{src: "(decl fn(a b int) bool a)", pass: false},
		{src: "(decl fn bool 24)", pass: false},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestFile(t *testing.T) {
	tests := []Test{
		{src: "(decl fn(a b int) int (+ a b))(decl main int (fn 2 3))",
			pass: true},
		{src: "(decl equal(a b int) bool (== a b))" +
			"(decl main int (if (equal(+ 2 3) (*4 2)) int 0 1))", pass: true},
		{src: "(decl equal(a b int) bool (== a b))" +
			"(decl main int (equal 2 3))", pass: false},
	}
	for i, test := range tests {
		test_file(t, fmt.Sprintf("file-example%d", i), test)
	}
}

func TestBinary(t *testing.T) {
	tests := []Test{
		{src: "(+ 2 3)", pass: true},
		{src: "(* 2 3 4 5 6)", pass: true},
		{src: "(/ (* 2 3) (% 4 5) (- 8 6))", pass: true},
		{src: "(decl main bool (!= 2 3))", pass: true},
		{src: "(decl main int (+ main 1))", pass: false},
		{src: "(decl main(a b int) bool (< a b))", pass: true},
		{src: "(decl main(a bool) bool (== a true))", pass: true},
		{src: "(decl main(a bool) int (== a true))", pass: false},
		{src: "(decl main int (+ main 1))", pass: false},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestIf(t *testing.T) {
	tests := []Test{
		{src: "(if (== 1 1) int 1 0)", pass: true},
		{src: "(if (!= 1 1) int 1 true)", pass: false},
		{src: "(if (< 1 1) int false 1)", pass: false},
		{src: "(decl main (a b int) int (if (<= a b) int 0 1))", pass: true},
		{src: "(decl main (a b int) int (if (> a b) int 0 1))", pass: true},
		{src: "(decl main (a b int) int (if (>= a b) int 0 1))", pass: true},
		{src: "(decl main (a bool) int (if (== a false) int 0 1))", pass: true},
		{src: "(decl main (a int) int (if (== a false) int 0 1))", pass: false},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestUnary(t *testing.T) {
	tests := []Test{
		{src: "-24", pass: true},
		{src: "+(- 3 5)", pass: true},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func TestVar(t *testing.T) {
	tests := []Test{
		{src: "(decl main int ((var a int) a))", pass: true},
		{src: "(decl main int ((var a bool) a))", pass: false},
		{src: "(decl main int ((var (= a 42)) a))", pass: true},
		{src: "(decl main int ((var (= main 3)) a))", pass: false},
		{src: "(decl main bool ((var (= a true)) a))", pass: true},
		{src: "(decl main int ((var (= a true)) a))", pass: false},
		{src: "(decl main int ((var (= a 24) bool) a))", pass: false},
		{src: "(decl main int ((var a int)(= a 42) a))", pass: true},
	}
	for i, test := range tests {
		test_expression(t, fmt.Sprintf("example%d", i), test)
	}
}

func test_expression(t *testing.T, name string, test Test) {
	expr, err := parse.ParseExpression(name, test.src)
	if err != nil {
		t.Fatal(err)
	}
	test_handler(t, test, name, expr)
}

func test_file(t *testing.T, name string, test Test) {
	f, err := parse.ParseFile(token.NewFileSet(), name, test.src)
	if err != nil {
		t.Fatal(err)
	}
	test_handler(t, test, name, &ast.Package{Files: []*ast.File{f}})
}

func test_handler(t *testing.T, test Test, name string, n ast.Node) {
	var o ir.Object
	switch t := n.(type) {
	case *ast.DeclExpr:
		o = ir.MakeDeclaration(t, ir.NewScope(nil))
	case *ast.Package:
		o = ir.MakePackage(t, name)
	case ast.Expr:
		o = ir.MakeExpr(t, ir.NewScope(nil))
	}
	t.Log(o)
	fset := token.NewFileSet()
	fset.Add(name, test.src)
	if err := ir.TypeCheck(o, fset); (err == nil) != test.pass {
		t.Logf("expected %v got %v", test.pass, err == nil)
		if err != nil {
			t.Log(err)
		}
		t.Fail()
	}
	t.Log(ir.FoldConstants(o))
	ir.Tag(o)

}
<|endoftext|>