<commit_before>package task

import (
	"fmt"
	"io"
	"net/http"
	"octlink/rstore/modules/blobs"
	"octlink/rstore/modules/blobsmanifest"
	"octlink/rstore/modules/manifest"
	"octlink/rstore/utils"
	"octlink/rstore/utils/configuration"
	"octlink/rstore/utils/merrors"
	"octlink/rstore/utils/octlog"
	"octlink/rstore/utils/uuid"
	"os"
	"strings"
)

const (
	// TaskStatusNew to new status task
	TaskStatusNew = "new"

	// TaskStatusRunning for running status task
	TaskStatusRunning = "running"

	// TaskStatusFinished for finished status task
	TaskStatusFinished = "finished"

	// TaskStatusError for error status task
	TaskStatusError = "error"

	// TaskStatusStopped for stopped status task
	TaskStatusStopped = "stopped"
)

// Task for downloading and importing task management
type Task struct {
	ID         string `json:"id"`
	URL        string `josn:"url"`
	Status     string `json:"status"`
	CreateTime string `json:"createTime"`
	FinishTime string `json:"finishTime"`
	FilePath   string `json:"filePath"`
	FileName   string `json:"fileName"`
	FileLength int64  `json:"fileLength"`
	ImageName  string `json:"imageName"`
	callback   ImageCallBack
}

var logger *octlog.LogConfig

// InitLog for tasks
func InitLog(level int) {
	logger = octlog.InitLogConfig("task.log", level)
}

// GTasks all tasks map management
var GTasks = make(map[string]*Task, 0)

// GetTasks from web api
func GetTasks() map[string]*Task {
	return GTasks
}

// GetTaskByImage will get task by image id
func GetTaskByImage(imageID string) *Task {
	return nil
}

// ImageCallBack for image callback
type ImageCallBack func(string, int64, int64, string, string) error

// AddAndRun will add a new task to GTasks and run it
func (t *Task) AddAndRun(callback ImageCallBack) {
	t.callback = callback
	t.Status = TaskStatusRunning

	GTasks[t.ID] = t
	go t.Download()

	logger.Warnf("task of %s start to run, %s\n", t.ID, t.URL)
}

// GetTask by taskid
func GetTask(id string) *Task {
	return GTasks[id]
}

// UpdateFilePath update and write file path
func (t *Task) UpdateFilePath() {
	segs := strings.Split(t.URL, "/")
	t.FileName = segs[len(segs)-1]
	t.FilePath = configuration.GetConfig().RootDirectory + "/registry/temp/" +
		uuid.Generate().Simple() + "EEEEE" + t.FileName

	logger.Debugf("update file dst path %s\n", t.FilePath)
}

// ImportBlobs and then write blobs-manifest config
func (t *Task) ImportBlobs() (*blobsmanifest.BlobsManifest, error) {

	hashes, len, err := blobs.ImportBlobs(t.FilePath)
	if err != nil {
		logger.Errorf("got file hashlist error\n")
		return nil, err
	}

	if len != t.FileLength {
		logger.Errorf("filelen of blobs and http contentlen not match %d:%d\n", len, t.FileLength)
		return nil, fmt.Errorf("filelen %d not match imported len of %d", t.FileLength, len)
	}

	blobsum := blobsmanifest.CalcBlobSum(hashes)
	bm := blobsmanifest.GetBlobsManifest(blobsum)
	if bm != nil {
		logger.Warnf("blobs-manifest %s already exist, just return it\n", bm.BlobSum)
		return bm, nil
	}

	// write blobs-manifest config
	bm = new(blobsmanifest.BlobsManifest)
	bm.Size = t.FileLength
	bm.Chunks = hashes
	bm.BlobSum = blobsum
	err = bm.Write()
	if err != nil {
		logger.Errorf("write blobs-manifest %s error\n", bm.BlobSum)
		return nil, err
	}

	return bm, nil
}

// WriteManifest to write manifest file
func (t *Task) WriteManifest(blobsum string) (*manifest.Manifest, error) {

	m := manifest.GetManifest(t.ImageName, blobsum)
	if m != nil {
		logger.Warnf("manifest of %s:%s already exist\n", t.ImageName, blobsum)
		return m, nil
	}

	// write manifest config
	m = new(manifest.Manifest)
	m.Name = t.ImageName
	m.DiskSize = t.FileLength
	m.VirtualSize = utils.GetVirtualSize(t.FilePath)
	m.CreateTime = utils.CurrentTimeStr()
	m.BlobSum = blobsum

	err := m.Write()
	if err != nil {
		logger.Errorf("Create manifest error[%s]\n", err)
		return nil, err
	}

	logger.Debugf("Write manifest of %s OK\n", blobsum)

	return m, nil
}

// Download Image from URL
func (t *Task) Download() {

	r, err := http.Get(t.URL)
	if err != nil {
		logger.Errorf("create URL response error on url %s\n", t.URL)
		t.Error()
		return
	}

	defer r.Body.Close()

	t.UpdateFilePath()
	fd, err := os.Create(t.FilePath)
	if err != nil {
		logger.Errorf("create temp file %s error\n", t.FilePath)
		t.Error()
		return
	}

	defer fd.Close()

	buf := make([]byte, configuration.BlobSize)
	for {

		n, err := r.Body.Read(buf)
		if err == io.EOF {
			if n > 0 {
				t.FileLength += int64(n)
				wlen, err := fd.Write(buf[:n])
				if err != nil {
					logger.Warnf("got len %d and wroted %d, total %d\n", n, wlen, t.FileLength)
					t.Error()
					return
				}
			}
			break
		}

		if err != nil {
			logger.Errorf("read file error %s", err)
			// TBD clear
			t.Error()
			return
		}

		t.FileLength += int64(n)
		wlen, err := fd.Write(buf[:n])
		if err != nil {
			logger.Errorf("got len %d and wroted %d, total %d\n", n, wlen, t.FileLength)
			t.Error()
			return
		}
	}

	bm, err := t.ImportBlobs()
	if err != nil {
		logger.Errorf("import blobs error %s for %s\n", err, t.FileName)
		t.Error()
		// TBD remove template file
		return
	}

	m, err := t.WriteManifest(bm.BlobSum)
	if err != nil {
		logger.Errorf("write manifest error, imageid:%s, blobsum:%s\n", t.ImageName, bm.BlobSum)
		t.Error()
		return
	}

	// update task status and image info
	t.Finish(m.DiskSize, m.VirtualSize, m.BlobSum)

	logger.Debugf("got file length of %d, and wroted %s\n", t.FileLength, t.FilePath)

	return
}

// Stop this task
func (t *Task) Stop() int {
	t.callback(t.ImageName, 0, 0, "", TaskStatusError)
	return merrors.ErrSuccess
}

// Delete this task
func (t *Task) Delete() int {
	return merrors.ErrSuccess
}

// Finish task here
func (t *Task) Finish(diskSize int64, virtualSize int64, blobsum string) {
	t.Status = TaskStatusFinished
	t.FinishTime = utils.CurrentTimeStr()
	t.callback(t.ImageName, diskSize, virtualSize, blobsum, TaskStatusFinished)
}

func (t *Task) Error() {
	t.Status = TaskStatusError
	t.FinishTime = utils.CurrentTimeStr()
	t.callback(t.ImageName, 0, 0, "", TaskStatusError)
}
<commit_msg>maqg for api dowdloading and local import image<commit_after>package task

import (
	"fmt"
	"io"
	"net/http"
	"octlink/rstore/modules/blobs"
	"octlink/rstore/modules/blobsmanifest"
	"octlink/rstore/modules/manifest"
	"octlink/rstore/utils"
	"octlink/rstore/utils/configuration"
	"octlink/rstore/utils/merrors"
	"octlink/rstore/utils/octlog"
	"octlink/rstore/utils/uuid"
	"os"
	"strings"
)

const (
	// TaskStatusNew to new status task
	TaskStatusNew = "new"

	// TaskStatusRunning for running status task
	TaskStatusRunning = "running"

	// TaskStatusFinished for finished status task
	TaskStatusFinished = "finished"

	// TaskStatusError for error status task
	TaskStatusError = "error"

	// TaskStatusStopped for stopped status task
	TaskStatusStopped = "stopped"
)

// Task for downloading and importing task management
type Task struct {
	ID         string `json:"id"`
	URL        string `josn:"url"`
	Status     string `json:"status"`
	CreateTime string `json:"createTime"`
	FinishTime string `json:"finishTime"`
	FilePath   string `json:"filePath"`
	FileName   string `json:"fileName"`
	FileLength int64  `json:"fileLength"`
	ImageName  string `json:"imageName"`
	callback   ImageCallBack
}

var logger *octlog.LogConfig

// InitLog for tasks
func InitLog(level int) {
	logger = octlog.InitLogConfig("task.log", level)
}

// GTasks all tasks map management
var GTasks = make(map[string]*Task, 0)

// GetTasks from web api
func GetTasks() map[string]*Task {
	return GTasks
}

// GetTaskByImage will get task by image id
func GetTaskByImage(imageID string) *Task {
	return nil
}

// ImageCallBack for image callback
type ImageCallBack func(string, int64, int64, string, string) error

// AddAndRun will add a new task to GTasks and run it
func (t *Task) AddAndRun(callback ImageCallBack) {
	t.callback = callback
	t.Status = TaskStatusRunning

	GTasks[t.ID] = t
	go t.Download()

	logger.Warnf("task of %s start to run, %s\n", t.ID, t.URL)
}

// GetTask by taskid
func GetTask(id string) *Task {
	return GTasks[id]
}

// UpdateFilePath update and write file path
func (t *Task) UpdateFilePath() {

	if t.URL[:4] == "http" || t.URL[:4] == "ftp" {
		segs := strings.Split(t.URL, "/")
		t.FileName = segs[len(segs)-1]
		t.FilePath = configuration.GetConfig().RootDirectory + "/registry/temp/" +
				uuid.Generate().Simple() + "EEEEE" + t.FileName
	} else { // for local file import
		segs := strings.Split(t.URL, "/")
		t.FileName = segs[len(segs)-1]
		t.FilePath = t.URL
	}

	logger.Debugf("update file dst path %s\n", t.FilePath)
}

// ImportBlobs and then write blobs-manifest config
func (t *Task) ImportBlobs() (*blobsmanifest.BlobsManifest, error) {

	hashes, len, err := blobs.ImportBlobs(t.FilePath)
	if err != nil {
		logger.Errorf("got file hashlist error\n")
		return nil, err
	}

	if len != t.FileLength {
		logger.Errorf("filelen of blobs and http contentlen not match %d:%d\n", len, t.FileLength)
		return nil, fmt.Errorf("filelen %d not match imported len of %d", t.FileLength, len)
	}

	blobsum := blobsmanifest.CalcBlobSum(hashes)
	bm := blobsmanifest.GetBlobsManifest(blobsum)
	if bm != nil {
		logger.Warnf("blobs-manifest %s already exist, just return it\n", bm.BlobSum)
		return bm, nil
	}

	// write blobs-manifest config
	bm = new(blobsmanifest.BlobsManifest)
	bm.Size = t.FileLength
	bm.Chunks = hashes
	bm.BlobSum = blobsum
	err = bm.Write()
	if err != nil {
		logger.Errorf("write blobs-manifest %s error\n", bm.BlobSum)
		return nil, err
	}

	return bm, nil
}

// WriteManifest to write manifest file
func (t *Task) WriteManifest(blobsum string) (*manifest.Manifest, error) {

	m := manifest.GetManifest(t.ImageName, blobsum)
	if m != nil {
		logger.Warnf("manifest of %s:%s already exist\n", t.ImageName, blobsum)
		return m, nil
	}

	// write manifest config
	m = new(manifest.Manifest)
	m.Name = t.ImageName
	m.DiskSize = t.FileLength
	m.VirtualSize = utils.GetVirtualSize(t.FilePath)
	m.CreateTime = utils.CurrentTimeStr()
	m.BlobSum = blobsum

	err := m.Write()
	if err != nil {
		logger.Errorf("Create manifest error[%s]\n", err)
		return nil, err
	}

	logger.Debugf("Write manifest of %s OK\n", blobsum)

	return m, nil
}

func importImage(t *Task) {
	bm, err := t.ImportBlobs()
	if err != nil {
		logger.Errorf("import blobs error %s for %s\n", err, t.FileName)
		t.Error()

		// Remove temp file when failed to add image
		if t.URL[:4] == "http" || t.URL[:4] == "ftp" {
			utils.Remove(t.FilePath)
		}
		return
	}

	m, err := t.WriteManifest(bm.BlobSum)
	if err != nil {
		logger.Errorf("write manifest error, imageid:%s, blobsum:%s\n", t.ImageName, bm.BlobSum)
		t.Error()
		// Remove temp file when failed to add image
		if t.URL[:4] == "http" || t.URL[:4] == "ftp" {
			utils.Remove(t.FilePath)
		}
		return
	}

	// update task status and image info
	t.Finish(m.DiskSize, m.VirtualSize, m.BlobSum)

	logger.Debugf("got file length of %d, and wroted %s\n", t.FileLength, t.FilePath)
}

// Download Image from URL
func (t *Task) Download() {

	if t.URL[:4] != "http" && t.URL[:4] != "ftp" {

		t.UpdateFilePath()

		// import image from local
		if !utils.IsFileExist(t.URL) {
			logger.Errorf("local url to import of %s not exist", t.URL)
			t.Error()
			return
		}
		t.FileLength = utils.GetFileSize(t.URL)

		importImage(t)
		return
	}

	r, err := http.Get(t.URL)
	if err != nil {
		logger.Errorf("create URL response error on url %s\n", t.URL)
		t.Error()
		return
	}
	defer r.Body.Close()

	t.UpdateFilePath()
	fd, err := os.Create(t.FilePath)
	if err != nil {
		logger.Errorf("create temp file %s error\n", t.FilePath)
		t.Error()
		return
	}

	defer fd.Close()

	buf := make([]byte, configuration.BlobSize)
	for {

		n, err := r.Body.Read(buf)
		if err == io.EOF {
			if n > 0 {
				t.FileLength += int64(n)
				wlen, err := fd.Write(buf[:n])
				if err != nil {
					logger.Warnf("got len %d and wroted %d, total %d\n", n, wlen, t.FileLength)
					t.Error()
					return
				}
			}
			break
		}

		if err != nil {
			logger.Errorf("read file error %s", err)
			// TBD clear
			t.Error()
			return
		}

		t.FileLength += int64(n)
		wlen, err := fd.Write(buf[:n])
		if err != nil {
			logger.Errorf("got len %d and wroted %d, total %d\n", n, wlen, t.FileLength)
			t.Error()
			return
		}
	}

	importImage(t)

	return
}

// Stop this task
func (t *Task) Stop() int {
	t.callback(t.ImageName, 0, 0, "", TaskStatusError)
	return merrors.ErrSuccess
}

// Delete this task
func (t *Task) Delete() int {
	return merrors.ErrSuccess
}

// Finish task here
func (t *Task) Finish(diskSize int64, virtualSize int64, blobsum string) {
	t.Status = TaskStatusFinished
	t.FinishTime = utils.CurrentTimeStr()
	t.callback(t.ImageName, diskSize, virtualSize, blobsum, TaskStatusFinished)
}

func (t *Task) Error() {
	t.Status = TaskStatusError
	t.FinishTime = utils.CurrentTimeStr()
	t.callback(t.ImageName, 0, 0, "", TaskStatusError)
}
<|endoftext|>


<commit_before>package view
import (
    "github.com/ant0ine/go-json-rest/rest"
    "github.com/rugo/sacapi/modules/data"
    "strconv"
)

func GetJSONMessage(w rest.ResponseWriter, r *rest.Request) {
    time, err := strconv.Atoi(r.PathParam("time"))
    if err != nil {
        rest.Error(w, "Time has to be a number", 400)
        return
    }
    t := data.ClockInfo{
        Appointment: data.Appointment{
            Time: time,
            Name: "Meeting",
            Description: "Nöpe! Chuck Testa!",
        },
        Timezone: "UTC+01:00",
        Apivers: 0,
    }
    w.Header().Set("Content-Type", "application/json; charset=utf-8")
    w.WriteJson(t)
}

/*

*/
func RegisterDevice(w rest.ResponseWriter, r *rest.Request) {
    content := &RegisterDeviceRequest{}
    err := r.DecodeJsonPayload(&content)
    if err != nil {
        rest.Error(w, "Invalid request", 400)
        return
    }
    context := context.Background()
    calendar.Register(ctx, content)
}<commit_msg>Removed unused code<commit_after>package view
import (
    "github.com/ant0ine/go-json-rest/rest"
    "github.com/rugo/sacapi/modules/data"
    "strconv"
)

func GetJSONMessage(w rest.ResponseWriter, r *rest.Request) {
    time, err := strconv.Atoi(r.PathParam("time"))
    if err != nil {
        rest.Error(w, "Time has to be a number", 400)
        return
    }
    t := data.ClockInfo{
        Appointment: data.Appointment{
            Time: time,
            Name: "Meeting",
            Description: "Nöpe! Chuck Testa!",
        },
        Timezone: "UTC+01:00",
        Apivers: 0,
    }
    w.Header().Set("Content-Type", "application/json; charset=utf-8")
    w.WriteJson(t)
}<|endoftext|>


<commit_before>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax

import "bytes"

// Simplify simplifies a given program and returns whether any changes
// were made.
//
// The changes currently applied are:
//
//     Remove clearly useless parentheses       $(( (expr) ))
//     Remove dollars from vars in exprs        (($var))
//     Remove duplicate subshells               $( (stmts) )
//     Remove redundant quotes                  [[ "$var" == str ]]
//     Merge negations with unary operators     [[ ! -n $var ]]
//     Use single quotes to shorten literals    "\$foo"
//
// This function is EXPERIMENTAL; it may change or disappear at any
// point until this notice is removed.
func Simplify(n Node) bool {
	s := simplifier{}
	Walk(n, s.visit)
	return s.modified
}

type simplifier struct {
	modified bool
}

func (s *simplifier) visit(node Node) bool {
	switch x := node.(type) {
	case *Assign:
		if x.Index != nil {
			x.Index = s.removeParensArithm(x.Index)
			// Don't inline params, as x[i] and x[$i] mean
			// different things when x is an associative
			// array; the first means "i", the second "$i".
		}
	case *ParamExp:
		if x.Index != nil {
			x.Index = s.removeParensArithm(x.Index)
			// don't inline params - same as above.
		}
		if x.Slice == nil {
			break
		}
		if x.Slice.Offset != nil {
			x.Slice.Offset = s.removeParensArithm(x.Slice.Offset)
			x.Slice.Offset = s.inlineSimpleParams(x.Slice.Offset)
		}
		if x.Slice.Length != nil {
			x.Slice.Length = s.removeParensArithm(x.Slice.Length)
			x.Slice.Length = s.inlineSimpleParams(x.Slice.Length)
		}
	case *ArithmExp:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *ArithmCmd:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *ParenArithm:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *BinaryArithm:
		x.X = s.inlineSimpleParams(x.X)
		x.Y = s.inlineSimpleParams(x.Y)
	case *CmdSubst:
		x.Stmts = s.inlineSubshell(x.Stmts)
	case *Subshell:
		x.Stmts = s.inlineSubshell(x.Stmts)
	case *Word:
		x.Parts = s.simplifyWord(x.Parts)
	case *TestClause:
		x.X = s.removeParensTest(x.X)
		x.X = s.removeNegateTest(x.X)
	case *ParenTest:
		x.X = s.removeParensTest(x.X)
		x.X = s.removeNegateTest(x.X)
	case *BinaryTest:
		x.X = s.unquoteParams(x.X)
		x.X = s.removeNegateTest(x.X)
		switch x.Op {
		case TsMatch, TsNoMatch:
			// unquoting enables globbing
		default:
			x.Y = s.unquoteParams(x.Y)
		}
		x.Y = s.removeNegateTest(x.Y)
	case *UnaryTest:
		x.X = s.unquoteParams(x.X)
	}
	return true
}

func (s *simplifier) simplifyWord(wps []WordPart) []WordPart {
parts:
	for i, wp := range wps {
		dq, _ := wp.(*DblQuoted)
		if dq == nil || len(dq.Parts) != 1 {
			break
		}
		lit, _ := dq.Parts[0].(*Lit)
		if lit == nil {
			break
		}
		var buf bytes.Buffer
		escaped := false
		for _, r := range lit.Value {
			switch r {
			case '\\':
				escaped = !escaped
				if escaped {
					continue
				}
			case '\'':
				continue parts
			case '$', '"', '`':
				escaped = false
			default:
				if escaped {
					continue parts
				}
				escaped = false
			}
			buf.WriteRune(r)
		}
		newVal := buf.String()
		if newVal == lit.Value {
			break
		}
		s.modified = true
		wps[i] = &SglQuoted{
			Left:   dq.Pos(),
			Right:  dq.End(),
			Dollar: dq.Dollar,
			Value:  newVal,
		}
	}
	return wps
}

func (s *simplifier) removeParensArithm(x ArithmExpr) ArithmExpr {
	for {
		par, _ := x.(*ParenArithm)
		if par == nil {
			return x
		}
		s.modified = true
		x = par.X
	}
}

func (s *simplifier) inlineSimpleParams(x ArithmExpr) ArithmExpr {
	w, _ := x.(*Word)
	if w == nil || len(w.Parts) != 1 {
		return x
	}
	pe, _ := w.Parts[0].(*ParamExp)
	if pe == nil || !ValidName(pe.Param.Value) {
		return x
	}
	if pe.Excl || pe.Length || pe.Width || pe.Slice != nil ||
		pe.Repl != nil || pe.Exp != nil {
		return x
	}
	if pe.Index != nil {
		s.modified = true
		pe.Short = true
		return w
	}
	s.modified = true
	return &Word{Parts: []WordPart{pe.Param}}
}

func (s *simplifier) inlineSubshell(stmts []*Stmt) []*Stmt {
	for len(stmts) == 1 {
		st := stmts[0]
		if st.Negated || st.Background || st.Coprocess ||
			len(st.Redirs) > 0 {
			break
		}
		sub, _ := st.Cmd.(*Subshell)
		if sub == nil {
			break
		}
		s.modified = true
		stmts = sub.Stmts
	}
	return stmts
}

func (s *simplifier) unquoteParams(x TestExpr) TestExpr {
	w, _ := x.(*Word)
	if w == nil || len(w.Parts) != 1 {
		return x
	}
	dq, _ := w.Parts[0].(*DblQuoted)
	if dq == nil || len(dq.Parts) != 1 {
		return x
	}
	if _, ok := dq.Parts[0].(*ParamExp); !ok {
		return x
	}
	s.modified = true
	w.Parts = dq.Parts
	return w
}

func (s *simplifier) removeParensTest(x TestExpr) TestExpr {
	for {
		par, _ := x.(*ParenTest)
		if par == nil {
			return x
		}
		s.modified = true
		x = par.X
	}
}

func (s *simplifier) removeNegateTest(x TestExpr) TestExpr {
	u, _ := x.(*UnaryTest)
	if u == nil || u.Op != TsNot {
		return x
	}
	switch y := u.X.(type) {
	case *UnaryTest:
		switch y.Op {
		case TsEmpStr:
			y.Op = TsNempStr
			s.modified = true
			return y
		case TsNempStr:
			y.Op = TsEmpStr
			s.modified = true
			return y
		case TsNot:
			s.modified = true
			return y.X
		}
	case *BinaryTest:
		switch y.Op {
		case TsMatch:
			y.Op = TsNoMatch
			s.modified = true
			return y
		case TsNoMatch:
			y.Op = TsMatch
			s.modified = true
			return y
		}
	}
	return x
}
<commit_msg>syntax: no nil checks around arith simplifications<commit_after>// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax

import "bytes"

// Simplify simplifies a given program and returns whether any changes
// were made.
//
// The changes currently applied are:
//
//     Remove clearly useless parentheses       $(( (expr) ))
//     Remove dollars from vars in exprs        (($var))
//     Remove duplicate subshells               $( (stmts) )
//     Remove redundant quotes                  [[ "$var" == str ]]
//     Merge negations with unary operators     [[ ! -n $var ]]
//     Use single quotes to shorten literals    "\$foo"
//
// This function is EXPERIMENTAL; it may change or disappear at any
// point until this notice is removed.
func Simplify(n Node) bool {
	s := simplifier{}
	Walk(n, s.visit)
	return s.modified
}

type simplifier struct {
	modified bool
}

func (s *simplifier) visit(node Node) bool {
	switch x := node.(type) {
	case *Assign:
		x.Index = s.removeParensArithm(x.Index)
		// Don't inline params, as x[i] and x[$i] mean
		// different things when x is an associative
		// array; the first means "i", the second "$i".
	case *ParamExp:
		x.Index = s.removeParensArithm(x.Index)
		// don't inline params - same as above.

		if x.Slice == nil {
			break
		}
		x.Slice.Offset = s.removeParensArithm(x.Slice.Offset)
		x.Slice.Offset = s.inlineSimpleParams(x.Slice.Offset)
		x.Slice.Length = s.removeParensArithm(x.Slice.Length)
		x.Slice.Length = s.inlineSimpleParams(x.Slice.Length)
	case *ArithmExp:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *ArithmCmd:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *ParenArithm:
		x.X = s.removeParensArithm(x.X)
		x.X = s.inlineSimpleParams(x.X)
	case *BinaryArithm:
		x.X = s.inlineSimpleParams(x.X)
		x.Y = s.inlineSimpleParams(x.Y)
	case *CmdSubst:
		x.Stmts = s.inlineSubshell(x.Stmts)
	case *Subshell:
		x.Stmts = s.inlineSubshell(x.Stmts)
	case *Word:
		x.Parts = s.simplifyWord(x.Parts)
	case *TestClause:
		x.X = s.removeParensTest(x.X)
		x.X = s.removeNegateTest(x.X)
	case *ParenTest:
		x.X = s.removeParensTest(x.X)
		x.X = s.removeNegateTest(x.X)
	case *BinaryTest:
		x.X = s.unquoteParams(x.X)
		x.X = s.removeNegateTest(x.X)
		switch x.Op {
		case TsMatch, TsNoMatch:
			// unquoting enables globbing
		default:
			x.Y = s.unquoteParams(x.Y)
		}
		x.Y = s.removeNegateTest(x.Y)
	case *UnaryTest:
		x.X = s.unquoteParams(x.X)
	}
	return true
}

func (s *simplifier) simplifyWord(wps []WordPart) []WordPart {
parts:
	for i, wp := range wps {
		dq, _ := wp.(*DblQuoted)
		if dq == nil || len(dq.Parts) != 1 {
			break
		}
		lit, _ := dq.Parts[0].(*Lit)
		if lit == nil {
			break
		}
		var buf bytes.Buffer
		escaped := false
		for _, r := range lit.Value {
			switch r {
			case '\\':
				escaped = !escaped
				if escaped {
					continue
				}
			case '\'':
				continue parts
			case '$', '"', '`':
				escaped = false
			default:
				if escaped {
					continue parts
				}
				escaped = false
			}
			buf.WriteRune(r)
		}
		newVal := buf.String()
		if newVal == lit.Value {
			break
		}
		s.modified = true
		wps[i] = &SglQuoted{
			Left:   dq.Pos(),
			Right:  dq.End(),
			Dollar: dq.Dollar,
			Value:  newVal,
		}
	}
	return wps
}

func (s *simplifier) removeParensArithm(x ArithmExpr) ArithmExpr {
	for {
		par, _ := x.(*ParenArithm)
		if par == nil {
			return x
		}
		s.modified = true
		x = par.X
	}
}

func (s *simplifier) inlineSimpleParams(x ArithmExpr) ArithmExpr {
	w, _ := x.(*Word)
	if w == nil || len(w.Parts) != 1 {
		return x
	}
	pe, _ := w.Parts[0].(*ParamExp)
	if pe == nil || !ValidName(pe.Param.Value) {
		return x
	}
	if pe.Excl || pe.Length || pe.Width || pe.Slice != nil ||
		pe.Repl != nil || pe.Exp != nil {
		return x
	}
	if pe.Index != nil {
		s.modified = true
		pe.Short = true
		return w
	}
	s.modified = true
	return &Word{Parts: []WordPart{pe.Param}}
}

func (s *simplifier) inlineSubshell(stmts []*Stmt) []*Stmt {
	for len(stmts) == 1 {
		st := stmts[0]
		if st.Negated || st.Background || st.Coprocess ||
			len(st.Redirs) > 0 {
			break
		}
		sub, _ := st.Cmd.(*Subshell)
		if sub == nil {
			break
		}
		s.modified = true
		stmts = sub.Stmts
	}
	return stmts
}

func (s *simplifier) unquoteParams(x TestExpr) TestExpr {
	w, _ := x.(*Word)
	if w == nil || len(w.Parts) != 1 {
		return x
	}
	dq, _ := w.Parts[0].(*DblQuoted)
	if dq == nil || len(dq.Parts) != 1 {
		return x
	}
	if _, ok := dq.Parts[0].(*ParamExp); !ok {
		return x
	}
	s.modified = true
	w.Parts = dq.Parts
	return w
}

func (s *simplifier) removeParensTest(x TestExpr) TestExpr {
	for {
		par, _ := x.(*ParenTest)
		if par == nil {
			return x
		}
		s.modified = true
		x = par.X
	}
}

func (s *simplifier) removeNegateTest(x TestExpr) TestExpr {
	u, _ := x.(*UnaryTest)
	if u == nil || u.Op != TsNot {
		return x
	}
	switch y := u.X.(type) {
	case *UnaryTest:
		switch y.Op {
		case TsEmpStr:
			y.Op = TsNempStr
			s.modified = true
			return y
		case TsNempStr:
			y.Op = TsEmpStr
			s.modified = true
			return y
		case TsNot:
			s.modified = true
			return y.X
		}
	case *BinaryTest:
		switch y.Op {
		case TsMatch:
			y.Op = TsNoMatch
			s.modified = true
			return y
		case TsNoMatch:
			y.Op = TsMatch
			s.modified = true
			return y
		}
	}
	return x
}
<|endoftext|>


<commit_before>// +build release

package pkcs11

import "fmt"

// Release is current version of the pkcs11 library.
var Release = R{1, 0, 1}

// R holds the version of this library.
type R struct {
	Major, Minor, Patch int
}

func (r R) String() string {
	return fmt.Sprintf("%d.%d.%d", r.Major, r.Minor, r.Patch)
}
<commit_msg>Release 1.0.2<commit_after>// +build release

package pkcs11

import "fmt"

// Release is current version of the pkcs11 library.
var Release = R{1, 0, 2}

// R holds the version of this library.
type R struct {
	Major, Minor, Patch int
}

func (r R) String() string {
	return fmt.Sprintf("%d.%d.%d", r.Major, r.Minor, r.Patch)
}
<|endoftext|>


<commit_before>package fingerprint

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	log "github.com/hashicorp/go-hclog"

	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// AwsMetadataTimeout is the timeout used when contacting the AWS metadata
	// service
	AwsMetadataTimeout = 2 * time.Second
)

// map of instance type to approximate speed, in Mbits/s
// Estimates from http://stackoverflow.com/a/35806587
// This data is meant for a loose approximation
var ec2InstanceSpeedMap = map[*regexp.Regexp]int{
	regexp.MustCompile("t2.nano"):      30,
	regexp.MustCompile("t2.micro"):     70,
	regexp.MustCompile("t2.small"):     125,
	regexp.MustCompile("t2.medium"):    300,
	regexp.MustCompile("m3.medium"):    400,
	regexp.MustCompile("c4.8xlarge"):   4000,
	regexp.MustCompile("x1.16xlarge"):  5000,
	regexp.MustCompile(`.*\.large`):    500,
	regexp.MustCompile(`.*\.xlarge`):   750,
	regexp.MustCompile(`.*\.2xlarge`):  1000,
	regexp.MustCompile(`.*\.4xlarge`):  2000,
	regexp.MustCompile(`.*\.8xlarge`):  10000,
	regexp.MustCompile(`.*\.10xlarge`): 10000,
	regexp.MustCompile(`.*\.16xlarge`): 10000,
	regexp.MustCompile(`.*\.32xlarge`): 10000,
}

// EnvAWSFingerprint is used to fingerprint AWS metadata
type EnvAWSFingerprint struct {
	StaticFingerprinter

	// endpoint for EC2 metadata as expected by AWS SDK
	endpoint string

	logger log.Logger
}

// NewEnvAWSFingerprint is used to create a fingerprint from AWS metadata
func NewEnvAWSFingerprint(logger log.Logger) Fingerprint {
	f := &EnvAWSFingerprint{
		logger:   logger.Named("env_aws"),
		endpoint: strings.TrimSuffix(os.Getenv("AWS_ENV_URL"), "/meta-data/"),
	}
	return f
}

func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error {
	cfg := request.Config

	timeout := AwsMetadataTimeout

	// Check if we should tighten the timeout
	if cfg.ReadBoolDefault(TightenNetworkTimeoutsConfig, false) {
		timeout = 1 * time.Millisecond
	}

	ec2meta := ec2MetaClient(f.endpoint, timeout)

	if !ec2meta.Available() {
		return nil
	}

	// newNetwork is populated and added to the Nodes resources
	newNetwork := &structs.NetworkResource{
		Device: "eth0",
	}

	// Keys and whether they should be namespaced as unique. Any key whose value
	// uniquely identifies a node, such as ip, should be marked as unique. When
	// marked as unique, the key isn't included in the computed node class.
	keys := map[string]bool{
		"ami-id":                      false,
		"hostname":                    true,
		"instance-id":                 true,
		"instance-type":               false,
		"local-hostname":              true,
		"local-ipv4":                  true,
		"public-hostname":             true,
		"public-ipv4":                 true,
		"placement/availability-zone": false,
	}
	for k, unique := range keys {
		resp, err := ec2meta.GetMetadata(k)
		if awsErr, ok := err.(awserr.RequestFailure); ok {
			f.logger.Debug("could not read attribute value", "attribute", k, "error", awsErr)
			continue
		} else if awsErr, ok := err.(awserr.Error); ok {
			// if it's a URL error, assume we're not in an AWS environment
			// TODO: better way to detect AWS? Check xen virtualization?
			if _, ok := awsErr.OrigErr().(*url.Error); ok {
				return nil
			}

			// not sure what other errors it would return
			return err
		}

		// assume we want blank entries
		key := "platform.aws." + strings.Replace(k, "/", ".", -1)
		if unique {
			key = structs.UniqueNamespace(key)
		}

		response.AddAttribute(key, strings.Trim(resp, "\n"))
	}

	// copy over network specific information
	if val, ok := response.Attributes["unique.platform.aws.local-ipv4"]; ok && val != "" {
		response.AddAttribute("unique.network.ip-address", val)
		newNetwork.IP = val
		newNetwork.CIDR = newNetwork.IP + "/32"
	}

	// find LinkSpeed from lookup
	throughput := cfg.NetworkSpeed
	if throughput == 0 {
		throughput = f.linkSpeed(ec2meta)
	}
	if throughput == 0 {
		// Failed to determine speed. Check if the network fingerprint got it
		found := false
		if request.Node.Resources != nil && len(request.Node.Resources.Networks) > 0 {
			for _, n := range request.Node.Resources.Networks {
				if n.IP == newNetwork.IP {
					throughput = n.MBits
					found = true
					break
				}
			}
		}

		// Nothing detected so default
		if !found {
			throughput = defaultNetworkSpeed
		}
	}

	newNetwork.MBits = throughput
	response.NodeResources = &structs.NodeResources{
		Networks: []*structs.NetworkResource{newNetwork},
	}

	// populate Links
	response.AddLink("aws.ec2", fmt.Sprintf("%s.%s",
		response.Attributes["platform.aws.placement.availability-zone"],
		response.Attributes["unique.platform.aws.instance-id"]))
	response.Detected = true

	return nil
}

// EnvAWSFingerprint uses lookup table to approximate network speeds
func (f *EnvAWSFingerprint) linkSpeed(ec2meta *ec2metadata.EC2Metadata) int {

	resp, err := ec2meta.GetMetadata("instance-type")
	if err != nil {
		f.logger.Error("error reading instance-type", "error", err)
		return 0
	}

	key := strings.Trim(resp, "\n")
	netSpeed := 0
	for reg, speed := range ec2InstanceSpeedMap {
		if reg.MatchString(key) {
			netSpeed = speed
			break
		}
	}

	return netSpeed
}

func ec2MetaClient(endpoint string, timeout time.Duration) *ec2metadata.EC2Metadata {
	client := &http.Client{
		Timeout:   timeout,
		Transport: cleanhttp.DefaultTransport(),
	}

	c := aws.NewConfig().WithHTTPClient(client)
	if endpoint != "" {
		c = c.WithEndpoint(endpoint)
	}
	return ec2metadata.New(session.New(), c)
}
<commit_msg>env_aws: Disable Retries and set Session cfg<commit_after>package fingerprint

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	log "github.com/hashicorp/go-hclog"

	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// AwsMetadataTimeout is the timeout used when contacting the AWS metadata
	// service
	AwsMetadataTimeout = 2 * time.Second
)

// map of instance type to approximate speed, in Mbits/s
// Estimates from http://stackoverflow.com/a/35806587
// This data is meant for a loose approximation
var ec2InstanceSpeedMap = map[*regexp.Regexp]int{
	regexp.MustCompile("t2.nano"):      30,
	regexp.MustCompile("t2.micro"):     70,
	regexp.MustCompile("t2.small"):     125,
	regexp.MustCompile("t2.medium"):    300,
	regexp.MustCompile("m3.medium"):    400,
	regexp.MustCompile("c4.8xlarge"):   4000,
	regexp.MustCompile("x1.16xlarge"):  5000,
	regexp.MustCompile(`.*\.large`):    500,
	regexp.MustCompile(`.*\.xlarge`):   750,
	regexp.MustCompile(`.*\.2xlarge`):  1000,
	regexp.MustCompile(`.*\.4xlarge`):  2000,
	regexp.MustCompile(`.*\.8xlarge`):  10000,
	regexp.MustCompile(`.*\.10xlarge`): 10000,
	regexp.MustCompile(`.*\.16xlarge`): 10000,
	regexp.MustCompile(`.*\.32xlarge`): 10000,
}

// EnvAWSFingerprint is used to fingerprint AWS metadata
type EnvAWSFingerprint struct {
	StaticFingerprinter

	// endpoint for EC2 metadata as expected by AWS SDK
	endpoint string

	logger log.Logger
}

// NewEnvAWSFingerprint is used to create a fingerprint from AWS metadata
func NewEnvAWSFingerprint(logger log.Logger) Fingerprint {
	f := &EnvAWSFingerprint{
		logger:   logger.Named("env_aws"),
		endpoint: strings.TrimSuffix(os.Getenv("AWS_ENV_URL"), "/meta-data/"),
	}
	return f
}

func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error {
	cfg := request.Config

	timeout := AwsMetadataTimeout

	// Check if we should tighten the timeout
	if cfg.ReadBoolDefault(TightenNetworkTimeoutsConfig, false) {
		timeout = 1 * time.Millisecond
	}

	ec2meta, err := ec2MetaClient(f.endpoint, timeout)
	if err != nil {
		return fmt.Errorf("failed to setup ec2 client: %v", err)
	}

	if !ec2meta.Available() {
		return nil
	}

	// newNetwork is populated and added to the Nodes resources
	newNetwork := &structs.NetworkResource{
		Device: "eth0",
	}

	// Keys and whether they should be namespaced as unique. Any key whose value
	// uniquely identifies a node, such as ip, should be marked as unique. When
	// marked as unique, the key isn't included in the computed node class.
	keys := map[string]bool{
		"ami-id":                      false,
		"hostname":                    true,
		"instance-id":                 true,
		"instance-type":               false,
		"local-hostname":              true,
		"local-ipv4":                  true,
		"public-hostname":             true,
		"public-ipv4":                 true,
		"placement/availability-zone": false,
	}
	for k, unique := range keys {
		resp, err := ec2meta.GetMetadata(k)
		if awsErr, ok := err.(awserr.RequestFailure); ok {
			f.logger.Debug("could not read attribute value", "attribute", k, "error", awsErr)
			continue
		} else if awsErr, ok := err.(awserr.Error); ok {
			// if it's a URL error, assume we're not in an AWS environment
			// TODO: better way to detect AWS? Check xen virtualization?
			if _, ok := awsErr.OrigErr().(*url.Error); ok {
				return nil
			}

			// not sure what other errors it would return
			return err
		}

		// assume we want blank entries
		key := "platform.aws." + strings.Replace(k, "/", ".", -1)
		if unique {
			key = structs.UniqueNamespace(key)
		}

		response.AddAttribute(key, strings.Trim(resp, "\n"))
	}

	// copy over network specific information
	if val, ok := response.Attributes["unique.platform.aws.local-ipv4"]; ok && val != "" {
		response.AddAttribute("unique.network.ip-address", val)
		newNetwork.IP = val
		newNetwork.CIDR = newNetwork.IP + "/32"
	}

	// find LinkSpeed from lookup
	throughput := cfg.NetworkSpeed
	if throughput == 0 {
		throughput = f.linkSpeed(ec2meta)
	}
	if throughput == 0 {
		// Failed to determine speed. Check if the network fingerprint got it
		found := false
		if request.Node.Resources != nil && len(request.Node.Resources.Networks) > 0 {
			for _, n := range request.Node.Resources.Networks {
				if n.IP == newNetwork.IP {
					throughput = n.MBits
					found = true
					break
				}
			}
		}

		// Nothing detected so default
		if !found {
			throughput = defaultNetworkSpeed
		}
	}

	newNetwork.MBits = throughput
	response.NodeResources = &structs.NodeResources{
		Networks: []*structs.NetworkResource{newNetwork},
	}

	// populate Links
	response.AddLink("aws.ec2", fmt.Sprintf("%s.%s",
		response.Attributes["platform.aws.placement.availability-zone"],
		response.Attributes["unique.platform.aws.instance-id"]))
	response.Detected = true

	return nil
}

// EnvAWSFingerprint uses lookup table to approximate network speeds
func (f *EnvAWSFingerprint) linkSpeed(ec2meta *ec2metadata.EC2Metadata) int {

	resp, err := ec2meta.GetMetadata("instance-type")
	if err != nil {
		f.logger.Error("error reading instance-type", "error", err)
		return 0
	}

	key := strings.Trim(resp, "\n")
	netSpeed := 0
	for reg, speed := range ec2InstanceSpeedMap {
		if reg.MatchString(key) {
			netSpeed = speed
			break
		}
	}

	return netSpeed
}

func ec2MetaClient(endpoint string, timeout time.Duration) (*ec2metadata.EC2Metadata, error) {
	client := &http.Client{
		Timeout:   timeout,
		Transport: cleanhttp.DefaultTransport(),
	}

	c := aws.NewConfig().WithHTTPClient(client).WithMaxRetries(0)
	if endpoint != "" {
		c = c.WithEndpoint(endpoint)
	}

	session, err := session.NewSession(c)
	if err != nil {
		return nil, err
	}
	return ec2metadata.New(session, c), nil
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/commands/show"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:      "show",
		Usage:     "displays information about your account and of your assets at Bytemark",
		UsageText: "show accounts, discs, groups, etc - see `bytemark help show <kind of thing> `",
		Description: `displays information about the given server, group, or account.

Plurals are scripting-friendly lists of your assets at Bytemark, showing the kind of object you request, one per line.

Perfect for piping into a bash while loop!`,
		Action:      cli.ShowSubcommandHelp,
		Subcommands: show.Commands,
	})
}
<commit_msg>fix tests<commit_after>package commands

import (
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/commands/show"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:      "show",
		Usage:     "displays information about your account and of your assets at Bytemark",
		UsageText: "show accounts, discs, groups, etc - see `bytemark help show <kind of thing> `",
		Description: `displays information about the given server, group, or account

Plurals are scripting-friendly lists of your assets at Bytemark, showing the kind of object you request, one per line.

Perfect for piping into a bash while loop!`,
		Action:      cli.ShowSubcommandHelp,
		Subcommands: show.Commands,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/lytics/gowrapmx4j"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	mx4j          gowrapmx4j.MX4JService
	host          string
	port          string
	loglvl        string
	gprefix       string
	hostnameid    string
	queryInterval int
)

// Query all registered MX4J endpoints and compose their data into the MX4JMetric
// array or return error
func QueryMX4J(mx4j gowrapmx4j.MX4JService) (*[]gowrapmx4j.MX4JMetric, error) {
	reg := gowrapmx4j.RegistryGetAll()

	for _, mm := range reg {
		var newData gowrapmx4j.MX4JData
		var err error
		data := mm.Data
		log.Debugf("Metric being queried: %#v", mm)

		// If first time querying endpoint, create data struct
		if data == nil {
			newData = gowrapmx4j.Bean{}
			mx4jData, err := newData.QueryMX4J(mx4j, mm)
			if err != nil {
				retErr := fmt.Errorf("QueryMX4J Error: %v%s", newData, err)
				return nil, retErr
			}
			gowrapmx4j.RegistrySet(mm, mx4jData)
		} else {
			newData, err = data.QueryMX4J(mx4j, mm)
			gowrapmx4j.RegistrySet(mm, newData)
		}

		if mm.MetricFunc != nil && newData != nil {
			log.Debugf("Metric func running: %s", mm.HumanName)
			mm.MetricFunc(mm.Data, mm.HumanName)
		}

		if newData == nil {
			log.Errorf("No data returned from querying; blanking the metric registries")
			metrics.DefaultRegistry.UnregisterAll()
			gowrapmx4j.RegistryFlush()
		}

		if err != nil {
			retErr := fmt.Errorf("QueryMX4J Error: %v %s", newData, err)
			return nil, retErr
		}
	}

	updated := gowrapmx4j.RegistryGetAll()
	return &updated, nil
}

// Cassandra MX4J status returns all data from the gowrapmx4j.registry
// in its raw form marshalled into JSON.
func cassStatus(w http.ResponseWriter, r *http.Request) {
	metrics, err := QueryMX4J(mx4j)
	if err != nil {
		errString := fmt.Sprintf("cassStatus Error: %#v", err)
		http.Error(w, errString, http.StatusServiceUnavailable)
	}
	log.Debugf("metrics: %#v", metrics)

	mbeans := gowrapmx4j.RegistryBeans() //metrics := mm.ValFunc(mm.MBean)
	js, err := json.Marshal(mbeans)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "cassStatus: Error marshaling JSON from MX4J data: %v", err)
	}
	fmt.Fprintf(w, "%s", js)
}

// API Endpoint which will execute the optionally specified metric function
// on the data structure for cleanup.
func cleanStatus(w http.ResponseWriter, r *http.Request) {
	metrics := gowrapmx4j.RegistryGetAll()

	mjs := make(map[string]interface{})
	for _, m := range metrics {
		if m.ValFunc != nil {
			log.Infof("%s", m.HumanName)
			mdata, err := m.ValFunc(m.Data)
			if err != nil {
				log.Errorf("Error running value function for %s: %v", m.HumanName, err)
				continue
			}
			mjs[m.HumanName] = mdata
		} else {
			mjs[m.HumanName] = m.Data
		}
	}

	js, err := json.Marshal(mjs)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		http.Error(w, fmt.Sprintf("CleanFuncAPI: Error marshaling JSON from MX4J data: %#v", err), 500)
	}
	fmt.Fprintf(w, "%s", js)
}

func nodeStatus(w http.ResponseWriter, r *http.Request) {
	mjs := make(map[string]interface{})
	nsb := gowrapmx4j.RegistryGet("NodeStatusBinary")
	metricMap, err := gowrapmx4j.DistillAttributeTypes(nsb.Data)
	if err != nil {
		mjs["ERR"] = "Error extracting node status data"
		mjs["error"] = fmt.Sprintf("%v", err)
	}

	states, ok := metricMap["SimpleStates"]
	log.Debugf("%#v", states)
	if !ok {
		mjs["ERR"] = "Error extracting node status data"
		mjs["error"] = "Key: SimpleStates not in data map"
	}
	ss := states.(map[string]interface{})

	var hostKey string
	for k, v := range ss {
		log.Debug("Nodestatus: %s %#v", k, v)
		hostMatch := regexp.MustCompile(fmt.Sprintf(".*%s.*", hostnameid))
		if hostMatch.MatchString(k) {
			hostKey = k
		}
	}

	mjs[hostKey] = ss[hostKey]
	js, err := json.Marshal(mjs)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		http.Error(w, fmt.Sprintf("nodeStatus: Error marshaling JSON from MX4J data: %#v", err), 500)
	}
	fmt.Fprintf(w, "%s", js)
}

func main() {
	flag.StringVar(&host, "host", "localhost", "mx4j host address")
	flag.StringVar(&port, "port", "8081", "mx4j port to query")
	flag.StringVar(&loglvl, "loglvl", "info", "Log level to use")
	flag.StringVar(&gprefix, "gprefix", "mx4jcass.", "Graphite prefix tag")
	flag.StringVar(&hostnameid, "hostid", "", "Hostname to use as graphite classifier for metrics")
	flag.IntVar(&queryInterval, "queryInterval", 10, "Interval seconds between querying MX4J")
	flag.Parse()

	//TODO: Sort out metrics writer
	//mlog := golog.New(os.Stderr, "", golog.LstdFlags)
	//go LogToWriter(metrics.DefaultRegistry, time.Second*time.Duration(queryInterval), mlog, gprefix, stop)

	log.Infof("Initializing Cassandra mx4j status endpoint")
	mx4j = gowrapmx4j.MX4JService{Host: host, Port: port}
	mx4j.Init()

	ll, err := log.ParseLevel(loglvl)
	if err != nil {
		log.SetLevel(log.InfoLevel)
	} else {
		log.SetLevel(ll)
	}

	// TODO: Improve the initialization of the Registry
	// TODO: Multiple definitions of the NewMX4JMetric function is confusing and un-go like

	// Query singlenton values from MX4J
	mm := gowrapmx4j.MX4JMetric{HumanName: "compactions.active", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		Format: "array", Attribute: "ActiveCount", ValFunc: gowrapmx4j.DistillAttribute}
	gowrapmx4j.RegistrySet(mm, nil)

	mm = gowrapmx4j.MX4JMetric{HumanName: "compactions.pending", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		Format: "array", Attribute: "PendingTasks", ValFunc: gowrapmx4j.DistillAttribute}
	gowrapmx4j.RegistrySet(mm, nil)

	//OR
	// Query MBean attribute maps
	mm = gowrapmx4j.MX4JMetric{HumanName: "NodeStatusBinary", ObjectName: "org.apache.cassandra.net:type=FailureDetector",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	mm = gowrapmx4j.MX4JMetric{HumanName: "CompactionExecutor", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	// Query Cluster information
	mm = gowrapmx4j.MX4JMetric{HumanName: "StorageService", ObjectName: "org.apache.cassandra.db:type=StorageService",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	// Simple run loop to query MX4J
	go func() {
		for {
			log.Debug("Querying MX4J")
			_, err := QueryMX4J(mx4j)
			if err != nil {
				log.Errorf("Error Querying MX4J: %v", err)
			}
			time.Sleep(time.Second * time.Duration(queryInterval))
		}
	}()

	http.HandleFunc("/", cassStatus)
	http.HandleFunc("/clean", cleanStatus)
	http.HandleFunc("/status", nodeStatus)
	http.ListenAndServe(":8082", nil)
}
<commit_msg>Removing functions which were generalized and added to the libs API<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/lytics/gowrapmx4j"
	"github.com/lytics/gowrapmx4j/cassandra"
)

var (
	mx4j          gowrapmx4j.MX4JService
	host          string
	port          string
	loglvl        string
	gprefix       string
	hostnameid    string
	queryInterval int
)

// Cassandra MX4J status returns all data from the gowrapmx4j.registry
// in its raw form marshalled into JSON.
func cassStatus(w http.ResponseWriter, r *http.Request) {
	metrics, err := gowrapmx4j.QueryMX4J(mx4j)
	if err != nil {
		errString := fmt.Sprintf("cassStatus Error: %#v", err)
		http.Error(w, errString, http.StatusServiceUnavailable)
	}
	log.Debugf("metrics: %#v", metrics)

	mbeans := gowrapmx4j.RegistryBeans() //metrics := mm.ValFunc(mm.MBean)
	js, err := json.Marshal(mbeans)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "cassStatus: Error marshaling JSON from MX4J data: %v", err)
	}
	fmt.Fprintf(w, "%s", js)
}

func nodeStatus(w http.ResponseWriter, r *http.Request) {
	mjs := make(map[string]interface{})
	nsb := gowrapmx4j.RegistryGet("NodeStatusBinary")
	metricMap, err := gowrapmx4j.DistillAttributeTypes(nsb.Data)
	if err != nil {
		mjs["ERR"] = "Error extracting node status data"
		mjs["error"] = fmt.Sprintf("%v", err)
	}

	states, ok := metricMap["SimpleStates"]
	log.Debugf("%#v", states)
	if !ok {
		mjs["ERR"] = "Error extracting node status data"
		mjs["error"] = "Key: SimpleStates not in data map"
	}
	ss := states.(map[string]interface{})

	var hostKey string
	for k, v := range ss {
		log.Debug("Nodestatus: %s %#v", k, v)
		hostMatch := regexp.MustCompile(fmt.Sprintf(".*%s.*", hostnameid))
		if hostMatch.MatchString(k) {
			hostKey = k
		}
	}

	mjs[hostKey] = ss[hostKey]
	js, err := json.Marshal(mjs)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		http.Error(w, fmt.Sprintf("nodeStatus: Error marshaling JSON from MX4J data: %#v", err), 500)
	}
	fmt.Fprintf(w, "%s", js)
}

func main() {
	flag.StringVar(&host, "host", "localhost", "mx4j host address")
	flag.StringVar(&port, "port", "8081", "mx4j port to query")
	flag.StringVar(&loglvl, "loglvl", "info", "Log level to use")
	flag.StringVar(&gprefix, "gprefix", "mx4jcass.", "Graphite prefix tag")
	flag.StringVar(&hostnameid, "hostid", "", "Hostname to use as graphite classifier for metrics")
	flag.IntVar(&queryInterval, "queryInterval", 10, "Interval seconds between querying MX4J")
	flag.Parse()

	//TODO: Sort out metrics writer
	//mlog := golog.New(os.Stderr, "", golog.LstdFlags)
	//go LogToWriter(metrics.DefaultRegistry, time.Second*time.Duration(queryInterval), mlog, gprefix, stop)

	log.Infof("Initializing Cassandra mx4j status endpoint")
	mx4j = gowrapmx4j.MX4JService{Host: host, Port: port}
	mx4j.Init()

	ll, err := log.ParseLevel(loglvl)
	if err != nil {
		log.SetLevel(log.InfoLevel)
	} else {
		log.SetLevel(ll)
	}

	// TODO: Improve the initialization of the Registry
	// TODO: Multiple definitions of the NewMX4JMetric function is confusing and un-go like

	// Query singlenton values from MX4J
	mm := gowrapmx4j.MX4JMetric{HumanName: "compactions.active", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		Format: "array", Attribute: "ActiveCount", ValFunc: gowrapmx4j.DistillAttribute}
	gowrapmx4j.RegistrySet(mm, nil)

	mm = gowrapmx4j.MX4JMetric{HumanName: "compactions.pending", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		Format: "array", Attribute: "PendingTasks", ValFunc: gowrapmx4j.DistillAttribute}
	gowrapmx4j.RegistrySet(mm, nil)

	//OR
	// Query MBean attribute maps
	mm = gowrapmx4j.MX4JMetric{HumanName: "NodeStatusBinary", ObjectName: "org.apache.cassandra.net:type=FailureDetector",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	mm = gowrapmx4j.MX4JMetric{HumanName: "CompactionExecutor", ObjectName: "org.apache.cassandra.internal:type=CompactionExecutor",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	// Query Cluster information
	mm = gowrapmx4j.MX4JMetric{HumanName: "StorageService", ObjectName: "org.apache.cassandra.db:type=StorageService",
		ValFunc: gowrapmx4j.DistillAttributeTypes}
	gowrapmx4j.RegistrySet(mm, nil)

	// Simple run loop to query MX4J
	go func() {
		for {
			log.Debug("Querying MX4J")
			_, err := gowrapmx4j.QueryMX4J(mx4j)
			if err != nil {
				log.Errorf("Error Querying MX4J: %v", err)
			}
			time.Sleep(time.Second * time.Duration(queryInterval))
		}
	}()

	http.HandleFunc("/", cassStatus)
	http.HandleFunc("/clean", cassandra.HttpCleanStatus)
	//http.HandleFunc("/status", nodeStatus)
	http.ListenAndServe(":8082", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"thorium-go/requests"
	"thorium-go/usage"
	"time"
)

import _ "github.com/lib/pq"
import "github.com/go-martini/martini"

var registerData request.MachineRegisterResponse

func main() {
	fmt.Println("hello world")

	timeNow := time.Now()
	rand.Seed(int64(timeNow.Second()))
	port := rand.Intn(50000)
	port = port + 10000

	fmt.Println(strconv.Itoa(port), "\n")

	reqData := &request.RegisterMachine{Port: port}
	jsonBytes, err := json.Marshal(reqData)
	if err != nil {
		log.Fatal(err)
	}
	request, err := http.NewRequest("POST", "http://localhost:6960/machines/register", bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Fatal(err)
	}

	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if response.StatusCode != 200 {
		log.Print("Error registering with master")
		os.Exit(1)
	}
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	json.Unmarshal([]byte(body), &registerData)
	fmt.Println("Registered As Machine#", registerData.MachineId)

	m := martini.Classic()

	m.Get("/ping", handlePingRequest)
	//m.Post("/games/new_request", handleGameRequest)
	//m.Post("/games/:id/register_server", handleRegisterServer)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	go func() {
		fmt.Println(<-c)
		shutdown()
		os.Exit(1)
	}()
	defer shutdown()

	ticker := time.NewTicker(2 * time.Second)
	go func() {
		for {
			select {
			case <-ticker.C:
				sendHeartbeat()
			}
		}
	}()

	thisIp := fmt.Sprint("localhost:", strconv.Itoa(port))
	m.RunOnAddr(thisIp)

}

func sendHeartbeat() {
	var err error
	statusData := &request.MachineStatus{}
	statusData.MachineToken = registerData.MachineToken
	statusData.UsageCPU, _ = usage.GetCPU()
	statusData.UsageNetwork, _ = usage.GetNetworkUtilization()
	statusData.PlayerCapacity = 0.0

	jsonBytes, err := json.Marshal(statusData)
	if err != nil {
		log.Print(err)
		return
	}

	request, err := http.NewRequest("POST", "http://localhost:6960/machines/status", bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Print(err)
		return
	}
	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		log.Print(err)
		return
	}
}

func handlePingRequest() (int, string) {
	return 200, "OK"
}

func shutdown() {

	var reqData request.UnregisterMachine
	reqData.MachineToken = registerData.MachineToken
	jsonBytes, err := json.Marshal(&reqData)
	if err != nil {
		return
	}

	var req *http.Request
	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:6960/machines/%d/disconnect", registerData.MachineId), bytes.NewBuffer(jsonBytes))
	if err != nil {
		return
	}

	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		log.Print("failed to disconnect properly")
		return
	}
}
<commit_msg>removed unused variable in gameserver<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"thorium-go/requests"
	"thorium-go/usage"
	"time"
)

import _ "github.com/lib/pq"
import "github.com/go-martini/martini"

var registerData request.MachineRegisterResponse

func main() {
	fmt.Println("hello world")

	timeNow := time.Now()
	rand.Seed(int64(timeNow.Second()))
	port := rand.Intn(50000)
	port = port + 10000

	fmt.Println(strconv.Itoa(port), "\n")

	reqData := &request.RegisterMachine{Port: port}
	jsonBytes, err := json.Marshal(reqData)
	if err != nil {
		log.Fatal(err)
	}
	request, err := http.NewRequest("POST", "http://localhost:6960/machines/register", bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Fatal(err)
	}

	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if response.StatusCode != 200 {
		log.Print("Error registering with master")
		os.Exit(1)
	}
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	json.Unmarshal([]byte(body), &registerData)
	fmt.Println("Registered As Machine#", registerData.MachineId)

	m := martini.Classic()

	m.Get("/ping", handlePingRequest)
	//m.Post("/games/new_request", handleGameRequest)
	//m.Post("/games/:id/register_server", handleRegisterServer)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	go func() {
		fmt.Println(<-c)
		shutdown()
		os.Exit(1)
	}()
	defer shutdown()

	ticker := time.NewTicker(2 * time.Second)
	go func() {
		for {
			select {
			case <-ticker.C:
				sendHeartbeat()
			}
		}
	}()

	thisIp := fmt.Sprint("localhost:", strconv.Itoa(port))
	m.RunOnAddr(thisIp)

}

func sendHeartbeat() {
	var err error
	statusData := &request.MachineStatus{}
	statusData.MachineToken = registerData.MachineToken
	statusData.UsageCPU, _ = usage.GetCPU()
	statusData.UsageNetwork, _ = usage.GetNetworkUtilization()
	statusData.PlayerCapacity = 0.0

	jsonBytes, err := json.Marshal(statusData)
	if err != nil {
		log.Print(err)
		return
	}

	request, err := http.NewRequest("POST", "http://localhost:6960/machines/status", bytes.NewBuffer(jsonBytes))
	if err != nil {
		log.Print(err)
		return
	}
	request.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	_, err := client.Do(request)
	if err != nil {
		log.Print(err)
		return
	}
}

func handlePingRequest() (int, string) {
	return 200, "OK"
}

func shutdown() {

	var reqData request.UnregisterMachine
	reqData.MachineToken = registerData.MachineToken
	jsonBytes, err := json.Marshal(&reqData)
	if err != nil {
		return
	}

	var req *http.Request
	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:6960/machines/%d/disconnect", registerData.MachineId), bytes.NewBuffer(jsonBytes))
	if err != nil {
		return
	}

	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		log.Print("failed to disconnect properly")
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	"github.com/1and1/soma/internal/adm"
	"github.com/1and1/soma/internal/cmpl"
	"github.com/1and1/soma/lib/proto"
	"github.com/codegangsta/cli"
)

func registerLevels(app cli.App) *cli.App {
	app.Commands = append(app.Commands,
		[]cli.Command{
			// levels
			{
				Name:  "levels",
				Usage: "SUBCOMMANDS for notification levels",
				Subcommands: []cli.Command{
					{
						Name:         "create",
						Usage:        "Create a new notification level",
						Action:       runtime(cmdLevelCreate),
						BashComplete: cmpl.LevelCreate,
					},
					{
						Name:   "delete",
						Usage:  "Delete a notification level",
						Action: runtime(cmdLevelDelete),
					},
					{
						Name:   "list",
						Usage:  "List notification levels",
						Action: runtime(cmdLevelList),
					},
					{
						Name:   "show",
						Usage:  "Show details about a notification level",
						Action: runtime(cmdLevelShow),
					},
				},
			}, // end levels
		}...,
	)
	return &app
}

func cmdLevelCreate(c *cli.Context) error {
	uniqKeys := []string{"shortname", "numeric"}
	opts := map[string][]string{}

	if err := adm.ParseVariadicArguments(
		opts,
		[]string{},
		uniqKeys,
		uniqKeys,
		c.Args().Tail()); err != nil {
		return err
	}

	req := proto.Request{}
	req.Level = &proto.Level{}
	req.Level.Name = c.Args().First()
	req.Level.ShortName = opts["shortname"][0]
	l, err := strconv.ParseUint(opts["numeric"][0], 10, 16)
	adm.AbortOnError(err, "Syntax error, numeric argument not numeric")
	req.Level.Numeric = uint16(l)

	resp := utl.PostRequestWithBody(Client, req, "/levels/")
	fmt.Println(resp)
	return nil
}

func cmdLevelDelete(c *cli.Context) error {
	if err := adm.VerifySingleArgument(c); err != nil {
		return err
	}

	path := fmt.Sprintf("/levels/%s", c.Args().First())

	resp := utl.DeleteRequest(Client, path)
	fmt.Println(resp)
	return nil
}

func cmdLevelList(c *cli.Context) error {
	resp := utl.GetRequest(Client, "/levels/")
	fmt.Println(resp)
	return nil
}

func cmdLevelShow(c *cli.Context) error {
	if err := adm.VerifySingleArgument(c); err != nil {
		return err
	}

	path := fmt.Sprintf("/levels/%s", c.Args().First())

	resp := utl.GetRequest(Client, path)
	fmt.Println(resp)
	return nil
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>somaadm: convert cmdLevel* from util to adm<commit_after>package main

import (
	"fmt"
	"strconv"

	"github.com/1and1/soma/internal/adm"
	"github.com/1and1/soma/internal/cmpl"
	"github.com/1and1/soma/lib/proto"
	"github.com/codegangsta/cli"
)

func registerLevels(app cli.App) *cli.App {
	app.Commands = append(app.Commands,
		[]cli.Command{
			// levels
			{
				Name:  "levels",
				Usage: "SUBCOMMANDS for notification levels",
				Subcommands: []cli.Command{
					{
						Name:         "create",
						Usage:        "Create a new notification level",
						Action:       runtime(cmdLevelCreate),
						BashComplete: cmpl.LevelCreate,
					},
					{
						Name:   "delete",
						Usage:  "Delete a notification level",
						Action: runtime(cmdLevelDelete),
					},
					{
						Name:   "list",
						Usage:  "List notification levels",
						Action: runtime(cmdLevelList),
					},
					{
						Name:   "show",
						Usage:  "Show details about a notification level",
						Action: runtime(cmdLevelShow),
					},
				},
			}, // end levels
		}...,
	)
	return &app
}

func cmdLevelCreate(c *cli.Context) error {
	uniqKeys := []string{"shortname", "numeric"}
	opts := map[string][]string{}

	if err := adm.ParseVariadicArguments(
		opts,
		[]string{},
		uniqKeys,
		uniqKeys,
		c.Args().Tail()); err != nil {
		return err
	}

	req := proto.Request{}
	req.Level = &proto.Level{}
	req.Level.Name = c.Args().First()
	req.Level.ShortName = opts["shortname"][0]

	l, err := strconv.ParseUint(opts["numeric"][0], 10, 16)
	if err != nil {
		return fmt.Errorf(
			"Syntax error, numeric argument not numeric: %s",
			err.Error())
	}
	req.Level.Numeric = uint16(l)

	if resp, err := adm.PostReqBody(req, `/levels/`); err != nil {
		return err
	} else {
		return adm.FormatOut(c, resp, `command`)
	}
}

func cmdLevelDelete(c *cli.Context) error {
	if err := adm.VerifySingleArgument(c); err != nil {
		return err
	}

	path := fmt.Sprintf("/levels/%s", c.Args().First())
	if resp, err := adm.DeleteReq(path); err != nil {
		return err
	} else {
		return adm.FormatOut(c, resp, `command`)
	}
}

func cmdLevelList(c *cli.Context) error {
	if err := adm.VerifyNoArgument(c); err != nil {
		return err
	}

	if resp, err := adm.GetReq(`/levels/`); err != nil {
		return err
	} else {
		return adm.FormatOut(c, resp, `list`)
	}
}

func cmdLevelShow(c *cli.Context) error {
	if err := adm.VerifySingleArgument(c); err != nil {
		return err
	}

	path := fmt.Sprintf("/levels/%s", c.Args().First())
	if resp, err := adm.GetReq(path); err != nil {
		return err
	} else {
		return adm.FormatOut(c, resp, `show`)
	}
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt deployment:
// * recycling logs
// * periodically fetching new data.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		log.Println(cmd.Args)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		cmd := exec.Command("zoekt-git-index",
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir, "-incremental", dir)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {

			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {

				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	u, err := url.Parse(repo.URL)
	if err != nil {
		return err
	}

	_, err = os.Stat(gitindex.Path(repoDir, u))
	if os.IsNotExist(err) {
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fractoin of the cores for indexing.")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	indexDir := filepath.Join(*dataDir, "index")
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	_, err := readConfig(*mirrorConfig)
	if err != nil {
		log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
	} else {
		go periodicMirror(repoDir, *mirrorConfig, *mirrorInterval)
	}
	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(indexDir, repoDir, *fetchInterval)

	refresh(repoDir, indexDir, *indexConfig, *fetchInterval, *cpuFraction)
}
<commit_msg>add -git_index_flags to zoekt-indexserver<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt deployment:
// * recycling logs
// * periodically fetching new data.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile, indexFlags string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		log.Println(cmd.Args)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental", dir,
		}
		if indexFlags != "" {
			args = append(args, strings.Split(indexFlags, " ")...)
		}
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {

			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {

				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	u, err := url.Parse(repo.URL)
	if err != nil {
		return err
	}

	_, err = os.Stat(gitindex.Path(repoDir, u))
	if os.IsNotExist(err) {
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fractoin of the cores for indexing.")
	indexFlags := flag.String("git_index_flags", "", "flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	indexDir := filepath.Join(*dataDir, "index")
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	_, err := readConfig(*mirrorConfig)
	if err != nil {
		log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
	} else {
		go periodicMirror(repoDir, *mirrorConfig, *mirrorInterval)
	}
	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(indexDir, repoDir, *fetchInterval)

	refresh(repoDir, indexDir, *indexConfig, *indexFlags, *fetchInterval, *cpuFraction)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/bmizerany/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db
}

func main() {
	defer worker.Db.Close()

	for {
		switch {
		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime) " +
			"SELECT $1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 " +
			"WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	tx, err := worker.Db.Begin()

	if err != nil {
		log.Fatal("error beginning transaction", err)
	}

	var closed_time pq.NullTime
	if t := req.ExtractClosedDatetime(); !t.IsZero() {
		closed_time.Time = t
	}

	_, err = tx.Stmt(stmt).Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	err = tx.Commit()
	if err != nil {
		log.Fatal("error closing transaction", err)
	}

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("located closed date %s for sr: %s", parsed_date, req)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchRequests() (requests []Open311Request) {
	// find the most recent SR that we know about in the database
	rows, err := worker.Db.Query("SELECT MAX(updated_datetime) FROM service_requests;")
	if err != nil {
		log.Fatal("error finding most recent service request", err)
	}

	last_updated_at := time.Now()
	for rows.Next() {
		if err := rows.Scan(&last_updated_at); err != nil {
			log.Print("error finding most recent SR", err)
		}

		log.Printf("most recent SR timestamp %s", last_updated_at)
	}

	// janky hack to transform the last updated timestamp into
	// a format that plays nicely with the Open311 API
	// FIXME: there HAS to be a better way to handle this.
	formatted_date_string := last_updated_at.Format(time.RFC3339)
	formatted_date_string_with_tz := formatted_date_string[0:len(formatted_date_string)-1] + "-0500" // trunc the trailing 'Z' and tack on timezone

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + formatted_date_string_with_tz

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>rm redundant clause on insert stmt<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/bmizerany/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db
}

func main() {
	defer worker.Db.Close()

	for {
		switch {
		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime) " +
			"VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17); ")

		        // "WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	tx, err := worker.Db.Begin()

	if err != nil {
		log.Fatal("error beginning transaction", err)
	}

	var closed_time pq.NullTime
	if t := req.ExtractClosedDatetime(); !t.IsZero() {
		closed_time.Time = t
	}

	_, err = tx.Stmt(stmt).Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	err = tx.Commit()
	if err != nil {
		log.Fatal("error closing transaction", err)
	}

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("located closed date %s for sr: %s", parsed_date, req)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchRequests() (requests []Open311Request) {
	// find the most recent SR that we know about in the database
	rows, err := worker.Db.Query("SELECT MAX(updated_datetime) FROM service_requests;")
	if err != nil {
		log.Fatal("error finding most recent service request", err)
	}

	last_updated_at := time.Now()
	for rows.Next() {
		if err := rows.Scan(&last_updated_at); err != nil {
			log.Print("error finding most recent SR", err)
		}

		log.Printf("most recent SR timestamp %s", last_updated_at)
	}

	// janky hack to transform the last updated timestamp into
	// a format that plays nicely with the Open311 API
	// FIXME: there HAS to be a better way to handle this.
	formatted_date_string := last_updated_at.Format(time.RFC3339)
	formatted_date_string_with_tz := formatted_date_string[0:len(formatted_date_string)-1] + "-0500" // trunc the trailing 'Z' and tack on timezone

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + formatted_date_string_with_tz

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	gocache "github.com/abhiyerra/gowebcommons/cache"
	render "github.com/abhiyerra/gowebcommons/render"
	"github.com/coreos/go-etcd/etcd"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"log"
	"net/http"
	"os"
	"strconv"
)

const (
	DatabaseUrlKey = "/treemap/database_url"
)

var (
	db    gorm.DB
	cache gocache.Cache
)

type Tree struct {
	Id         int64    `json:"id"`
	CommonName string   `json:"common_name"`
	LatinName  string   `json:"latin_name"`
	GeomData   []string `json:"geom",sql:"-"`
	Area       float64  `json:"area",sql:"-"`
}

func (t *Tree) GetGeodata() {
	rows, err := db.Table("tree_geoms").Select("ST_AsGeoJSON(ST_CollectionExtract(geom, 3)) as geom2").Where("latin_name = ?", t.LatinName).Rows()
	if err != nil {
		log.Println(err)
	}

	for rows.Next() {
		var geodata string
		rows.Scan(&geodata)
		t.GeomData = append(t.GeomData, geodata)
	}
}

func (t *Tree) GetArea() {
	var a struct {
		Area float64
	}
	db.Table("tree_geoms").Select("SUM(ST_Area(ST_Transform(geom, 900913))) as area").Where("latin_name = ?", t.LatinName).Scan(&a)

	t.Area = a.Area * 0.000189394 * 0.000189394 // Get the miles
	log.Println("Area:", t.Area)
}

type NationalPark struct {
	UnitName string `json:"name"`
	GeomData string `json:"geom"`
}

func showTreesHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	treeId, _ := strconv.ParseInt(vars["treeId"], 10, 64)

	tree := cache.Get("tree/"+vars["treeId"], func() interface{} {
		tree := Tree{Id: int64(treeId)}
		db.First(&tree)
		tree.GetGeodata()
		tree.GetArea()

		return tree
	})

	render.RenderJson(w, tree)
}

func nearbyTreesHandler(w http.ResponseWriter, r *http.Request) {
	var trees []Tree

	longitude := r.URL.Query().Get("long")
	latitude := r.URL.Query().Get("lat")
	log.Println("Long:", longitude, "Lat:", latitude)

	err := db.Model(Tree{}).Select("id, latin_name, common_name").
		Where(fmt.Sprintf("latin_name in (select distinct(latin_name) From tree_geoms where ST_DWithin(ST_GeomFromText('POINT(%s %s)' , 4326)::geography, geom, 100 , true))", longitude, latitude)).
		Scan(&trees)

	if err != nil {
		log.Println(err)
	}

	render.RenderJson(w, trees)
}

func treesHandler(w http.ResponseWriter, r *http.Request) {
	trees := cache.Get("trees", func() interface{} {
		var trees []Tree

		err := db.Model(Tree{}).Select("id, latin_name, common_name").Scan(&trees)
		if err != nil {
			log.Println(err)
		}

		return trees
	})

	render.RenderJson(w, trees)
}

func parksHandler(w http.ResponseWriter, r *http.Request) {
	parks := cache.Get("parks", func() interface{} {
		var parks []NationalPark
		db.Model(NationalPark{}).Select("ST_AsGeoJSON(ST_CollectionExtract(geom, 3)) as geom_data, unit_name").Scan(&parks)
		return parks
	})

	render.RenderJson(w, parks)
}

func dbConnect(databaseUrl string) {
	log.Println("Connecting to database:", databaseUrl)
	var err error
	db, err = gorm.Open("postgres", databaseUrl)
	if err != nil {
		log.Println(err)
	}
}

func init() {
	etcdHosts := os.Getenv("ETCD_HOSTS")
	if etcdHosts == "" {
		etcdHosts = "http://127.0.0.1:4001"
	}

	etcdClient := etcd.NewClient([]string{etcdHosts})

	resp, err := etcdClient.Get(DatabaseUrlKey, false, false)
	if err != nil {
		panic(err)
	}

	databaseUrl := resp.Node.Value
	dbConnect(databaseUrl)

	cache = gocache.NewCache()
}

func main() {
	r := mux.NewRouter()
	// r.HandleFunc("/", HomeHandler)
	r.HandleFunc("/trees/nearby", nearbyTreesHandler)
	r.HandleFunc("/trees/{treeId}", showTreesHandler)
	r.HandleFunc("/trees", treesHandler)
	r.HandleFunc("/parks", parksHandler)
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./static/")))

	http.Handle("/", r)
	http.ListenAndServe(":3001", nil)
}
<commit_msg>Changed NewCache to New<commit_after>package main

import (
	"fmt"
	gocache "github.com/abhiyerra/gowebcommons/cache"
	render "github.com/abhiyerra/gowebcommons/render"
	"github.com/coreos/go-etcd/etcd"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"log"
	"net/http"
	"os"
	"strconv"
)

const (
	DatabaseUrlKey = "/treemap/database_url"
)

var (
	db    gorm.DB
	cache gocache.Cache
)

type Tree struct {
	Id         int64    `json:"id"`
	CommonName string   `json:"common_name"`
	LatinName  string   `json:"latin_name"`
	GeomData   []string `json:"geom",sql:"-"`
	Area       float64  `json:"area",sql:"-"`
}

func (t *Tree) GetGeodata() {
	rows, err := db.Table("tree_geoms").Select("ST_AsGeoJSON(ST_CollectionExtract(geom, 3)) as geom2").Where("latin_name = ?", t.LatinName).Rows()
	if err != nil {
		log.Println(err)
	}

	for rows.Next() {
		var geodata string
		rows.Scan(&geodata)
		t.GeomData = append(t.GeomData, geodata)
	}
}

func (t *Tree) GetArea() {
	var a struct {
		Area float64
	}
	db.Table("tree_geoms").Select("SUM(ST_Area(ST_Transform(geom, 900913))) as area").Where("latin_name = ?", t.LatinName).Scan(&a)

	t.Area = a.Area * 0.000189394 * 0.000189394 // Get the miles
	log.Println("Area:", t.Area)
}

type NationalPark struct {
	UnitName string `json:"name"`
	GeomData string `json:"geom"`
}

func showTreesHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	treeId, _ := strconv.ParseInt(vars["treeId"], 10, 64)

	tree := cache.Get("tree/"+vars["treeId"], func() interface{} {
		tree := Tree{Id: int64(treeId)}
		db.First(&tree)
		tree.GetGeodata()
		tree.GetArea()

		return tree
	})

	render.RenderJson(w, tree)
}

func nearbyTreesHandler(w http.ResponseWriter, r *http.Request) {
	var trees []Tree

	longitude := r.URL.Query().Get("long")
	latitude := r.URL.Query().Get("lat")
	log.Println("Long:", longitude, "Lat:", latitude)

	err := db.Model(Tree{}).Select("id, latin_name, common_name").
		Where(fmt.Sprintf("latin_name in (select distinct(latin_name) From tree_geoms where ST_DWithin(ST_GeomFromText('POINT(%s %s)' , 4326)::geography, geom, 100 , true))", longitude, latitude)).
		Scan(&trees)

	if err != nil {
		log.Println(err)
	}

	render.RenderJson(w, trees)
}

func treesHandler(w http.ResponseWriter, r *http.Request) {
	trees := cache.Get("trees", func() interface{} {
		var trees []Tree

		err := db.Model(Tree{}).Select("id, latin_name, common_name").Scan(&trees)
		if err != nil {
			log.Println(err)
		}

		return trees
	})

	render.RenderJson(w, trees)
}

func parksHandler(w http.ResponseWriter, r *http.Request) {
	parks := cache.Get("parks", func() interface{} {
		var parks []NationalPark
		db.Model(NationalPark{}).Select("ST_AsGeoJSON(ST_CollectionExtract(geom, 3)) as geom_data, unit_name").Scan(&parks)
		return parks
	})

	render.RenderJson(w, parks)
}

func dbConnect(databaseUrl string) {
	log.Println("Connecting to database:", databaseUrl)
	var err error
	db, err = gorm.Open("postgres", databaseUrl)
	if err != nil {
		log.Println(err)
	}
}

func init() {
	etcdHosts := os.Getenv("ETCD_HOSTS")
	if etcdHosts == "" {
		etcdHosts = "http://127.0.0.1:4001"
	}

	etcdClient := etcd.NewClient([]string{etcdHosts})

	resp, err := etcdClient.Get(DatabaseUrlKey, false, false)
	if err != nil {
		panic(err)
	}

	databaseUrl := resp.Node.Value
	dbConnect(databaseUrl)

	cache = gocache.New()
}

func main() {
	r := mux.NewRouter()
	// r.HandleFunc("/", HomeHandler)
	r.HandleFunc("/trees/nearby", nearbyTreesHandler)
	r.HandleFunc("/trees/{treeId}", showTreesHandler)
	r.HandleFunc("/trees", treesHandler)
	r.HandleFunc("/parks", parksHandler)
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./static/")))

	http.Handle("/", r)
	http.ListenAndServe(":3001", nil)
}
<|endoftext|>