<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/urfave/cli"
)

type runnerAPI interface {
	CreateRun(params rainforest.RunParams) (*rainforest.RunStatus, error)
	CreateTemporaryEnvironment(string) (*rainforest.Environment, error)
	rfmlAPI
}

type runner struct {
	client runnerAPI
}

func startRun(c cliContext) error {
	r := newRunner()
	return r.startRun(c)
}

func newRunner() *runner {
	return &runner{client: api}
}

// startRun starts a new Rainforest run & depending on passed flags monitors its execution
func (r *runner) startRun(c cliContext) error {
	// First check if we even want to crate new run or just monitor the existing one.
	if runIDStr := c.String("reattach"); runIDStr != "" {
		runID, err := strconv.Atoi(runIDStr)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return monitorRunStatus(c, runID)
	}

	var localTests []*rainforest.RFTest
	var err error
	if c.Bool("f") {
		localTests, err = r.prepareLocalRun(c)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
	}

	params, err := r.makeRunParams(c, localTests)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	if c.Bool("git-trigger") {
		var git gitTrigger
		git, err = newGitTrigger()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		if !git.checkTrigger() {
			log.Printf("Git trigger enabled, but %v was not found in latest commit. Exiting...", git.Trigger)
			return nil
		}
		if tags := git.getTags(); len(tags) > 0 {
			if len(params.Tags) == 0 {
				log.Print("Found tag list in the commit message, overwriting argument.")
			} else {
				log.Print("Found tag list in the commit message.")
			}
			params.Tags = tags
		}
	}

	err = preRunCSVUpload(c, api)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	runStatus, err := r.client.CreateRun(params)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	log.Printf("Run %v has been created.", runStatus.ID)

	// if background flag is enabled we'll skip monitoring run status
	if c.Bool("bg") {
		return nil
	}

	return monitorRunStatus(c, runStatus.ID)
}

func (r *runner) prepareLocalRun(c cliContext) ([]*rainforest.RFTest, error) {
	tags := getTags(c)
	files := c.Args()
	tests, err := readRFMLFiles(files)
	if err != nil {
		return nil, err
	}

	uploads, err := filterUploadTests(tests, tags)
	if err != nil {
		return nil, err
	}
	err = uploadRFMLFiles(uploads, true, r.client)
	if err != nil {
		return nil, err
	}

	forceExecute := map[string]bool{}
	for _, path := range c.StringSlice("force-execute") {
		abs, err := filepath.Abs(path)
		if err != nil {
			log.Printf("%v is not a valid path", path)
			continue
		}
		forceExecute[abs] = true
	}

	forceSkip := map[string]bool{}
	for _, path := range c.StringSlice("exclude") {
		abs, err := filepath.Abs(path)
		if err != nil {
			log.Printf("%v is not a valid path", path)
			continue
		}
		forceSkip[abs] = true
	}
	return filterExecuteTests(tests, tags, forceExecute, forceSkip), nil
}

// filterUploadTests pre-filters tests for upload. The rule is: upload anything
// with the tag *plus* anything that is depended on by a tagged test.
func filterUploadTests(tests []*rainforest.RFTest, tags []string) ([]*rainforest.RFTest, error) {
	testsByID := map[string]*rainforest.RFTest{}
	for _, test := range tests {
		testsByID[test.RFMLID] = test
	}

	// DFS for filtered tests + embeds
	includedTests := make(map[*rainforest.RFTest]bool)
	var q []*rainforest.RFTest

	// Start with tag-filtered tests
	for _, test := range tests {
		if tags == nil || anyMember(tags, test.Tags) {
			q = append(q, test)
		}
	}
	for len(q) > 0 {
		t := q[len(q)-1]
		q = q[:len(q)-1]
		includedTests[t] = true

		for _, step := range t.Steps {
			if embed, ok := step.(rainforest.RFEmbeddedTest); ok {
				embeddedTest, ok := testsByID[embed.RFMLID]
				if !ok {
					return nil, fmt.Errorf("Could not find embedded test %v", embed.RFMLID)
				}
				if _, ok := includedTests[embeddedTest]; !ok {
					q = append(q, embeddedTest)
				}
			}
		}
	}

	result := make([]*rainforest.RFTest, 0, len(includedTests))
	for t := range includedTests {
		result = append(result, t)
	}

	return result, nil
}

// filterExecuteTests filters for tests that should execute. The rules are: it
// should execute if it's tagged properly *and* has Execute set to true (or is
// in forceExecute) *and* isn't in forceSkip.
func filterExecuteTests(tests []*rainforest.RFTest, tags []string, forceExecute, forceSkip map[string]bool) []*rainforest.RFTest {
	var result []*rainforest.RFTest
	for _, test := range tests {
		path, err := filepath.Abs(test.RFMLPath)
		if err != nil {
			path = ""
		}
		if !forceSkip[path] &&
			(test.Execute || forceExecute[path]) &&
			(tags == nil || anyMember(tags, test.Tags)) {

			result = append(result, test)
		}
	}

	return result
}

func monitorRunStatus(c cliContext, runID int) error {
	failed_attempts := 1

	for {
		status, msg, done, err := getRunStatus(c.Bool("fail-fast"), runID)
		log.Print(msg)

		if done {
			if status.FrontendURL != "" {
				log.Printf("The detailed results are available at %v\n", status.FrontendURL)
			}

			postRunJUnitReport(c, runID)

			if status.Result != "passed" {
				return cli.NewExitError("", 1)
			}

			return nil
		}

		// If we've had too many errors, give up
		if failed_attempts >= 5 {
			msg := fmt.Sprintf("Can not get run status after %d attempts, giving up", failed_attempts)
			return cli.NewExitError(msg, 1)
		}

		// If we hit an error, record it
		if err != nil {
			failed_attempts++
		} else {
			// Reset attempts
			failed_attempts = 1
		}

		time.Sleep(runStatusPollInterval)
	}
}

func getRunStatus(failFast bool, runID int) (*rainforest.RunStatus, string, bool, error) {
	newStatus, err := api.CheckRunStatus(runID)
	if err != nil {
		msg := fmt.Sprintf("API error: %v\n", err)
		return newStatus, msg, false, err
	}

	if newStatus.StateDetails.IsFinalState {
		msg := fmt.Sprintf("Run %v is now %v and has %v\n", runID, newStatus.State, newStatus.Result)
		return newStatus, msg, false, nil
	}

	msg := fmt.Sprintf("Run %v is %v and is %v%% complete\n", runID, newStatus.State, newStatus.CurrentProgress.Percent)
	if newStatus.Result == "failed" && failFast {
		return newStatus, msg, true, nil
	}
	return newStatus, msg, false, nil
}

// makeRunParams parses and validates command line arguments + options
// and makes RunParams struct out of them
func (r *runner) makeRunParams(c cliContext, localTests []*rainforest.RFTest) (rainforest.RunParams, error) {
	var err error
	localOnly := localTests != nil

	var smartFolderID int
	if s := c.String("folder"); !localOnly && s != "" {
		smartFolderID, err = strconv.Atoi(c.String("folder"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	var siteID int
	if s := c.String("site"); s != "" {
		siteID, err = strconv.Atoi(c.String("site"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	var crowd string
	if crowd = c.String("crowd"); crowd != "" && crowd != "default" && crowd != "on_premise_crowd" {
		return rainforest.RunParams{}, errors.New("Invalid crowd option specified")
	}

	var conflict string
	if conflict = c.String("conflict"); conflict != "" && conflict != "abort" && conflict != "abort-all" {
		return rainforest.RunParams{}, errors.New("Invalid conflict option specified")
	}

	featureID := c.Int("feature")
	runGroupID := c.Int("run-group")

	browsers := c.StringSlice("browser")
	expandedBrowsers := expandStringSlice(browsers)

	description := c.String("description")

	var environmentID int
	if s := c.String("custom-url"); s != "" {
		var customURL *url.URL
		customURL, err = url.Parse(s)
		if err != nil {
			return rainforest.RunParams{}, err
		}

		if (customURL.Scheme != "http") && (customURL.Scheme != "https") {
			return rainforest.RunParams{}, errors.New("custom URL scheme must be http or https")
		}

		var environment *rainforest.Environment
		environment, err = r.client.CreateTemporaryEnvironment(customURL.String())
		if err != nil {
			return rainforest.RunParams{}, err
		}

		log.Printf("Created temporary environment with name %v", environment.Name)
		environmentID = environment.ID
	} else if s := c.String("environment-id"); s != "" {
		environmentID, err = strconv.Atoi(c.String("environment-id"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	// Figure out test/RFML IDs
	var testIDs interface{}
	var rfmlIDs []string
	testIDsArgs := c.Args()

	if localOnly {
		for _, t := range localTests {
			rfmlIDs = append(rfmlIDs, t.RFMLID)
		}
	} else if testIDsArgs.First() != "all" && testIDsArgs.First() != "" {
		testIDs = []int{}
		for _, arg := range testIDsArgs {
			nextTestIDs, err := stringToIntSlice(arg)
			if err != nil {
				return rainforest.RunParams{}, err
			}
			testIDs = append(testIDs.([]int), nextTestIDs...)
		}
	} else if testIDsArgs.First() == "all" {
		testIDs = "all"
	}

	tags := getTags(c)

	return rainforest.RunParams{
		Tests:         testIDs,
		RFMLIDs:       rfmlIDs,
		Tags:          tags,
		SmartFolderID: smartFolderID,
		SiteID:        siteID,
		Crowd:         crowd,
		Conflict:      conflict,
		Browsers:      expandedBrowsers,
		Description:   description,
		EnvironmentID: environmentID,
		FeatureID:     featureID,
		RunGroupID:    runGroupID,
	}, nil
}

// stringToIntSlice takes a string of comma separated integers and returns a slice of them
func stringToIntSlice(s string) ([]int, error) {
	if s == "" {
		return nil, nil
	}
	splitString := strings.Split(s, ",")
	var slicedInt []int
	for _, slice := range splitString {
		newInt, err := strconv.Atoi(strings.TrimSpace(slice))
		if err != nil {
			return slicedInt, err
		}
		slicedInt = append(slicedInt, newInt)
	}
	return slicedInt, nil
}

// getTags get tags from a CLI context. It supports expanding comma-separated
// sublists.
func getTags(c cliContext) []string {
	tags := c.StringSlice("tag")
	return expandStringSlice(tags)
}

// expandStringSlice takes a slice of strings and expands any comma separated sublists
// into one slice. This allows us to accept args like: -tag abc -tag qwe,xyz
func expandStringSlice(slice []string) []string {
	var result []string
	for _, element := range slice {
		splitElement := strings.Split(element, ",")
		for _, singleElement := range splitElement {
			result = append(result, strings.TrimSpace(singleElement))
		}
	}
	return result
}
<commit_msg>Revert "RF-2852 Backoff if the API returns an error"<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/urfave/cli"
)

type runnerAPI interface {
	CreateRun(params rainforest.RunParams) (*rainforest.RunStatus, error)
	CreateTemporaryEnvironment(string) (*rainforest.Environment, error)
	rfmlAPI
}

type runner struct {
	client runnerAPI
}

func startRun(c cliContext) error {
	r := newRunner()
	return r.startRun(c)
}

func newRunner() *runner {
	return &runner{client: api}
}

// startRun starts a new Rainforest run & depending on passed flags monitors its execution
func (r *runner) startRun(c cliContext) error {
	// First check if we even want to crate new run or just monitor the existing one.
	if runIDStr := c.String("reattach"); runIDStr != "" {
		runID, err := strconv.Atoi(runIDStr)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return monitorRunStatus(c, runID)
	}

	var localTests []*rainforest.RFTest
	var err error
	if c.Bool("f") {
		localTests, err = r.prepareLocalRun(c)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
	}

	params, err := r.makeRunParams(c, localTests)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	if c.Bool("git-trigger") {
		var git gitTrigger
		git, err = newGitTrigger()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		if !git.checkTrigger() {
			log.Printf("Git trigger enabled, but %v was not found in latest commit. Exiting...", git.Trigger)
			return nil
		}
		if tags := git.getTags(); len(tags) > 0 {
			if len(params.Tags) == 0 {
				log.Print("Found tag list in the commit message, overwriting argument.")
			} else {
				log.Print("Found tag list in the commit message.")
			}
			params.Tags = tags
		}
	}

	err = preRunCSVUpload(c, api)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	runStatus, err := r.client.CreateRun(params)
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	log.Printf("Run %v has been created.", runStatus.ID)

	// if background flag is enabled we'll skip monitoring run status
	if c.Bool("bg") {
		return nil
	}

	return monitorRunStatus(c, runStatus.ID)
}

func (r *runner) prepareLocalRun(c cliContext) ([]*rainforest.RFTest, error) {
	tags := getTags(c)
	files := c.Args()
	tests, err := readRFMLFiles(files)
	if err != nil {
		return nil, err
	}

	uploads, err := filterUploadTests(tests, tags)
	if err != nil {
		return nil, err
	}
	err = uploadRFMLFiles(uploads, true, r.client)
	if err != nil {
		return nil, err
	}

	forceExecute := map[string]bool{}
	for _, path := range c.StringSlice("force-execute") {
		abs, err := filepath.Abs(path)
		if err != nil {
			log.Printf("%v is not a valid path", path)
			continue
		}
		forceExecute[abs] = true
	}

	forceSkip := map[string]bool{}
	for _, path := range c.StringSlice("exclude") {
		abs, err := filepath.Abs(path)
		if err != nil {
			log.Printf("%v is not a valid path", path)
			continue
		}
		forceSkip[abs] = true
	}
	return filterExecuteTests(tests, tags, forceExecute, forceSkip), nil
}

// filterUploadTests pre-filters tests for upload. The rule is: upload anything
// with the tag *plus* anything that is depended on by a tagged test.
func filterUploadTests(tests []*rainforest.RFTest, tags []string) ([]*rainforest.RFTest, error) {
	testsByID := map[string]*rainforest.RFTest{}
	for _, test := range tests {
		testsByID[test.RFMLID] = test
	}

	// DFS for filtered tests + embeds
	includedTests := make(map[*rainforest.RFTest]bool)
	var q []*rainforest.RFTest

	// Start with tag-filtered tests
	for _, test := range tests {
		if tags == nil || anyMember(tags, test.Tags) {
			q = append(q, test)
		}
	}
	for len(q) > 0 {
		t := q[len(q)-1]
		q = q[:len(q)-1]
		includedTests[t] = true

		for _, step := range t.Steps {
			if embed, ok := step.(rainforest.RFEmbeddedTest); ok {
				embeddedTest, ok := testsByID[embed.RFMLID]
				if !ok {
					return nil, fmt.Errorf("Could not find embedded test %v", embed.RFMLID)
				}
				if _, ok := includedTests[embeddedTest]; !ok {
					q = append(q, embeddedTest)
				}
			}
		}
	}

	result := make([]*rainforest.RFTest, 0, len(includedTests))
	for t := range includedTests {
		result = append(result, t)
	}

	return result, nil
}

// filterExecuteTests filters for tests that should execute. The rules are: it
// should execute if it's tagged properly *and* has Execute set to true (or is
// in forceExecute) *and* isn't in forceSkip.
func filterExecuteTests(tests []*rainforest.RFTest, tags []string, forceExecute, forceSkip map[string]bool) []*rainforest.RFTest {
	var result []*rainforest.RFTest
	for _, test := range tests {
		path, err := filepath.Abs(test.RFMLPath)
		if err != nil {
			path = ""
		}
		if !forceSkip[path] &&
			(test.Execute || forceExecute[path]) &&
			(tags == nil || anyMember(tags, test.Tags)) {

			result = append(result, test)
		}
	}

	return result
}

func monitorRunStatus(c cliContext, runID int) error {
	// Create two channels to communicate with the polling goroutine
	// One that will tick when it's time to poll and the other to gather final state
	t := time.NewTicker(runStatusPollInterval)
	statusChan := make(chan statusWithError, 0)
	go updateRunStatus(c, runID, t, statusChan)

	// This channel readout will block until updateRunStatus pushed final result to it
	finalState := <-statusChan
	if finalState.err != nil {
		return cli.NewExitError(finalState.err.Error(), 1)
	}

	if finalState.status.FrontendURL != "" {
		log.Printf("The detailed results are available at %v\n", finalState.status.FrontendURL)
	}

	postRunJUnitReport(c, runID)

	if finalState.status.Result != "passed" {
		return cli.NewExitError("", 1)
	}

	return nil
}

// statusWithError is a helper type used to send RunStatus and error using single channel
type statusWithError struct {
	status *rainforest.RunStatus
	err    error
}

func updateRunStatus(c cliContext, runID int, t *time.Ticker, resChan chan statusWithError) {
	for {
		// Wait for tick
		<-t.C
		newStatus, err := api.CheckRunStatus(runID)
		if err != nil {
			resChan <- statusWithError{status: newStatus, err: err}
			return
		}

		isFinalState := newStatus.StateDetails.IsFinalState
		state := newStatus.State
		currentPercent := newStatus.CurrentProgress.Percent

		if !isFinalState {
			log.Printf("Run %v is %v and is %v%% complete\n", runID, state, currentPercent)
			if newStatus.Result == "failed" && c.Bool("fail-fast") {
				resChan <- statusWithError{status: newStatus, err: nil}
			}
		} else {
			log.Printf("Run %v is now %v and has %v\n", runID, state, newStatus.Result)
			resChan <- statusWithError{status: newStatus, err: nil}
		}
	}
}

// makeRunParams parses and validates command line arguments + options
// and makes RunParams struct out of them
func (r *runner) makeRunParams(c cliContext, localTests []*rainforest.RFTest) (rainforest.RunParams, error) {
	var err error
	localOnly := localTests != nil

	var smartFolderID int
	if s := c.String("folder"); !localOnly && s != "" {
		smartFolderID, err = strconv.Atoi(c.String("folder"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	var siteID int
	if s := c.String("site"); s != "" {
		siteID, err = strconv.Atoi(c.String("site"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	var crowd string
	if crowd = c.String("crowd"); crowd != "" && crowd != "default" && crowd != "on_premise_crowd" {
		return rainforest.RunParams{}, errors.New("Invalid crowd option specified")
	}

	var conflict string
	if conflict = c.String("conflict"); conflict != "" && conflict != "abort" && conflict != "abort-all" {
		return rainforest.RunParams{}, errors.New("Invalid conflict option specified")
	}

	featureID := c.Int("feature")
	runGroupID := c.Int("run-group")

	browsers := c.StringSlice("browser")
	expandedBrowsers := expandStringSlice(browsers)

	description := c.String("description")

	var environmentID int
	if s := c.String("custom-url"); s != "" {
		var customURL *url.URL
		customURL, err = url.Parse(s)
		if err != nil {
			return rainforest.RunParams{}, err
		}

		if (customURL.Scheme != "http") && (customURL.Scheme != "https") {
			return rainforest.RunParams{}, errors.New("custom URL scheme must be http or https")
		}

		var environment *rainforest.Environment
		environment, err = r.client.CreateTemporaryEnvironment(customURL.String())
		if err != nil {
			return rainforest.RunParams{}, err
		}

		log.Printf("Created temporary environment with name %v", environment.Name)
		environmentID = environment.ID
	} else if s := c.String("environment-id"); s != "" {
		environmentID, err = strconv.Atoi(c.String("environment-id"))
		if err != nil {
			return rainforest.RunParams{}, err
		}
	}

	// Figure out test/RFML IDs
	var testIDs interface{}
	var rfmlIDs []string
	testIDsArgs := c.Args()

	if localOnly {
		for _, t := range localTests {
			rfmlIDs = append(rfmlIDs, t.RFMLID)
		}
	} else if testIDsArgs.First() != "all" && testIDsArgs.First() != "" {
		testIDs = []int{}
		for _, arg := range testIDsArgs {
			nextTestIDs, err := stringToIntSlice(arg)
			if err != nil {
				return rainforest.RunParams{}, err
			}
			testIDs = append(testIDs.([]int), nextTestIDs...)
		}
	} else if testIDsArgs.First() == "all" {
		testIDs = "all"
	}

	tags := getTags(c)

	return rainforest.RunParams{
		Tests:         testIDs,
		RFMLIDs:       rfmlIDs,
		Tags:          tags,
		SmartFolderID: smartFolderID,
		SiteID:        siteID,
		Crowd:         crowd,
		Conflict:      conflict,
		Browsers:      expandedBrowsers,
		Description:   description,
		EnvironmentID: environmentID,
		FeatureID:     featureID,
		RunGroupID:    runGroupID,
	}, nil
}

// stringToIntSlice takes a string of comma separated integers and returns a slice of them
func stringToIntSlice(s string) ([]int, error) {
	if s == "" {
		return nil, nil
	}
	splitString := strings.Split(s, ",")
	var slicedInt []int
	for _, slice := range splitString {
		newInt, err := strconv.Atoi(strings.TrimSpace(slice))
		if err != nil {
			return slicedInt, err
		}
		slicedInt = append(slicedInt, newInt)
	}
	return slicedInt, nil
}

// getTags get tags from a CLI context. It supports expanding comma-separated
// sublists.
func getTags(c cliContext) []string {
	tags := c.StringSlice("tag")
	return expandStringSlice(tags)
}

// expandStringSlice takes a slice of strings and expands any comma separated sublists
// into one slice. This allows us to accept args like: -tag abc -tag qwe,xyz
func expandStringSlice(slice []string) []string {
	var result []string
	for _, element := range slice {
		splitElement := strings.Split(element, ",")
		for _, singleElement := range splitElement {
			result = append(result, strings.TrimSpace(singleElement))
		}
	}
	return result
}
<|endoftext|>


<commit_before>/* To-be HTTP unit tests. There is no way to gracefully shutdown HTTP server, therefore this test cannot be made at this stage. */
package v2

import (
	"fmt"
	"io/ioutil"
	"loveoneanother.at/tiedot/db"
	"net/http"
	"net/url"
	"os"
	"testing"
	"time"
)

const serverPrefix = "http://localhost:65500/"

type ApiResponse struct {
	status   int
	response []byte
}

func UseAPI(endpoint string, get map[string]string, post map[string]string) (apiResp ApiResponse, err error) {
	reqUrl := serverPrefix + endpoint + "?"
	if get != nil {
		for k, v := range get {
			reqUrl += url.QueryEscape(k) + "=" + url.QueryEscape(v) + "&"
		}
	}
	formVals := url.Values{}
	if post != nil {
		for k, v := range post {
			formVals.Add(k, v)
		}
	}
	httpResp, err := http.PostForm(reqUrl, formVals)
	if err != nil {
		return
	}
	apiResp.status = httpResp.StatusCode
	var httpRespContent []byte
	httpRespContent, err = ioutil.ReadAll(httpResp.Body)
	apiResp.response = httpRespContent
	return
}

func TestAPIV2(t *testing.T) {
	tmp := "/tmp/tiedot_apiv2_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	db, err := db.OpenDB(tmp)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>remove api test<commit_after><|endoftext|>


<commit_before>package raven

import (
	"fmt"
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

type FunctionNameTest struct {
	skip int
	pack string
	name string
}

var (
	thisFile          string
	thisPackage       string
	functionNameTests []FunctionNameTest
)

func TestFunctionName(t *testing.T) {
	for _, test := range functionNameTests {
		pc, _, _, _ := runtime.Caller(test.skip)
		pack, name := functionName(pc)

		if pack != test.pack {
			t.Errorf("incorrect package; got %s, want %s", pack, test.pack)
		}
		if name != test.name {
			t.Errorf("incorrect function; got %s, want %s", name, test.name)
		}
	}
}

func TestStacktrace(t *testing.T) {
	st := trace()
	if st == nil {
		t.Error("got nil stacktrace")
	}
	if len(st.Frames) == 0 {
		t.Error("got zero frames")
	}

	f := st.Frames[len(st.Frames)-1]
	if f.Filename != thisFile {
		t.Errorf("incorrect Filename; got %s, want %s", f.Filename, thisFile)
	}
	if !strings.HasSuffix(f.AbsolutePath, thisFile) {
		t.Error("incorrect AbsolutePath:", f.AbsolutePath)
	}
	if f.Function != "trace" {
		t.Error("incorrect Function:", f.Function)
	}
	if f.Module != thisPackage {
		t.Error("incorrect Module:", f.Module)
	}
	if f.Lineno != 83 {
		t.Error("incorrect Lineno:", f.Lineno)
	}
	if f.ContextLine != "\treturn NewStacktrace(0, 2, []string{thisPackage})" {
		t.Errorf("incorrect ContextLine: %#v", f.ContextLine)
	}
	if len(f.PreContext) != 2 || f.PreContext[0] != "// a" || f.PreContext[1] != "func trace() *Stacktrace {" {
		t.Errorf("incorrect PreContext %#v", f.PreContext)
	}
	if len(f.PostContext) != 2 || f.PostContext[0] != "\t// b" || f.PostContext[1] != "}" {
		t.Errorf("incorrect PostContext %#v", f.PostContext)
	}
	if !*f.InApp {
		t.Error("expected InApp to be true")
	}

	if st.Culprit() != fmt.Sprintf("%s.trace", thisPackage) {
		t.Error("incorrect Culprit:", st.Culprit())
	}
}

// a
func trace() *Stacktrace {
	return NewStacktrace(0, 2, []string{thisPackage})
	// b
}

func derivePackage() (file, pack string) {
	// Get file name by seeking caller's file name.
	_, callerFile, _, ok := runtime.Caller(1)
	if !ok {
		return
	}

	// Trim file name
	file = callerFile
	for _, dir := range build.Default.SrcDirs() {
		dir := dir + string(filepath.Separator)
		if trimmed := strings.TrimPrefix(callerFile, dir); len(trimmed) < len(file) {
			file = trimmed
		}
	}

	// Now derive package name
	dir := filepath.Dir(callerFile)

	dirPkg, err := build.ImportDir(dir, build.AllowBinary)
	if err != nil {
		return
	}

	pack = dirPkg.ImportPath
	return
}

func init() {
	thisFile, thisPackage = derivePackage()
	functionNameTests = []FunctionNameTest{
		{0, thisPackage, "TestFunctionName"},
		{1, "testing", "tRunner"},
		{2, "runtime", "goexit"},
		{100, "", ""},
	}
}<commit_msg>EOF newline<commit_after>package raven

import (
	"fmt"
	"go/build"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

type FunctionNameTest struct {
	skip int
	pack string
	name string
}

var (
	thisFile          string
	thisPackage       string
	functionNameTests []FunctionNameTest
)

func TestFunctionName(t *testing.T) {
	for _, test := range functionNameTests {
		pc, _, _, _ := runtime.Caller(test.skip)
		pack, name := functionName(pc)

		if pack != test.pack {
			t.Errorf("incorrect package; got %s, want %s", pack, test.pack)
		}
		if name != test.name {
			t.Errorf("incorrect function; got %s, want %s", name, test.name)
		}
	}
}

func TestStacktrace(t *testing.T) {
	st := trace()
	if st == nil {
		t.Error("got nil stacktrace")
	}
	if len(st.Frames) == 0 {
		t.Error("got zero frames")
	}

	f := st.Frames[len(st.Frames)-1]
	if f.Filename != thisFile {
		t.Errorf("incorrect Filename; got %s, want %s", f.Filename, thisFile)
	}
	if !strings.HasSuffix(f.AbsolutePath, thisFile) {
		t.Error("incorrect AbsolutePath:", f.AbsolutePath)
	}
	if f.Function != "trace" {
		t.Error("incorrect Function:", f.Function)
	}
	if f.Module != thisPackage {
		t.Error("incorrect Module:", f.Module)
	}
	if f.Lineno != 83 {
		t.Error("incorrect Lineno:", f.Lineno)
	}
	if f.ContextLine != "\treturn NewStacktrace(0, 2, []string{thisPackage})" {
		t.Errorf("incorrect ContextLine: %#v", f.ContextLine)
	}
	if len(f.PreContext) != 2 || f.PreContext[0] != "// a" || f.PreContext[1] != "func trace() *Stacktrace {" {
		t.Errorf("incorrect PreContext %#v", f.PreContext)
	}
	if len(f.PostContext) != 2 || f.PostContext[0] != "\t// b" || f.PostContext[1] != "}" {
		t.Errorf("incorrect PostContext %#v", f.PostContext)
	}
	if !*f.InApp {
		t.Error("expected InApp to be true")
	}

	if st.Culprit() != fmt.Sprintf("%s.trace", thisPackage) {
		t.Error("incorrect Culprit:", st.Culprit())
	}
}

// a
func trace() *Stacktrace {
	return NewStacktrace(0, 2, []string{thisPackage})
	// b
}

func derivePackage() (file, pack string) {
	// Get file name by seeking caller's file name.
	_, callerFile, _, ok := runtime.Caller(1)
	if !ok {
		return
	}

	// Trim file name
	file = callerFile
	for _, dir := range build.Default.SrcDirs() {
		dir := dir + string(filepath.Separator)
		if trimmed := strings.TrimPrefix(callerFile, dir); len(trimmed) < len(file) {
			file = trimmed
		}
	}

	// Now derive package name
	dir := filepath.Dir(callerFile)

	dirPkg, err := build.ImportDir(dir, build.AllowBinary)
	if err != nil {
		return
	}

	pack = dirPkg.ImportPath
	return
}

func init() {
	thisFile, thisPackage = derivePackage()
	functionNameTests = []FunctionNameTest{
		{0, thisPackage, "TestFunctionName"},
		{1, "testing", "tRunner"},
		{2, "runtime", "goexit"},
		{100, "", ""},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

var conn dbox.IConnection
var count int

// var mwg sync.WaitGroup

var (
	t0                                                time.Time
	custgroup, prodgroup, plcode, ref                 string
	value, fiscalyear, iscount, gscount, scount, step int
	globalgross, globalsga                            float64
	mapsperiod, mapkeysvalue, mapsgross               map[string]float64
	masters                                           toolkit.M
	mwg                                               sync.WaitGroup
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmaster() {
	masters = toolkit.M{}

	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))

	ccs := buildmap(map[string]*gdrj.CostCenter{},
		func() orm.IModel {
			return new(gdrj.CostCenter)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.CostCenter)
			o := obj.(*gdrj.CostCenter)
			h[o.ID] = o
		}).(map[string]*gdrj.CostCenter)
	masters.Set("costcenter", ccs)
}

func main() {
	t0 = time.Now()
	mapkeysvalue = make(map[string]float64)
	mapsperiod = make(map[string]float64)
	mapsgross = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.StringVar(&ref, "ref", "", "Reference from document or other. Default is blank")
	flag.Parse()

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	speriode := eperiode.AddDate(-1, 0, 0)

	// if plcode == "" || value == 0 {
	// 	toolkit.Println("PLCode and Value are mandatory to fill")
	// 	os.Exit(1)
	// }

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Get Data Master...")
	prepmaster()

	toolkit.Println("Count SGA Data Process...")
	// conn, _ = modules.GetDboxIConnection("db_godrej")
	conn, _ := modules.GetDboxIConnection("db_godrej")
	csga, _ := conn.NewQuery().Select().From("tmpsgaallocs").Where(dbox.Eq("y", fiscalyear-1)).Cursor(nil)
	defer csga.Close()
	defer conn.Close()

	i := 0
	ssga := csga.Count()
	ccs := masters.Get("costcenter").(map[string]*gdrj.CostCenter)
	for {

		i++
		tsga := toolkit.M{}
		e := csga.Fetch(&tsga, 1, false)
		if e != nil {
			break
		}

		date := time.Date(tsga.Get("y", 0).(int), time.Month(tsga.Get("p", 0).(int)), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)
		pval := toolkit.Sprintf("%d_%d", date.Year(), int(date.Month()))

		group := ""
		tcc, exist := ccs[tsga.Get("ccid", "").(string)]
		if exist {
			group = tcc.CostGroup01
		}

		if group == "" {
			group = "Other"
		}

		kval := toolkit.Sprintf("%v_%v", pval, group)
		mapkeysvalue[kval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		mapsperiod[pval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		globalsga += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)

		if ssga%500 == 0 {
			toolkit.Printfn("Prepare sga master %d of %d in %s",
				i, ssga,
				time.Since(t0).String())
		}

	}

	subtot := 0.0
	for _, v := range mapkeysvalue {
		subtot += v
	}
	toolkit.Printfn("subtotal 1 : %v", subtot)

	subtot = 0.0
	for _, v := range mapsperiod {
		subtot += v
	}
	toolkit.Printfn("subtotal 2 : %v", subtot)

	toolkit.Println("Start Get Gross Ratio...")
	cgrossratio, _ := conn.NewQuery().Select().From("tmpsgasalesgrossratio").Cursor(nil)
	defer cgrossratio.Close()

	for {

		tgr := toolkit.M{}
		e := cgrossratio.Fetch(&tgr, 1, false)
		if e != nil {
			break
		}

		key := toolkit.Sprintf("%v_%v", tgr.Get("year", ""), tgr.Get("month", ""))
		mapsgross[key] = toolkit.ToFloat64(tgr.Get("amount", 0), 6, toolkit.RoundingAuto)
		globalgross += toolkit.ToFloat64(tgr.Get("amount", 0), 6, toolkit.RoundingAuto)

	}

	toolkit.Println("Start Data Process...")
	filter := dbox.And(dbox.Gte("date.date", speriode), dbox.Lt("date.date", eperiode), dbox.Gt("skuid_vdist", ""))
	// filter = dbox.Eq("_id", "RK/IMN/15000001_1")

	c, _ := gdrj.Find(new(gdrj.SalesPL), filter, nil)
	defer c.Close()

	scount = c.Count()
	iscount = 0
	gscount = 0
	step = scount / 100

	jobs := make(chan *gdrj.SalesPL, count)
	toolkit.Println("Prepare Worker")
	for wi := 0; wi < 20; wi++ {
		mwg.Add(1)
		go worker(wi, jobs)
	}
	// ====================================
	// for {
	// 	iscount++

	// 	spl := new(gdrj.SalesPL)
	// 	e := c.Fetch(spl, 1, false)
	// 	if e != nil {
	// 		toolkit.Println("EOF")
	// 		break
	// 	}

	// 	globalgross += spl.GrossAmount
	// 	pval := toolkit.Sprintf("%d_%d", spl.Date.Year, spl.Date.Month)
	// 	mapsgross[pval] += spl.GrossAmount

	// 	if step == 0 {
	// 		step = 100
	// 	}

	// 	if iscount%step == 0 {
	// 		toolkit.Printfn("Preparing %d of %d (%d) in %s", iscount, scount, iscount/step,
	// 			time.Since(t0).String())
	// 	}
	// }

	// c.ResetFetch()
	// iscount = 0
	// ===========================
	for {
		iscount++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- spl
		if iscount%step == 0 {
			toolkit.Printfn("Processing %d of %d (%d) in %s", iscount, scount, iscount/step,
				time.Since(t0).String())
		}

	}

	close(jobs)
	mwg.Wait()

	toolkit.Printfn("Saved %d of %d (%d) in %s",
		gscount, scount, gscount/step,
		time.Since(t0).String())

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func worker(wi int, jobs <-chan *gdrj.SalesPL) {
	workerConn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerConn.Close()
	defer mwg.Done()

	var j *gdrj.SalesPL
	table := j.TableName()
	plmodels := masters.Get("plmodel").(map[string]*gdrj.PLModel)

	for j = range jobs {

		gscount++

		aplmodel := j.PLDatas
		for k, _ := range aplmodel {
			if strings.Contains(k, "PL33") || strings.Contains(k, "PL34") || strings.Contains(k, "PL35") {
				delete(aplmodel, k)
			}
		}

		j.PLDatas = aplmodel

		key := toolkit.Sprintf("%d_%d", j.Date.Year, int(j.Date.Month))

		ratio := j.GrossAmount / mapsgross[key]
		totsgaperiod, _ := mapsperiod[key]
		totsgaline := ratio * totsgaperiod

		if totsgaperiod == 0 {
			continue
		}

		for k, v := range mapkeysvalue {
			skey := strings.Split(k, "_")
			if toolkit.ToInt(skey[0], toolkit.RoundingAuto) != j.Date.Year ||
				toolkit.ToInt(skey[1], toolkit.RoundingAuto) != int(j.Date.Month) {
				continue
			}

			j.AddDataCC("PL33", (-totsgaline*v/totsgaperiod)*0.7, skey[2], plmodels)
			j.AddDataCC("PL34", (-totsgaline*v/totsgaperiod)*0.26, skey[2], plmodels)
			j.AddDataCC("PL35", (-totsgaline*v/totsgaperiod)*0.04, skey[2], plmodels)

		}

		j.CalcSum(masters)

		e := workerConn.NewQuery().From(table).
			Save().Exec(toolkit.M{}.Set("data", j))

		if e != nil {
			toolkit.Printfn("Unable to save %s = %s",
				j.ID, e.Error())
		}

		if step == 0 {
			step = 100
		}

		if gscount%step == 0 {
			toolkit.Printfn("Saved %d of %d (%d) in %s",
				gscount, scount, gscount/step,
				time.Since(t0).String())
		}
	}
}
<commit_msg>update<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

var conn dbox.IConnection
var count int

// var mwg sync.WaitGroup

var (
	t0                                                time.Time
	custgroup, prodgroup, plcode, ref                 string
	value, fiscalyear, iscount, gscount, scount, step int
	globalgross, globalsga                            float64
	mapsperiod, mapkeysvalue, mapsgross               map[string]float64
	masters                                           toolkit.M
	mwg                                               sync.WaitGroup
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmaster() {
	masters = toolkit.M{}

	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))

	ccs := buildmap(map[string]*gdrj.CostCenter{},
		func() orm.IModel {
			return new(gdrj.CostCenter)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.CostCenter)
			o := obj.(*gdrj.CostCenter)
			h[o.ID] = o
		}).(map[string]*gdrj.CostCenter)
	masters.Set("costcenter", ccs)
}

func main() {
	t0 = time.Now()
	mapkeysvalue = make(map[string]float64)
	mapsperiod = make(map[string]float64)
	mapsgross = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.StringVar(&ref, "ref", "", "Reference from document or other. Default is blank")
	flag.Parse()

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	speriode := eperiode.AddDate(-1, 0, 0)

	// if plcode == "" || value == 0 {
	// 	toolkit.Println("PLCode and Value are mandatory to fill")
	// 	os.Exit(1)
	// }

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Get Data Master...")
	prepmaster()

	toolkit.Println("Count SGA Data Process...")
	// conn, _ = modules.GetDboxIConnection("db_godrej")
	conn, _ := modules.GetDboxIConnection("db_godrej")
	csga, _ := conn.NewQuery().Select().From("tmpsgaallocs").Where(dbox.Eq("y", fiscalyear-1)).Cursor(nil)
	defer csga.Close()
	defer conn.Close()

	i := 0
	ssga := csga.Count()
	ccs := masters.Get("costcenter").(map[string]*gdrj.CostCenter)
	for {

		i++
		tsga := toolkit.M{}
		e := csga.Fetch(&tsga, 1, false)
		if e != nil {
			break
		}

		date := time.Date(tsga.Get("y", 0).(int), time.Month(tsga.Get("p", 0).(int)), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)
		pval := toolkit.Sprintf("%d_%d", date.Year(), int(date.Month()))

		group := ""
		tcc, exist := ccs[tsga.Get("ccid", "").(string)]
		if exist {
			group = tcc.CostGroup01
		}

		if group == "" {
			group = "Other"
		}

		kval := toolkit.Sprintf("%v_%v", pval, group)
		mapkeysvalue[kval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		mapsperiod[pval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		globalsga += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)

		if ssga%500 == 0 {
			toolkit.Printfn("Prepare sga master %d of %d in %s",
				i, ssga,
				time.Since(t0).String())
		}

	}

	subtot := 0.0
	for _, v := range mapkeysvalue {
		subtot += v
	}
	toolkit.Printfn("subtotal 1 : %v", subtot)

	subtot = 0.0
	for _, v := range mapsperiod {
		subtot += v
	}
	toolkit.Printfn("subtotal 2 : %v", subtot)

	toolkit.Println("Start Get Gross Ratio...")
	cgrossratio, _ := conn.NewQuery().Select().From("tmpsgasalesgrossratio").Cursor(nil)
	defer cgrossratio.Close()

	for {

		tgr := toolkit.M{}
		e := cgrossratio.Fetch(&tgr, 1, false)
		if e != nil {
			break
		}

		key := toolkit.Sprintf("%v_%v", tgr.Get("year", ""), tgr.Get("month", ""))
		mapsgross[key] = toolkit.ToFloat64(tgr.Get("amount", 0), 6, toolkit.RoundingAuto)
		globalgross += toolkit.ToFloat64(tgr.Get("amount", 0), 6, toolkit.RoundingAuto)

	}

	toolkit.Println("Start Data Process...")
	filter := dbox.And(dbox.Gte("date.date", speriode), dbox.Lt("date.date", eperiode), dbox.Gt("skuid_vdist", ""))
	// filter = dbox.Eq("_id", "RK/IMN/15000001_1")

	c, _ := gdrj.Find(new(gdrj.SalesPL), filter, nil)
	defer c.Close()

	scount = c.Count()
	iscount = 0
	gscount = 0
	step = scount / 100

	jobs := make(chan *gdrj.SalesPL, count)
	toolkit.Println("Prepare Worker")
	for wi := 0; wi < 20; wi++ {
		mwg.Add(1)
		go worker(wi, jobs)
	}
	// ====================================
	// for {
	// 	iscount++

	// 	spl := new(gdrj.SalesPL)
	// 	e := c.Fetch(spl, 1, false)
	// 	if e != nil {
	// 		toolkit.Println("EOF")
	// 		break
	// 	}

	// 	globalgross += spl.GrossAmount
	// 	pval := toolkit.Sprintf("%d_%d", spl.Date.Year, spl.Date.Month)
	// 	mapsgross[pval] += spl.GrossAmount

	// 	if step == 0 {
	// 		step = 100
	// 	}

	// 	if iscount%step == 0 {
	// 		toolkit.Printfn("Preparing %d of %d (%d) in %s", iscount, scount, iscount/step,
	// 			time.Since(t0).String())
	// 	}
	// }

	// c.ResetFetch()
	// iscount = 0
	// ===========================
	for {
		iscount++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- spl
		if iscount%step == 0 {
			toolkit.Printfn("Processing %d of %d (%d) in %s", iscount, scount, iscount/step,
				time.Since(t0).String())
		}

	}

	close(jobs)
	mwg.Wait()

	toolkit.Printfn("Saved %d of %d (%d) in %s",
		gscount, scount, gscount/step,
		time.Since(t0).String())

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func worker(wi int, jobs <-chan *gdrj.SalesPL) {
	workerConn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerConn.Close()
	defer mwg.Done()

	var j *gdrj.SalesPL
	table := j.TableName()
	plmodels := masters.Get("plmodel").(map[string]*gdrj.PLModel)

	for j = range jobs {

		gscount++

		aplmodel := j.PLDatas
		for k, _ := range aplmodel {
			if strings.Contains(k, "PL33") || strings.Contains(k, "PL34") || strings.Contains(k, "PL35") {
				delete(aplmodel, k)
			}
		}

		j.PLDatas = aplmodel

		key := toolkit.Sprintf("%d_%d", j.Date.Year, int(j.Date.Month))

		ratio := j.GrossAmount / mapsgross[key]
		totsgaperiod, _ := mapsperiod[key]
		totsgaline := ratio * totsgaperiod

		if totsgaperiod == 0 {
			continue
		}

		for k, v := range mapkeysvalue {
			skey := strings.Split(k, "_")
			if toolkit.ToInt(skey[0], toolkit.RoundingAuto) != j.Date.Year ||
				toolkit.ToInt(skey[1], toolkit.RoundingAuto) != int(j.Date.Month) {
				continue
			}
			// 0.698721471, 0.265383214, 0.035895315
			j.AddDataCC("PL33", (-totsgaline*v/totsgaperiod)*0.698721471, skey[2], plmodels)
			j.AddDataCC("PL34", (-totsgaline*v/totsgaperiod)*0.265383214, skey[2], plmodels)
			j.AddDataCC("PL35", (-totsgaline*v/totsgaperiod)*0.035895315, skey[2], plmodels)

		}

		j.CalcSum(masters)

		e := workerConn.NewQuery().From(table).
			Save().Exec(toolkit.M{}.Set("data", j))

		if e != nil {
			toolkit.Printfn("Unable to save %s = %s",
				j.ID, e.Error())
		}

		if step == 0 {
			step = 100
		}

		if gscount%step == 0 {
			toolkit.Printfn("Saved %d of %d (%d) in %s",
				gscount, scount, gscount/step,
				time.Since(t0).String())
		}
	}
}
<|endoftext|>


<commit_before>// An LRU cache aimed at holding a small set of values
package scache

import (
	"github.com/karlseguin/nd"
	"math/rand"
	"sort"
	"sync"
	"sync/atomic"
	"time"
)

type Cache struct {
	*Configuration
	sync.RWMutex
	times   Times
	scratch []string
	lookup  map[string]*Item
}

type Item struct {
	key      string
	value    interface{}
	expires  int64
	accessed int64
}

type Times []int64

func (t Times) Len() int {
	return len(t)
}

func (t Times) Less(i, j int) bool {
	return t[i] < t[j]
}

func (t Times) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

func New(config *Configuration) *Cache {
	c := &Cache{
		Configuration: config,
		lookup:        make(map[string]*Item),
		times:         make(Times, config.workSize),
		scratch:       make([]string, config.workSize),
	}
	go c.gc()
	return c
}

func (c *Cache) Get(key string) interface{} {
	c.RLock()
	item, ok := c.lookup[key]
	c.RUnlock()
	if ok == false {
		return nil
	}
	now := nd.Now().Unix()
	if atomic.LoadInt64(&item.expires) < now {
		return nil
	}
	atomic.StoreInt64(&item.accessed, now)
	return item.value
}

func (c *Cache) Set(key string, value interface{}) {
	now := nd.Now()
	item := &Item{
		key:      key,
		value:    value,
		accessed: now.Unix(),
		expires:  now.Add(c.ttl).Unix(),
	}
	c.Lock()
	c.lookup[key] = item
	c.Unlock()
}

func (c *Cache) gc() {
	for {
		time.Sleep(c.pruneFrequency)
		c.RLock()
		l := len(c.lookup)
		c.RUnlock()
		if l > c.maxItems {
			c.prune(int32(l))
		}
	}
}

func (c *Cache) prune(l int32) {
	c.RLock()
	found := 0
	for _, item := range c.lookup {
		if rand.Int31n(l) > int32(c.workSize) {
			continue
		}
		c.times[found] = atomic.LoadInt64(&item.accessed)
		found++
		if found == c.workSize {
			break
		}
	}
	c.RUnlock()

	sort.Sort(c.times[:found])
	target := c.times[0]

	c.RLock()
	found = 0
	for key, item := range c.lookup {
		if atomic.LoadInt64(&item.accessed) > target {
			continue
		}
		c.scratch[found] = key
		found++
		if found == c.workSize {
			break
		}
	}
	c.RUnlock()

	c.Lock()
	for _, key := range c.scratch[:found] {
		delete(c.lookup, key)
	}
	c.Unlock()
}
<commit_msg>Keep pruning until our cache size is smaller than MaxItems<commit_after>// An LRU cache aimed at holding a small set of values
package scache

import (
	"github.com/karlseguin/nd"
	"math/rand"
	"sort"
	"sync"
	"sync/atomic"
	"time"
)

type Cache struct {
	*Configuration
	sync.RWMutex
	times   Times
	scratch []string
	lookup  map[string]*Item
}

type Item struct {
	key      string
	value    interface{}
	expires  int64
	accessed int64
}

type Times []int64

func (t Times) Len() int {
	return len(t)
}

func (t Times) Less(i, j int) bool {
	return t[i] < t[j]
}

func (t Times) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}

func New(config *Configuration) *Cache {
	c := &Cache{
		Configuration: config,
		lookup:        make(map[string]*Item),
		times:         make(Times, config.workSize),
		scratch:       make([]string, config.workSize),
	}
	go c.gc()
	return c
}

func (c *Cache) Get(key string) interface{} {
	c.RLock()
	item, ok := c.lookup[key]
	c.RUnlock()
	if ok == false {
		return nil
	}
	now := nd.Now().Unix()
	if atomic.LoadInt64(&item.expires) < now {
		return nil
	}
	atomic.StoreInt64(&item.accessed, now)
	return item.value
}

func (c *Cache) Set(key string, value interface{}) {
	now := nd.Now()
	item := &Item{
		key:      key,
		value:    value,
		accessed: now.Unix(),
		expires:  now.Add(c.ttl).Unix(),
	}
	c.Lock()
	c.lookup[key] = item
	c.Unlock()
}

func (c *Cache) gc() {
	for {
		time.Sleep(c.pruneFrequency)
		for {
			c.RLock()
			l := len(c.lookup)
			c.RUnlock()
			if l > c.maxItems {
				c.prune()
			} else {
				break
			}
		}
	}
}

func (c *Cache) prune() {
	c.RLock()
	l := int32(len(c.lookup))
	found := 0
	for _, item := range c.lookup {
		if rand.Int31n(l) > int32(c.workSize) {
			continue
		}
		c.times[found] = atomic.LoadInt64(&item.accessed)
		found++
		if found == c.workSize {
			break
		}
	}
	c.RUnlock()

	sort.Sort(c.times[:found])
	target := c.times[0]

	c.RLock()
	found = 0
	for key, item := range c.lookup {
		if atomic.LoadInt64(&item.accessed) > target {
			continue
		}
		c.scratch[found] = key
		found++
		if found == c.workSize {
			break
		}
	}
	c.RUnlock()

	c.Lock()
	for _, key := range c.scratch[:found] {
		delete(c.lookup, key)
	}
	c.Unlock()
}
<|endoftext|>


<commit_before>package statsd

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"

	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/net/context"
)

const packetSizeUDP = 1500

// Handler interface can be used to handle metrics and events for a Receiver.
type Handler interface {
	DispatchMetric(context.Context, *types.Metric) error
	DispatchEvent(context.Context, *types.Event) error
}

// RunableHandler extends Handler interface to add the Run method that needs to be executed in a separate
// goroutine to make the handler work.
type RunableHandler interface {
	Handler
	Run(context.Context) error
}

// Receiver receives data on its PacketConn and converts lines into Metrics.
// For each types.Metric it calls Handler.HandleMetric()
type Receiver interface {
	Receive(context.Context, net.PacketConn) error
	GetStats() ReceiverStats
}

// ReceiverStats holds statistics for a Receiver.
type ReceiverStats struct {
	LastPacket      time.Time
	BadLines        uint64
	PacketsReceived uint64
	MetricsReceived uint64
	EventsReceived  uint64
}

type metricReceiver struct {
	// Counter fields below must be read/written only using atomic instructions.
	// 64-bit fields must be the first fields in the struct to guarantee proper memory alignment.
	// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastPacket      int64 // When last packet was received. Unix timestamp in nsec.
	badLines        uint64
	packetsReceived uint64
	metricsReceived uint64
	eventsReceived  uint64
	handler         Handler    // handler to invoke
	namespace       string     // Namespace to prefix all metrics
	tags            types.Tags // Tags to add to all metrics
}

// NewMetricReceiver initialises a new Receiver.
func NewMetricReceiver(ns string, tags []string, handler Handler) Receiver {
	return &metricReceiver{
		handler:   handler,
		namespace: ns,
		tags:      tags,
	}
}

// GetStats returns current Receiver stats. Safe for concurrent use.
func (mr *metricReceiver) GetStats() ReceiverStats {
	return ReceiverStats{
		LastPacket:      time.Unix(0, atomic.LoadInt64(&mr.lastPacket)),
		BadLines:        atomic.LoadUint64(&mr.badLines),
		PacketsReceived: atomic.LoadUint64(&mr.packetsReceived),
		MetricsReceived: atomic.LoadUint64(&mr.metricsReceived),
		EventsReceived:  atomic.LoadUint64(&mr.eventsReceived),
	}
}

// Receive accepts incoming datagrams on c, parses them and calls Handler.DispatchMetric() for each metric
// and Handler.DispatchEvent() for each event.
func (mr *metricReceiver) Receive(ctx context.Context, c net.PacketConn) error {
	buf := make([]byte, packetSizeUDP)
	for {
		// This will error out when the socket is closed.
		nbytes, addr, err := c.ReadFrom(buf)
		if err != nil {
			if netErr, ok := err.(net.Error); ok && !netErr.Temporary() {
				select {
				case <-ctx.Done():
				default:
					return fmt.Errorf("non-temporary error reading from socket: %v", err)
				}
				return nil
			}
			log.Warnf("Error reading from socket: %v", err)
			continue
		}
		// TODO consider updating counter for every N-th iteration to reduce contention
		atomic.AddUint64(&mr.packetsReceived, 1)
		atomic.StoreInt64(&mr.lastPacket, time.Now().UnixNano())
		if err := mr.handlePacket(ctx, addr, buf[:nbytes]); err != nil {
			if err == context.Canceled || err == context.DeadlineExceeded {
				return err
			}
			log.Warnf("Failed to handle packet: %v", err)
		}
	}
}

// handlePacket handles the contents of a datagram and calls Handler.DispatchMetric()
// for each line that successfully parses into a types.Metric and Handler.DispatchEvent() for each event.
func (mr *metricReceiver) handlePacket(ctx context.Context, addr net.Addr, msg []byte) error {
	var numMetrics, numEvents uint16
	var exitError error
	var ip types.IP
	buf := bytes.NewBuffer(msg)
	addrStr := addr.String()
	n := strings.LastIndexByte(addrStr, ':')
	if n <= 0 {
		// This should not ever happen.
		log.Errorf("Cannot parse source address %q", addrStr)
	} else {
		ip = types.IP(addrStr[:n])
	}
	for {
		line, readerr := buf.ReadBytes('\n')

		// protocol does not require line to end in \n, if EOF use received line if valid
		if readerr != nil && readerr != io.EOF {
			exitError = fmt.Errorf("error reading message from %s: %v", addr, readerr)
			break
		} else if readerr != io.EOF {
			// remove newline, only if not EOF
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
		}

		if len(line) > 1 {
			metric, event, err := mr.parseLine(line)
			if err != nil {
				// logging as debug to avoid spamming logs when a bad actor sends
				// badly formatted messages
				log.Debugf("Error parsing line %q from %s: %v", line, addr, err)
				atomic.AddUint64(&mr.badLines, 1)
				continue
			}
			if metric != nil {
				numMetrics++
				metric.Tags = append(metric.Tags, mr.tags...)
				metric.SourceIP = ip
				err = mr.handler.DispatchMetric(ctx, metric)
			} else if event != nil {
				numEvents++
				event.Tags = append(event.Tags, mr.tags...)
				event.SourceIP = ip
				if event.DateHappened == 0 {
					event.DateHappened = time.Now().Unix()
				}
				err = mr.handler.DispatchEvent(ctx, event)
			} else {
				// Should never happen.
				log.Panic("Both event and metric are nil")
			}
			if err != nil {
				exitError = err
				break
			}
		}

		if readerr == io.EOF {
			// if was EOF, finished handling
			break
		}
	}
	atomic.AddUint64(&mr.metricsReceived, uint64(numMetrics))
	atomic.AddUint64(&mr.eventsReceived, uint64(numEvents))
	return exitError
}

// parseLine with lexer impl.
func (mr *metricReceiver) parseLine(line []byte) (*types.Metric, *types.Event, error) {
	l := lexer{}
	return l.run(line, mr.namespace)
}
<commit_msg>Bump max incoming packet size<commit_after>package statsd

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"strings"
	"sync/atomic"
	"time"

	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/net/context"
)

// ip packet size is stored in two bytes and that is how big in theory the packet can be.
// In practice it is highly unlikely but still possible to get packets bigger than usual MTU of 1500.
const packetSizeUDP = 0xffff

// Handler interface can be used to handle metrics and events for a Receiver.
type Handler interface {
	DispatchMetric(context.Context, *types.Metric) error
	DispatchEvent(context.Context, *types.Event) error
}

// RunableHandler extends Handler interface to add the Run method that needs to be executed in a separate
// goroutine to make the handler work.
type RunableHandler interface {
	Handler
	Run(context.Context) error
}

// Receiver receives data on its PacketConn and converts lines into Metrics.
// For each types.Metric it calls Handler.HandleMetric()
type Receiver interface {
	Receive(context.Context, net.PacketConn) error
	GetStats() ReceiverStats
}

// ReceiverStats holds statistics for a Receiver.
type ReceiverStats struct {
	LastPacket      time.Time
	BadLines        uint64
	PacketsReceived uint64
	MetricsReceived uint64
	EventsReceived  uint64
}

type metricReceiver struct {
	// Counter fields below must be read/written only using atomic instructions.
	// 64-bit fields must be the first fields in the struct to guarantee proper memory alignment.
	// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastPacket      int64 // When last packet was received. Unix timestamp in nsec.
	badLines        uint64
	packetsReceived uint64
	metricsReceived uint64
	eventsReceived  uint64
	handler         Handler    // handler to invoke
	namespace       string     // Namespace to prefix all metrics
	tags            types.Tags // Tags to add to all metrics
}

// NewMetricReceiver initialises a new Receiver.
func NewMetricReceiver(ns string, tags []string, handler Handler) Receiver {
	return &metricReceiver{
		handler:   handler,
		namespace: ns,
		tags:      tags,
	}
}

// GetStats returns current Receiver stats. Safe for concurrent use.
func (mr *metricReceiver) GetStats() ReceiverStats {
	return ReceiverStats{
		LastPacket:      time.Unix(0, atomic.LoadInt64(&mr.lastPacket)),
		BadLines:        atomic.LoadUint64(&mr.badLines),
		PacketsReceived: atomic.LoadUint64(&mr.packetsReceived),
		MetricsReceived: atomic.LoadUint64(&mr.metricsReceived),
		EventsReceived:  atomic.LoadUint64(&mr.eventsReceived),
	}
}

// Receive accepts incoming datagrams on c, parses them and calls Handler.DispatchMetric() for each metric
// and Handler.DispatchEvent() for each event.
func (mr *metricReceiver) Receive(ctx context.Context, c net.PacketConn) error {
	buf := make([]byte, packetSizeUDP)
	for {
		// This will error out when the socket is closed.
		nbytes, addr, err := c.ReadFrom(buf)
		if err != nil {
			if netErr, ok := err.(net.Error); ok && !netErr.Temporary() {
				select {
				case <-ctx.Done():
				default:
					return fmt.Errorf("non-temporary error reading from socket: %v", err)
				}
				return nil
			}
			log.Warnf("Error reading from socket: %v", err)
			continue
		}
		// TODO consider updating counter for every N-th iteration to reduce contention
		atomic.AddUint64(&mr.packetsReceived, 1)
		atomic.StoreInt64(&mr.lastPacket, time.Now().UnixNano())
		if err := mr.handlePacket(ctx, addr, buf[:nbytes]); err != nil {
			if err == context.Canceled || err == context.DeadlineExceeded {
				return err
			}
			log.Warnf("Failed to handle packet: %v", err)
		}
	}
}

// handlePacket handles the contents of a datagram and calls Handler.DispatchMetric()
// for each line that successfully parses into a types.Metric and Handler.DispatchEvent() for each event.
func (mr *metricReceiver) handlePacket(ctx context.Context, addr net.Addr, msg []byte) error {
	var numMetrics, numEvents uint16
	var exitError error
	var ip types.IP
	buf := bytes.NewBuffer(msg)
	addrStr := addr.String()
	n := strings.LastIndexByte(addrStr, ':')
	if n <= 0 {
		// This should not ever happen.
		log.Errorf("Cannot parse source address %q", addrStr)
	} else {
		ip = types.IP(addrStr[:n])
	}
	for {
		line, readerr := buf.ReadBytes('\n')

		// protocol does not require line to end in \n, if EOF use received line if valid
		if readerr != nil && readerr != io.EOF {
			exitError = fmt.Errorf("error reading message from %s: %v", addr, readerr)
			break
		} else if readerr != io.EOF {
			// remove newline, only if not EOF
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
		}

		if len(line) > 1 {
			metric, event, err := mr.parseLine(line)
			if err != nil {
				// logging as debug to avoid spamming logs when a bad actor sends
				// badly formatted messages
				log.Debugf("Error parsing line %q from %s: %v", line, addr, err)
				atomic.AddUint64(&mr.badLines, 1)
				continue
			}
			if metric != nil {
				numMetrics++
				metric.Tags = append(metric.Tags, mr.tags...)
				metric.SourceIP = ip
				err = mr.handler.DispatchMetric(ctx, metric)
			} else if event != nil {
				numEvents++
				event.Tags = append(event.Tags, mr.tags...)
				event.SourceIP = ip
				if event.DateHappened == 0 {
					event.DateHappened = time.Now().Unix()
				}
				err = mr.handler.DispatchEvent(ctx, event)
			} else {
				// Should never happen.
				log.Panic("Both event and metric are nil")
			}
			if err != nil {
				exitError = err
				break
			}
		}

		if readerr == io.EOF {
			// if was EOF, finished handling
			break
		}
	}
	atomic.AddUint64(&mr.metricsReceived, uint64(numMetrics))
	atomic.AddUint64(&mr.eventsReceived, uint64(numEvents))
	return exitError
}

// parseLine with lexer impl.
func (mr *metricReceiver) parseLine(line []byte) (*types.Metric, *types.Event, error) {
	l := lexer{}
	return l.run(line, mr.namespace)
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"time"
)

type Path struct {
	Hits  uint16
	First time.Time
}

type Paths map[string]Path

type Storage struct {
	Filename string
	Paths    Paths
}

func NewStorage(filename string) *Storage {
	s := &Storage{Filename: filename}
	setup(filename)

	s.Read()
	return s
}

func (s *Storage) Read() {
	f, _ := ioutil.ReadFile(s.Filename)
	json.Unmarshal(f, &s.Paths)
}

func (s *Storage) Write() {
	j, _ := json.Marshal(s.Paths)
	ioutil.WriteFile(s.Filename, j, 0666)
}

func setup(filename string) {
	dir := path.Dir(filename)
	os.MkdirAll(dir, 0700)
}
<commit_msg>handle nil Paths<commit_after>package storage

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"time"
)

type Path struct {
	Hits  uint16
	First time.Time
}

type Paths map[string]Path

type Storage struct {
	Filename string
	Paths    Paths
}

func NewStorage(filename string) *Storage {
	setup(filename)

	s := &Storage{filename, make(Paths)}
	s.Read()
	return s
}

func (s *Storage) Read() {
	f, _ := ioutil.ReadFile(s.Filename)
	json.Unmarshal(f, &s.Paths)
}

func (s *Storage) Write() {
	j, _ := json.Marshal(s.Paths)
	ioutil.WriteFile(s.Filename, j, 0666)
}

func setup(filename string) {
	dir := path.Dir(filename)
	os.MkdirAll(dir, 0700)
}
<|endoftext|>


<commit_before>package store

import "io"

// Searcher
type Searcher interface {
	// Expr returns a sqlite snippet used in the WHERE clause.
	Expr(string) string
	//Params returns the required parameters for the sqlite snippet.
	Params() []interface{}
}

// Between is a searcher which searches for values between (inclusive) the
// given values.
type Between struct {
	from, to int
}

func (Between) Expr(col string) string {
	return "[" + col + "] BETWEEN ? AND ?"
}

func (b Between) Params() []interface{} {
	return []interface{}{b.from, b.to}
}

// Like implements a search which uses the LIKE syntax in a WHERE clause.
type Like string

func (Like) Expr(col string) string {
	return "[" + col + "] LIKE ?"
}

func (l Like) Params() []interface{} {
	return []interface{}{string(l)}
}

// Search is used for a custom (non primary key) search on a table.
//
// Returns the number of items found and an error if any occurred.
func (s *Store) Search(params map[string]Searcher, offset int, data ...Interface) (int, error) {
	if len(params) == 0 {
		return 0, NoParams{}
	}
	if len(data) == 0 {
		return 0, nil
	}
	table := tableName(data[0])
	cols := data[0].Get()
	vars := make([]string, len(cols))
	first := true
	columns := ""
	for col := range cols {
		if first {
			first = false
		} else {
			columns += ", "
		}
		columns += "[" + col + "]"
		vars = append(vars, col)
	}
	clause := ""
	first = true
	paramVars := make([]interface{}, 0, len(params)+2)
	for col, param := range params {
		if _, ok := cols[col]; !ok {
			return 0, UnknownColumn(col)
		}
		if first {
			first = false
		} else {
			clause += " AND "
		}
		clause += param.Expr(col)
		paramVars = append(paramVars, param.Params()...)
	}
	sql := "SELECT " + columns + " FROM [" + table + "] WHERE " + clause + " LIMIT ? OFFSET ?;"
	stmt, err := s.db.Query(sql, append(paramVars, len(data), offset)...)
	stmtVars := statement{Stmt: stmt, vars: vars}
	pos := 0
	for ; err == nil; err = stmt.Next() {
		if typeName := tableName(data[pos]); typeName != table {
			err = UnmatchedType{table, typeName}
		} else {
			err = stmtVars.Scan(stmtVars.Vars(data[pos].Get())...)
			pos++
		}
	}
	if err != nil && err != io.EOF {
		return pos, err
	}

	return pos, nil
}

//Errors

// NoParams is an error that occurs when no search parameters are given to
// Search.
type NoParams struct{}

func (NoParams) Error() string {
	return "no search parameters given"
}

// UnknownColumn is an error that occurrs when a search parameter requires a
// column which does not exist for the given type.
type UnknownColumn string

func (u UnknownColumn) Error() string {
	return "unknown column: " + string(u)
}
<commit_msg>Got search code working<commit_after>package store

import "io"

// Searcher
type Searcher interface {
	// expr returns a sqlite snippet used in the WHERE clause.
	expr() string
	// params returns the required parameters for the sqlite snippet.
	params() []interface{}
	// column returns the column name in the table
	column() string
}

// Between is a searcher which searches for values between (inclusive) the
// given values.
type between struct {
	col      string
	from, to int
}

// Between returns a Searcher than looks for
func Between(column string, from, to int) Searcher {
	return &between{column, from, to}
}

func (b *between) expr() string {
	return "[" + b.col + "] BETWEEN ? AND ?"
}

func (b *between) params() []interface{} {
	return []interface{}{b.from, b.to}
}

func (b *between) column() string {
	return b.col
}

// Like implements a search which uses the LIKE syntax in a WHERE clause.
type like struct {
	col, likeStr string
}

func Like(column, likeStr string) Searcher {
	return &like{column, likeStr}
}

func (l *like) expr() string {
	return "[" + l.col + "] LIKE ?"
}

func (l *like) params() []interface{} {
	return []interface{}{l.likeStr}
}

func (l *like) column() string {
	return l.col
}

// Search is used for a custom (non primary key) search on a table.
//
// Returns the number of items found and an error if any occurred.
func (s *Store) Search(data []Interface, offset int, params ...Searcher) (int, error) {
	if len(params) == 0 {
		return 0, NoParams{}
	}
	if len(data) == 0 {
		return 0, nil
	}
	table := tableName(data[0])
	cols := data[0].Get()
	vars := make([]string, 0, len(cols))
	first := true
	columns := ""
	for col := range cols {
		if first {
			first = false
		} else {
			columns += ", "
		}
		columns += "[" + col + "]"
		vars = append(vars, col)
	}
	clause := ""
	first = true
	paramVars := make([]interface{}, 0, len(params)+2)
	for _, param := range params {
		col := param.column()
		if _, ok := cols[col]; !ok {
			return 0, UnknownColumn(col)
		}
		if first {
			first = false
		} else {
			clause += " AND "
		}
		clause += param.expr()
		paramVars = append(paramVars, param.params()...)
	}
	sql := "SELECT " + columns + " FROM [" + table + "] WHERE " + clause + " ORDER BY [" + data[0].Key() + "] LIMIT ? OFFSET ?;"
	stmt, err := s.db.Query(sql, append(paramVars, len(data), offset)...)
	stmtVars := statement{Stmt: stmt, vars: vars}
	pos := 0
	for ; err == nil; err = stmt.Next() {
		if typeName := tableName(data[pos]); typeName != table {
			err = UnmatchedType{table, typeName}
		} else {
			err = stmtVars.Scan(stmtVars.Vars(data[pos].Get())...)
			pos++
		}
	}
	if err != nil && err != io.EOF {
		return pos, err
	}

	return pos, nil
}

//Errors

// NoParams is an error that occurs when no search parameters are given to
// Search.
type NoParams struct{}

func (NoParams) Error() string {
	return "no search parameters given"
}

// UnknownColumn is an error that occurrs when a search parameter requires a
// column which does not exist for the given type.
type UnknownColumn string

func (u UnknownColumn) Error() string {
	return "unknown column: " + string(u)
}
<|endoftext|>


<commit_before>package secure

import (
	"fmt"
	"net/http"
	"strings"
)

const (
	stsHeader           = "Strict-Transport-Security"
	stsSubdomainString  = "; includeSubdomains"
	stsPreloadString    = "; preload"
	frameOptionsHeader  = "X-Frame-Options"
	frameOptionsValue   = "DENY"
	contentTypeHeader   = "X-Content-Type-Options"
	contentTypeValue    = "nosniff"
	xssProtectionHeader = "X-XSS-Protection"
	xssProtectionValue  = "1; mode=block"
	cspHeader           = "Content-Security-Policy"
)

func defaultBadHostHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Bad Host", http.StatusInternalServerError)
}

// Options is a struct for specifying configuration options for the secure.Secure middleware.
type Options struct {
	// AllowedHosts is a list of fully qualified domain names that are allowed. Default is empty list, which allows any and all host names.
	AllowedHosts []string
	// If SSLRedirect is set to true, then only allow https requests. Default is false.
	SSLRedirect bool
	// If SSLTemporaryRedirect is true, the a 302 will be used while redirecting. Default is false (301).
	SSLTemporaryRedirect bool
	// SSLHost is the host name that is used to redirect http requests to https. Default is "", which indicates to use the same host.
	SSLHost string
	// SSLProxyHeaders is set of header keys with associated values that would indicate a valid https request. Useful when using Nginx: `map[string]string{"X-Forwarded-Proto": "https"}`. Default is blank map.
	SSLProxyHeaders map[string]string
	// STSSeconds is the max-age of the Strict-Transport-Security header. Default is 0, which would NOT include the header.
	STSSeconds int64
	// If STSIncludeSubdomains is set to true, the `includeSubdomains` will be appended to the Strict-Transport-Security header. Default is false.
	STSIncludeSubdomains bool
	// If STSPreload is set to true, the `preload` flag will be appended to the Strict-Transport-Security header. Default is false.
	STSPreload bool
	// If ForceSTSHeader is set to true, the STS header will be added even when the connection is HTTP. Default is false.
	ForceSTSHeader bool
	// If FrameDeny is set to true, adds the X-Frame-Options header with the value of `DENY`. Default is false.
	FrameDeny bool
	// CustomFrameOptionsValue allows the X-Frame-Options header value to be set with a custom value. This overrides the FrameDeny option.
	CustomFrameOptionsValue string
	// If ContentTypeNosniff is true, adds the X-Content-Type-Options header with the value `nosniff`. Default is false.
	ContentTypeNosniff bool
	// If BrowserXssFilter is true, adds the X-XSS-Protection header with the value `1; mode=block`. Default is false.
	BrowserXssFilter bool
	// ContentSecurityPolicy allows the Content-Security-Policy header value to be set with a custom value. Default is "".
	ContentSecurityPolicy string
	// When developing, the AllowedHosts, SSL, and STS options can cause some unwanted effects. Usually testing happens on http, not https, and on localhost, not your production domain... so set this to true for dev environment.
	// If you would like your development environment to mimic production with complete Host blocking, SSL redirects, and STS headers, leave this as false. Default if false.
	IsDevelopment bool
}

// Secure is a middleware that helps setup a few basic security features. A single secure.Options struct can be
// provided to configure which features should be enabled, and the ability to override a few of the default values.
type Secure struct {
	// Customize Secure with an Options struct.
	opt Options

	// Handlers for when an error occurs (ie bad host).
	badHostHandler http.Handler
}

// New constructs a new Secure instance with supplied options.
func New(options ...Options) *Secure {
	var o Options
	if len(options) == 0 {
		o = Options{}
	} else {
		o = options[0]
	}

	return &Secure{
		opt:            o,
		badHostHandler: http.HandlerFunc(defaultBadHostHandler),
	}
}

// SetBadHostHandler sets the handler to call when secure rejects the host name.
func (s *Secure) SetBadHostHandler(handler http.Handler) {
	s.badHostHandler = handler
}

// Handler implements the http.HandlerFunc for integration with the standard net/http lib.
func (s *Secure) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Let secure process the request. If it returns an error,
		// that indicates the request should not continue.
		err := s.Process(w, r)

		// If there was an error, do not continue.
		if err != nil {
			return
		}

		h.ServeHTTP(w, r)
	})
}

// HandlerFuncWithNext is a special implementation for Negroni, but could be used elsewhere.
func (s *Secure) HandlerFuncWithNext(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	err := s.Process(w, r)

	// If there was an error, do not call next.
	if err == nil && next != nil {
		next(w, r)
	}
}

// Process runs the actual checks and returns an error if the middleware chain should stop.
func (s *Secure) Process(w http.ResponseWriter, r *http.Request) error {
	// Allowed hosts check.
	if len(s.opt.AllowedHosts) > 0 && !s.opt.IsDevelopment {
		isGoodHost := false
		for _, allowedHost := range s.opt.AllowedHosts {
			if strings.EqualFold(allowedHost, r.Host) {
				isGoodHost = true
				break
			}
		}

		if !isGoodHost {
			s.badHostHandler.ServeHTTP(w, r)
			return fmt.Errorf("Bad host name: %s", r.Host)
		}
	}

	// Determine if we are on HTTPS.
	isSSL := false
	if strings.EqualFold(r.URL.Scheme, "https") || r.TLS != nil {
		isSSL = true
	} else {
		for k, v := range s.opt.SSLProxyHeaders {
			if r.Header.Get(k) == v {
				isSSL = true
				break
			}
		}
	}

	// SSL check.
	if s.opt.SSLRedirect && !isSSL && !s.opt.IsDevelopment {
		url := r.URL
		url.Scheme = "https"
		url.Host = r.Host

		if len(s.opt.SSLHost) > 0 {
			url.Host = s.opt.SSLHost
		}

		status := http.StatusMovedPermanently
		if s.opt.SSLTemporaryRedirect {
			status = http.StatusTemporaryRedirect
		}

		http.Redirect(w, r, url.String(), status)
		return fmt.Errorf("Redirecting to HTTPS")
	}

	// Strict Transport Security header. Only add header when we know it's an SSL connection.
	// See https://tools.ietf.org/html/rfc6797#section-7.2 for details.
	if s.opt.STSSeconds != 0 && (isSSL || s.opt.ForceSTSHeader) && !s.opt.IsDevelopment {
		stsSub := ""
		if s.opt.STSIncludeSubdomains {
			stsSub = stsSubdomainString
		}

		if s.opt.STSPreload {
			stsSub += stsPreloadString
		}

		w.Header().Add(stsHeader, fmt.Sprintf("max-age=%d%s", s.opt.STSSeconds, stsSub))
	}

	// Frame Options header.
	if len(s.opt.CustomFrameOptionsValue) > 0 {
		w.Header().Add(frameOptionsHeader, s.opt.CustomFrameOptionsValue)
	} else if s.opt.FrameDeny {
		w.Header().Add(frameOptionsHeader, frameOptionsValue)
	}

	// Content Type Options header.
	if s.opt.ContentTypeNosniff {
		w.Header().Add(contentTypeHeader, contentTypeValue)
	}

	// XSS Protection header.
	if s.opt.BrowserXssFilter {
		w.Header().Add(xssProtectionHeader, xssProtectionValue)
	}

	// Content Security Policy header.
	if len(s.opt.ContentSecurityPolicy) > 0 {
		w.Header().Add(cspHeader, s.opt.ContentSecurityPolicy)
	}

	return nil
}
<commit_msg>added HPKP<commit_after>package secure

import (
	"fmt"
	"net/http"
	"strings"
)

const (
	stsHeader           = "Strict-Transport-Security"
	stsSubdomainString  = "; includeSubdomains"
	stsPreloadString    = "; preload"
	frameOptionsHeader  = "X-Frame-Options"
	frameOptionsValue   = "DENY"
	contentTypeHeader   = "X-Content-Type-Options"
	contentTypeValue    = "nosniff"
	xssProtectionHeader = "X-XSS-Protection"
	xssProtectionValue  = "1; mode=block"
	cspHeader           = "Content-Security-Policy"
	hpkpHeader          = "Public-Key-Pins"
)

func defaultBadHostHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Bad Host", http.StatusInternalServerError)
}

// Options is a struct for specifying configuration options for the secure.Secure middleware.
type Options struct {
	// AllowedHosts is a list of fully qualified domain names that are allowed. Default is empty list, which allows any and all host names.
	AllowedHosts []string
	// If SSLRedirect is set to true, then only allow https requests. Default is false.
	SSLRedirect bool
	// If SSLTemporaryRedirect is true, the a 302 will be used while redirecting. Default is false (301).
	SSLTemporaryRedirect bool
	// SSLHost is the host name that is used to redirect http requests to https. Default is "", which indicates to use the same host.
	SSLHost string
	// SSLProxyHeaders is set of header keys with associated values that would indicate a valid https request. Useful when using Nginx: `map[string]string{"X-Forwarded-Proto": "https"}`. Default is blank map.
	SSLProxyHeaders map[string]string
	// STSSeconds is the max-age of the Strict-Transport-Security header. Default is 0, which would NOT include the header.
	STSSeconds int64
	// If STSIncludeSubdomains is set to true, the `includeSubdomains` will be appended to the Strict-Transport-Security header. Default is false.
	STSIncludeSubdomains bool
	// If STSPreload is set to true, the `preload` flag will be appended to the Strict-Transport-Security header. Default is false.
	STSPreload bool
	// If ForceSTSHeader is set to true, the STS header will be added even when the connection is HTTP. Default is false.
	ForceSTSHeader bool
	// If FrameDeny is set to true, adds the X-Frame-Options header with the value of `DENY`. Default is false.
	FrameDeny bool
	// CustomFrameOptionsValue allows the X-Frame-Options header value to be set with a custom value. This overrides the FrameDeny option.
	CustomFrameOptionsValue string
	// If ContentTypeNosniff is true, adds the X-Content-Type-Options header with the value `nosniff`. Default is false.
	ContentTypeNosniff bool
	// If BrowserXSSFilter is true, adds the X-XSS-Protection header with the value `1; mode=block`. Default is false.
	BrowserXSSFilter bool
	// ContentSecurityPolicy allows the Content-Security-Policy header value to be set with a custom value. Default is "".
	ContentSecurityPolicy string
	// PublicKey implements HPKP to prevent MITM attacks with forged certificates. Default is []string.
	PublicKey string
	// When developing, the AllowedHosts, SSL, and STS options can cause some unwanted effects. Usually testing happens on http, not https, and on localhost, not your production domain... so set this to true for dev environment.
	// If you would like your development environment to mimic production with complete Host blocking, SSL redirects, and STS headers, leave this as false. Default if false.
	IsDevelopment bool
}

// Secure is a middleware that helps setup a few basic security features. A single secure.Options struct can be
// provided to configure which features should be enabled, and the ability to override a few of the default values.
type Secure struct {
	// Customize Secure with an Options struct.
	opt Options

	// Handlers for when an error occurs (ie bad host).
	badHostHandler http.Handler
}

var defaultHeader = make(http.Header)

// New constructs a new Secure instance with supplied options.
func New(options ...Options) *Secure {
	var o Options
	if len(options) == 0 {
		o = Options{}
	} else {
		o = options[0]
	}

	return &Secure{
		opt:            o,
		badHostHandler: http.HandlerFunc(defaultBadHostHandler),
	}
}

// SetBadHostHandler sets the handler to call when secure rejects the host name.
func (s *Secure) SetBadHostHandler(handler http.Handler) {
	s.badHostHandler = handler
}

// Handler implements the http.HandlerFunc for integration with the standard net/http lib.
func (s *Secure) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Let secure process the request. If it returns an error,
		// that indicates the request should not continue.
		err := s.Process(w, r)

		// If there was an error, do not continue.
		if err != nil {
			return
		}

		h.ServeHTTP(w, r)
	})
}

// HandlerFuncWithNext is a special implementation for Negroni, but could be used elsewhere.
func (s *Secure) HandlerFuncWithNext(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	err := s.Process(w, r)

	// If there was an error, do not call next.
	if err == nil && next != nil {
		next(w, r)
	}
}

// Process runs the actual checks and returns an error if the middleware chain should stop.
func (s *Secure) Process(w http.ResponseWriter, r *http.Request) error {
	// Allowed hosts check.
	if len(s.opt.AllowedHosts) > 0 && !s.opt.IsDevelopment {
		isGoodHost := false
		for _, allowedHost := range s.opt.AllowedHosts {
			if strings.EqualFold(allowedHost, r.Host) {
				isGoodHost = true
				break
			}
		}

		if !isGoodHost {
			s.badHostHandler.ServeHTTP(w, r)
			return fmt.Errorf("Bad host name: %s", r.Host)
		}
	}

	// Determine if we are on HTTPS.
	isSSL := strings.EqualFold(r.URL.Scheme, "https") || r.TLS != nil
	if !isSSL {
		for k, v := range s.opt.SSLProxyHeaders {
			if r.Header.Get(k) == v {
				isSSL = true
				break
			}
		}
	}

	// SSL check.
	if s.opt.SSLRedirect && !isSSL && !s.opt.IsDevelopment {
		url := r.URL
		url.Scheme = "https"
		url.Host = r.Host

		if len(s.opt.SSLHost) > 0 {
			url.Host = s.opt.SSLHost
		}

		status := http.StatusMovedPermanently
		if s.opt.SSLTemporaryRedirect {
			status = http.StatusTemporaryRedirect
		}

		http.Redirect(w, r, url.String(), status)
		return fmt.Errorf("Redirecting to HTTPS")
	}

	// Strict Transport Security header. Only add header when we know it's an SSL connection.
	// See https://tools.ietf.org/html/rfc6797#section-7.2 for details.
	if s.opt.STSSeconds != 0 && (isSSL || s.opt.ForceSTSHeader) && !s.opt.IsDevelopment {
		stsSub := ""
		if s.opt.STSIncludeSubdomains {
			stsSub = stsSubdomainString
		}

		if s.opt.STSPreload {
			stsSub += stsPreloadString
		}

		w.Header().Add(stsHeader, fmt.Sprintf("max-age=%d%s", s.opt.STSSeconds, stsSub))
	}

	// Frame Options header.
	if len(s.opt.CustomFrameOptionsValue) > 0 {
		w.Header().Add(frameOptionsHeader, s.opt.CustomFrameOptionsValue)
	} else if s.opt.FrameDeny {
		w.Header().Add(frameOptionsHeader, frameOptionsValue)
	}

	// Content Type Options header.
	if s.opt.ContentTypeNosniff {
		w.Header().Add(contentTypeHeader, contentTypeValue)
	}

	// XSS Protection header.
	if s.opt.BrowserXSSFilter {
		w.Header().Add(xssProtectionHeader, xssProtectionValue)
	}

	if len(s.opt.PublicKey) > 0 {
		w.Header().Add(hpkpHeader, s.opt.PublicKey)
	}

	// Content Security Policy header.
	if len(s.opt.ContentSecurityPolicy) > 0 {
		w.Header().Add(cspHeader, s.opt.ContentSecurityPolicy)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"

	"github.com/ar-maged/guc-api/factory"
	"github.com/ar-maged/guc-api/graphql"
	"github.com/ar-maged/guc-api/util"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/graphql-go/handler"
)

func main() {
	router := mux.NewRouter()

	router.HandleFunc("/api/login", loginHandler).Methods("GET")
	router.HandleFunc("/api/coursework", courseworkHandler).Methods("GET")
	router.HandleFunc("/api/midterms", midtermsHandler).Methods("GET")
	router.HandleFunc("/api/attendance", attendanceHandler).Methods("GET")
	router.HandleFunc("/api/exams", examsHandler).Methods("GET")
	router.HandleFunc("/api/schedule", scheduleHandler).Methods("GET")

	router.Handle("/graphql", handler.New(&handler.Config{
		Schema:   &graphql.Schema,
		Pretty:   true,
		GraphiQL: true,
	}))

	http.Handle("/", router)

	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}

	allowedHeaders := handlers.AllowedHeaders([]string{"Authorization"})
	allowedOrigins := handlers.AllowedOrigins([]string{"*"})
	allowedMethods := handlers.AllowedMethods([]string{"GET", "POST", "OPTIONS"})

	fmt.Printf("Server listening on port %s...\n", port)
	http.ListenAndServe(":"+port, handlers.CORS(allowedHeaders, allowedOrigins, allowedMethods)(router))
}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		sendDataJSONResponse(w, factory.IsUserAuthorized(username, password))
	}
}

func courseworkHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if coursework, err := factory.GetUserCoursework(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, coursework)
		}
	}
}

func midtermsHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if midterms, err := factory.GetUserMidterms(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, midterms)
		}
	}
}

func attendanceHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if reports, err := factory.GetUserAbsenceReports(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, reports)
		}
	}
}

func examsHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if exams, err := factory.GetUserExams(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, exams)
		}
	}
}

func scheduleHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if schedules, err := factory.GetUserSchedule(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, schedules)
		}
	}
}

func sendUnauthorizedJSONResponse(w http.ResponseWriter, err error) {
	w.WriteHeader(http.StatusUnauthorized)
	util.SendJSONResponse(w, factory.ResponseAPI{Error: err.Error(), Data: nil})
}

func sendDataJSONResponse(w http.ResponseWriter, data interface{}) {
	util.SendJSONResponse(w, factory.ResponseAPI{Error: nil, Data: data})
}
<commit_msg>:bug: Fix CORS<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"

	"github.com/ar-maged/guc-api/factory"
	"github.com/ar-maged/guc-api/graphql"
	"github.com/ar-maged/guc-api/util"
	"github.com/gorilla/mux"
	"github.com/graphql-go/handler"
	"github.com/rs/cors"
)

func main() {
	router := mux.NewRouter()

	router.HandleFunc("/api/login", loginHandler).Methods("GET")
	router.HandleFunc("/api/coursework", courseworkHandler).Methods("GET")
	router.HandleFunc("/api/midterms", midtermsHandler).Methods("GET")
	router.HandleFunc("/api/attendance", attendanceHandler).Methods("GET")
	router.HandleFunc("/api/exams", examsHandler).Methods("GET")
	router.HandleFunc("/api/schedule", scheduleHandler).Methods("GET")

	router.Handle("/graphql", handler.New(&handler.Config{
		Schema:   &graphql.Schema,
		Pretty:   true,
		GraphiQL: true,
	}))

	http.Handle("/", router)

	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}

	corsOptions := cors.New(cors.Options{
		AllowedOrigins:   []string{"*"},
		AllowedHeaders:   []string{"*"},
		AllowedMethods:   []string{"GET", "POST", "OPTIONS"},
		AllowCredentials: true,
	})

	fmt.Printf("Server listening on port %s...\n", port)
	http.ListenAndServe(":"+port, corsOptions.Handler(router))
}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		sendDataJSONResponse(w, factory.IsUserAuthorized(username, password))
	}
}

func courseworkHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if coursework, err := factory.GetUserCoursework(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, coursework)
		}
	}
}

func midtermsHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if midterms, err := factory.GetUserMidterms(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, midterms)
		}
	}
}

func attendanceHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if reports, err := factory.GetUserAbsenceReports(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, reports)
		}
	}
}

func examsHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if exams, err := factory.GetUserExams(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, exams)
		}
	}
}

func scheduleHandler(w http.ResponseWriter, r *http.Request) {
	if username, password, err := util.BasicAuthentication(r); err != nil {
		sendUnauthorizedJSONResponse(w, err)
	} else {
		if schedules, err := factory.GetUserSchedule(username, password); err != nil {
			sendUnauthorizedJSONResponse(w, err)
		} else {
			sendDataJSONResponse(w, schedules)
		}
	}
}

func sendUnauthorizedJSONResponse(w http.ResponseWriter, err error) {
	w.WriteHeader(http.StatusUnauthorized)
	util.SendJSONResponse(w, factory.ResponseAPI{Error: err.Error(), Data: nil})
}

func sendDataJSONResponse(w http.ResponseWriter, data interface{}) {
	util.SendJSONResponse(w, factory.ResponseAPI{Error: nil, Data: data})
}
<|endoftext|>


<commit_before>package webapi

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strconv"

	"github.com/gophergala2016/goad"
	"github.com/gophergala2016/goad/sqsadaptor"
	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", ":8080", "http service address")
var upgrader = websocket.Upgrader{}

func jsonFromRegionsAggData(result sqsadaptor.RegionsAggData) (string, error) {
	data, jsonerr := json.Marshal(result)
	if jsonerr != nil {
		return "", jsonerr
	}
	return string(data), nil
}

func serveResults(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/goad" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	url := r.URL.Query().Get("url")
	if len(url) == 0 {
		http.Error(w, "Missing URL", 400)
		return
	}

	concurrencyStr := r.URL.Query().Get("c")
	concurrency, cerr := strconv.Atoi(concurrencyStr)
	if cerr != nil {
		http.Error(w, "Invalid concurrency", 400)
		return
	}

	totStr := r.URL.Query().Get("tot")
	tot, toterr := strconv.Atoi(totStr)
	if toterr != nil {
		http.Error(w, "Invalid total", 400)
		return
	}

	timeoutStr := r.URL.Query().Get("timeout")
	timeout, timeouterr := strconv.Atoi(timeoutStr)
	if timeouterr != nil {
		http.Error(w, "Invalid timeout", 400)
		return
	}

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("Websocket upgrade:", err)
		return
	}
	defer c.Close()

	resultChan := make(chan sqsadaptor.RegionsAggData)

	config := TestConfig{
		url,
		concurrency,
		tot,
		timeout,
		"",
	}

	test := goad.NewTest(config)
	resultChan := test.Start()

	for {
		result, more := <-resultChan
		if !more {
			break
		}

		message, jsonerr := jsonFromRegionsAggData(result)
		if jsonerr != nil {
			log.Println(jsonerr)
			break
		}
		err = c.WriteMessage(websocket.TextMessage, []byte(message))
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

// Serve waits for connections and serves the results
func Serve() {
	http.HandleFunc("/goad", serveResults)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Add read loop, various fixes<commit_after>package webapi

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/gophergala2016/goad"
	"github.com/gophergala2016/goad/sqsadaptor"
	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", ":8080", "http service address")
var upgrader = websocket.Upgrader{}

func jsonFromRegionsAggData(result sqsadaptor.RegionsAggData) (string, error) {
	data, jsonerr := json.Marshal(result)
	if jsonerr != nil {
		return "", jsonerr
	}
	return string(data), nil
}

func serveResults(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/goad" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	url := r.URL.Query().Get("url")
	if len(url) == 0 {
		http.Error(w, "Missing URL", 400)
		return
	}

	concurrencyStr := r.URL.Query().Get("c")
	concurrency, cerr := strconv.Atoi(concurrencyStr)
	if cerr != nil {
		http.Error(w, "Invalid concurrency", 400)
		return
	}

	totStr := r.URL.Query().Get("tot")
	tot, toterr := strconv.Atoi(totStr)
	if toterr != nil {
		http.Error(w, "Invalid total", 400)
		return
	}

	timeoutStr := r.URL.Query().Get("timeout")
	timeout, timeouterr := strconv.Atoi(timeoutStr)
	if timeouterr != nil {
		http.Error(w, "Invalid timeout", 400)
		return
	}

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("Websocket upgrade:", err)
		return
	}
	defer c.Close()

	config := goad.TestConfig{
		url,
		uint(concurrency),
		uint(tot),
		time.Duration(timeout),
		"eu-west-1",
	}

	test := goad.NewTest(&config)
	resultChan := test.Start()

	for result := range resultChan {
		message, jsonerr := jsonFromRegionsAggData(result)
		if jsonerr != nil {
			log.Println(jsonerr)
			break
		}
		go readLoop(c)
		err = c.WriteMessage(websocket.TextMessage, []byte(message))
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

func readLoop(c *websocket.Conn) {
	for {
		if _, _, err := c.NextReader(); err != nil {
			c.Close()
			break
		}
	}
}

// Serve waits for connections and serves the results
func Serve() {
	http.HandleFunc("/goad", serveResults)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	"github.com/gin-gonic/gin"
	"github.com/jmcvetta/neoism"
)

//DB stock the Database variable
var DB *neoism.Database

//APIGetAllLines return All lines of bus
func APIGetAllLines(c *gin.Context) {
	lines, err := GetAllLines()
	if err != nil {
		return
	}
	if value, ok := c.GetQuery("format"); ok == true && value == "json" {
		c.JSON(http.StatusOK, lines)
		return
	}
	content, err := json.Marshal(lines)
	if err != nil {
		return
	}
	c.String(http.StatusOK, string(content))
}

//APIGetStopsForLine return all stops for lineID
func APIGetStopsForLine(c *gin.Context) {
	lineID := c.Params.ByName("lineID")

	stops, err := GetStopsFromLineID(lineID)
	if err != nil {
		return
	}
	if value, ok := c.GetQuery("format"); ok == true && value == "json" {
		c.JSON(http.StatusOK, stops)
		return
	}
	content, err := json.Marshal(stops)
	if err != nil {
		return
	}
	c.String(http.StatusOK, string(content))
}

func init() {
	var err error

	protocol := os.Getenv("NEO4J_PROT")
	user := os.Getenv("NEO4J_USER")
	password := os.Getenv("NEO4J_PASSWD")
	host := os.Getenv("NEO4J_HOST")
	port := os.Getenv("NEO4J_PORT")
	DB, err = neoism.Connect(fmt.Sprintf("%s://%s:%s@%s:%s", protocol, user, password, host, port))
	if err != nil {
		return
	}
}

func main() {
	app := gin.Default()

	app.GET("/", APIGetAllLines)
	app.GET("/:lineID", APIGetStopsForLine)
	app.Run(fmt.Sprintf(":%s", os.Getenv("API_PORT")))
	app.NoRoute(func(c *gin.Context) {
		c.JSON(http.StatusNotFound, gin.H{"code": "404", "message": "Page not found"})
	})
}
<commit_msg>appending strings better<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"os"

	"github.com/gin-gonic/gin"
	"github.com/jmcvetta/neoism"
)

//DB stock the Database variable
var DB *neoism.Database

//APIGetAllLines return All lines of bus
func APIGetAllLines(c *gin.Context) {
	lines, err := GetAllLines()
	if err != nil {
		return
	}
	if value, ok := c.GetQuery("format"); ok == true && value == "json" {
		c.JSON(http.StatusOK, lines)
		return
	}
	content, err := json.Marshal(lines)
	if err != nil {
		return
	}
	c.String(http.StatusOK, string(content))
}

//APIGetStopsForLine return all stops for lineID
func APIGetStopsForLine(c *gin.Context) {
	lineID := c.Params.ByName("lineID")

	stops, err := GetStopsFromLineID(lineID)
	if err != nil {
		return
	}
	if value, ok := c.GetQuery("format"); ok == true && value == "json" {
		c.JSON(http.StatusOK, stops)
		return
	}
	content, err := json.Marshal(stops)
	if err != nil {
		return
	}
	c.String(http.StatusOK, string(content))
}

func init() {
	var err error

	protocol := os.Getenv("NEO4J_PROT")
	user := os.Getenv("NEO4J_USER")
	password := os.Getenv("NEO4J_PASSWD")
	host := os.Getenv("NEO4J_HOST")
	port := os.Getenv("NEO4J_PORT")
	DB, err = neoism.Connect(protocol + "://" + user + ":" + password + "@" + host + ":" + port)
	if err != nil {
		return
	}
}

func main() {
	app := gin.Default()

	app.GET("/", APIGetAllLines)
	app.GET("/:lineID", APIGetStopsForLine)
	app.Run(":" + os.Getenv("API_PORT"))
	app.NoRoute(func(c *gin.Context) {
		c.JSON(http.StatusNotFound, gin.H{"code": "404", "message": "Page not found"})
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 Bruno Albuquerque (bga@bug-br.org.br).
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package main

import (
	"./guestbook"
	"./webservice"

	"github.com/codegangsta/martini"
)

func main() {
	martiniClassic := martini.Classic()
	guestBook := guestbook.NewGuestBook()
	webservice.RegisterWebService(guestBook, martiniClassic)
	martiniClassic.Run()
}
<commit_msg>Fix import paths.<commit_after>// Copyright 2013 Bruno Albuquerque (bga@bug-br.org.br).
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package main

import (
	"github.com/brunoga/go-webservice-sample/guestbook"
	"github.com/brunoga/go-webservice-sample/webservice"
	"github.com/codegangsta/martini"
)

func main() {
	martiniClassic := martini.Classic()
	guestBook := guestbook.NewGuestBook()
	webservice.RegisterWebService(guestBook, martiniClassic)
	martiniClassic.Run()
}
<|endoftext|>