<commit_before>// Package words provides a scanner for Unicode text segmentation word boundaries: https://unicode.org/reports/tr29/#Word_Boundaries
// It does not implement the entire specification, but many of the most important rules.
package words

import (
	"bufio"
	"io"
	"unicode"

	"github.com/clipperhouse/uax29/emoji"
)

// NewScanner tokenizes a reader into a stream of tokens. Iterate through the stream by calling Scan() or Next().
//
// Its uses several specs from Unicode Text Segmentation word boundaries https://unicode.org/reports/tr29/#Word_Boundaries. It's not a full implementation, but a decent approximation for many mainstream cases.
//
// It returns all tokens (including white space), so text can be reconstructed with fidelity.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		incoming: bufio.NewReaderSize(r, 64*1024),
		sot:      true,
	}
}

// Scanner is the structure for scanning an input Reader. Use NewScanner to instantiate.
type Scanner struct {
	incoming *bufio.Reader
	buffer   []rune

	text string
	err  error

	// sot indicates "start of text": https://unicode.org/reports/tr29/#WB1
	sot bool
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
func (sc *Scanner) Scan() bool {
	for {
		current, eof, err := sc.readRune()
		if err != nil {
			sc.err = err
			return false
		}

		// Some funcs below require lookahead; better to do I/O here than there
		// (we don't care about eof for lookahead, irrelevant)
		lookahead, _, err := sc.peekRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case sc.WB1(eof):
			// true indicates continue
			sc.sot = false
			sc.accept(current)
			continue
		case sc.WB2(eof):
			// true indicates break
			sc.text = sc.token()
			sc.err = nil
			return sc.text != ""
		case
			sc.WB3(current):
			// true indicates continue
			sc.accept(current)
			continue
		case
			sc.WB3a(current),
			sc.WB3b(current):
			// true indicates break
			goto breaking
		case
			sc.WB3d(current),
			sc.WB5(current),
			sc.WB6(current, lookahead),
			sc.WB7(current),
			sc.WB7a(current),
			sc.WB7b(current, lookahead),
			sc.WB7c(current),
			sc.WB8(current),
			sc.WB9(current),
			sc.WB10(current),
			sc.WB11(current),
			sc.WB12(current, lookahead),
			sc.WB13(current):
			// true indicates continue
			sc.accept(current)
			continue
		}

	breaking:
		// If we fall through all the above rules, it's a word break
		// WB999 implements https://unicode.org/reports/tr29/#WB999

		if len(sc.buffer) > 0 {
			sc.text = sc.token()
			sc.err = nil
			sc.accept(current)
			return true
		}

		sc.accept(current)
		continue
	}
}

// Text returns the current token, given a successful call to Scan
func (sc *Scanner) Text() string {
	return sc.text
}

// Err returns the current error, given an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// Word boundary rules: https://unicode.org/reports/tr29/#Word_Boundaries
// Typically they take the form of Category1 × Category2; × means don't break between runes of these categories.
// The funcs below test the 'left' side first, when len(buffer) == 0, i.e. beginning of token.
// Then, they test the 'right' side, if something is already in the buffer.
// In most cases, returning true means 'keep going'.

var is = unicode.Is

// WB1 implements https://unicode.org/reports/tr29/#WB1
func (sc *Scanner) WB1(eof bool) (continues bool) {
	// A bit silly, but reads consistently in Scan above
	return sc.sot && !eof
}

// WB2 implements https://unicode.org/reports/tr29/#WB2
func (sc *Scanner) WB2(eof bool) (breaks bool) {
	// A bit silly, but reads consistently in Scan above
	return eof
}

// WB3 implements https://unicode.org/reports/tr29/#WB3
func (sc *Scanner) WB3(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(CR, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) && is(LF, current)
}

// WB3a implements https://unicode.org/reports/tr29/#WB3a
func (sc *Scanner) WB3a(current rune) (breaks bool) {
	if len(sc.buffer) == 0 {
		return is(CR, current) || is(LF, current) || is(Newline, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) || is(LF, previous) || is(Newline, previous)
}

// WB3b implements https://unicode.org/reports/tr29/#WB3b
func (sc *Scanner) WB3b(current rune) (breaks bool) {
	return is(CR, current) || is(LF, current) || is(Newline, current)
}

// WB3c implements https://unicode.org/reports/tr29/#WB3c
func (sc *Scanner) WB3c(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(ZWJ, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(ZWJ, previous) && is(emoji.Extended_Pictographic, current)
}

// WB3d implements https://unicode.org/reports/tr29/#WB3d
func (sc *Scanner) WB3d(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(WSegSpace, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(WSegSpace, previous) && is(WSegSpace, current)
}

// WB5 implements https://unicode.org/reports/tr29/#WB5
func (sc *Scanner) WB5(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(AHLetter, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(AHLetter, current)
}

// WB6 implements https://unicode.org/reports/tr29/#WB6
func (sc *Scanner) WB6(current, lookahead rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(AHLetter, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && (is(MidLetter, current) || is(MidNumLetQ, current)) && is(AHLetter, lookahead)
}

// WB7 implements https://unicode.org/reports/tr29/#WB7
func (sc *Scanner) WB7(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(AHLetter, preprevious) && (is(MidLetter, previous) || is(MidNumLetQ, previous)) && is(AHLetter, current)
}

// WB7a implements https://unicode.org/reports/tr29/#WB7a
func (sc *Scanner) WB7a(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(Hebrew_Letter, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(Hebrew_Letter, previous) && is(Single_Quote, current)
}

// WB7b implements https://unicode.org/reports/tr29/#WB7b
func (sc *Scanner) WB7b(current, lookahead rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(Hebrew_Letter, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Double_Quote, current) && is(Hebrew_Letter, lookahead)
}

// WB7c implements https://unicode.org/reports/tr29/#WB7c
func (sc *Scanner) WB7c(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Hebrew_Letter, preprevious) && is(Double_Quote, previous) && is(Hebrew_Letter, current)
}

// WB8 implements https://unicode.org/reports/tr29/#WB8
func (sc *Scanner) WB8(current rune) (continues bool) {
	// If it's a new token and Numeric
	if len(sc.buffer) == 0 {
		return is(Numeric, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(Numeric, current)
}

// WB9 implements https://unicode.org/reports/tr29/#WB9
func (sc *Scanner) WB9(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(AHLetter, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Numeric, current)
}

// WB10 implements https://unicode.org/reports/tr29/#WB10
func (sc *Scanner) WB10(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(Numeric, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(AHLetter, current)
}

// WB11 implements https://unicode.org/reports/tr29/#WB11
func (sc *Scanner) WB11(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Numeric, preprevious) && (is(MidNum, previous) || is(MidNumLetQ, previous)) && is(Numeric, current)
}

// WB12 implements https://unicode.org/reports/tr29/#WB12
func (sc *Scanner) WB12(current, lookahead rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(Numeric, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && (is(MidNum, current) || is(MidNumLetQ, current)) && is(Numeric, lookahead)
}

// WB13 implements https://unicode.org/reports/tr29/#WB13
func (sc *Scanner) WB13(current rune) (continues bool) {
	// If it's a new token and Katakana
	if len(sc.buffer) == 0 {
		return is(Katakana, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(Katakana, previous) && is(Katakana, current)
}

// WB13a implements https://unicode.org/reports/tr29/#WB13a
func (sc *Scanner) WB13a(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(AHLetter, current) || is(Numeric, current) || is(Katakana, current) || is(ExtendNumLet, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return (is(AHLetter, previous) || is(Numeric, previous) || is(Katakana, previous) || is(ExtendNumLet, previous)) && is(ExtendNumLet, current)
}

// WB13b implements https://unicode.org/reports/tr29/#WB13b
func (sc *Scanner) WB13b(current rune) (continues bool) {
	if len(sc.buffer) == 0 {
		return is(ExtendNumLet, current)
	}

	previous := sc.buffer[len(sc.buffer)-1]
	return is(ExtendNumLet, previous) && (is(AHLetter, current) || is(Numeric, current) || is(Katakana, current))
}

func (sc *Scanner) token() string {
	if len(sc.buffer) == 0 {
		return ""
	}

	s := string(sc.buffer)
	sc.buffer = sc.buffer[:0]

	return s
}

func (sc *Scanner) accept(r rune) {
	sc.buffer = append(sc.buffer, r)
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

func (sc *Scanner) unreadRune() error {
	err := sc.incoming.UnreadRune()

	if err != nil {
		return err
	}

	return nil
}

// peekRune peeks the next rune, without advancing the reader
func (sc *Scanner) peekRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	// Unread ASAP!
	err = sc.incoming.UnreadRune()
	if err != nil {
		return r, false, err
	}

	return r, false, nil
}
<commit_msg>Better WB1<commit_after>// Package words provides a scanner for Unicode text segmentation word boundaries: https://unicode.org/reports/tr29/#Word_Boundaries
// It does not implement the entire specification, but many of the most important rules.
package words

import (
	"bufio"
	"io"
	"unicode"

	"github.com/clipperhouse/uax29/emoji"
)

// NewScanner tokenizes a reader into a stream of tokens. Iterate through the stream by calling Scan() or Next().
//
// Its uses several specs from Unicode Text Segmentation word boundaries https://unicode.org/reports/tr29/#Word_Boundaries. It's not a full implementation, but a decent approximation for many mainstream cases.
//
// It returns all tokens (including white space), so text can be reconstructed with fidelity.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		incoming: bufio.NewReaderSize(r, 64*1024),
		sot:      true,
	}
}

// Scanner is the structure for scanning an input Reader. Use NewScanner to instantiate.
type Scanner struct {
	incoming *bufio.Reader
	buffer   []rune

	text string
	err  error

	// sot indicates "start of text": https://unicode.org/reports/tr29/#WB1
	sot bool
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
func (sc *Scanner) Scan() bool {
	for {
		current, eof, err := sc.readRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case sc.WB1(eof):
			// true indicates continue
			sc.sot = false
			sc.accept(current)
			continue
		case sc.WB2(eof):
			// true indicates break
			sc.text = sc.token()
			sc.err = nil
			return sc.text != ""
		}

		// Some funcs below require lookahead; better to do I/O here than there
		// (we don't care about eof for lookahead, irrelevant)
		lookahead, _, err := sc.peekRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case
			sc.WB3(current):
			// true indicates continue
			sc.accept(current)
			continue
		case
			sc.WB3a(current),
			sc.WB3b(current):
			// true indicates break
			goto breaking
		case
			sc.WB3d(current),
			sc.WB5(current),
			sc.WB6(current, lookahead),
			sc.WB7(current),
			sc.WB7a(current),
			sc.WB7b(current, lookahead),
			sc.WB7c(current),
			sc.WB8(current),
			sc.WB9(current),
			sc.WB10(current),
			sc.WB11(current),
			sc.WB12(current, lookahead),
			sc.WB13(current):
			// true indicates continue
			sc.accept(current)
			continue
		}

	breaking:
		// If we fall through all the above rules, it's a word break
		// WB999 implements https://unicode.org/reports/tr29/#WB999

		if len(sc.buffer) > 0 {
			sc.text = sc.token()
			sc.err = nil
			sc.accept(current)
			return true
		}

		sc.accept(current)
		continue
	}
}

// Text returns the current token, given a successful call to Scan
func (sc *Scanner) Text() string {
	return sc.text
}

// Err returns the current error, given an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// Word boundary rules: https://unicode.org/reports/tr29/#Word_Boundaries
// In most cases, returning true means 'keep going'; check the name of the return var for clarity

var is = unicode.Is

// WB1 implements https://unicode.org/reports/tr29/#WB1
func (sc *Scanner) WB1(eof bool) (continues bool) {
	sot := len(sc.buffer) == 0 // "start of text"
	return sot && !eof
}

// WB2 implements https://unicode.org/reports/tr29/#WB2
func (sc *Scanner) WB2(eof bool) (breaks bool) {
	// A bit silly, but reads consistently in Scan above
	return eof
}

// WB3 implements https://unicode.org/reports/tr29/#WB3
func (sc *Scanner) WB3(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) && is(LF, current)
}

// WB3a implements https://unicode.org/reports/tr29/#WB3a
func (sc *Scanner) WB3a(current rune) (breaks bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) || is(LF, previous) || is(Newline, previous)
}

// WB3b implements https://unicode.org/reports/tr29/#WB3b
func (sc *Scanner) WB3b(current rune) (breaks bool) {
	return is(CR, current) || is(LF, current) || is(Newline, current)
}

// WB3c implements https://unicode.org/reports/tr29/#WB3c
func (sc *Scanner) WB3c(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ZWJ, previous) && is(emoji.Extended_Pictographic, current)
}

// WB3d implements https://unicode.org/reports/tr29/#WB3d
func (sc *Scanner) WB3d(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(WSegSpace, previous) && is(WSegSpace, current)
}

// WB5 implements https://unicode.org/reports/tr29/#WB5
func (sc *Scanner) WB5(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(AHLetter, current)
}

// WB6 implements https://unicode.org/reports/tr29/#WB6
func (sc *Scanner) WB6(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && (is(MidLetter, current) || is(MidNumLetQ, current)) && is(AHLetter, lookahead)
}

// WB7 implements https://unicode.org/reports/tr29/#WB7
func (sc *Scanner) WB7(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(AHLetter, preprevious) && (is(MidLetter, previous) || is(MidNumLetQ, previous)) && is(AHLetter, current)
}

// WB7a implements https://unicode.org/reports/tr29/#WB7a
func (sc *Scanner) WB7a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Hebrew_Letter, previous) && is(Single_Quote, current)
}

// WB7b implements https://unicode.org/reports/tr29/#WB7b
func (sc *Scanner) WB7b(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Double_Quote, current) && is(Hebrew_Letter, lookahead)
}

// WB7c implements https://unicode.org/reports/tr29/#WB7c
func (sc *Scanner) WB7c(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Hebrew_Letter, preprevious) && is(Double_Quote, previous) && is(Hebrew_Letter, current)
}

// WB8 implements https://unicode.org/reports/tr29/#WB8
func (sc *Scanner) WB8(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(Numeric, current)
}

// WB9 implements https://unicode.org/reports/tr29/#WB9
func (sc *Scanner) WB9(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Numeric, current)
}

// WB10 implements https://unicode.org/reports/tr29/#WB10
func (sc *Scanner) WB10(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(AHLetter, current)
}

// WB11 implements https://unicode.org/reports/tr29/#WB11
func (sc *Scanner) WB11(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Numeric, preprevious) && (is(MidNum, previous) || is(MidNumLetQ, previous)) && is(Numeric, current)
}

// WB12 implements https://unicode.org/reports/tr29/#WB12
func (sc *Scanner) WB12(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && (is(MidNum, current) || is(MidNumLetQ, current)) && is(Numeric, lookahead)
}

// WB13 implements https://unicode.org/reports/tr29/#WB13
func (sc *Scanner) WB13(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Katakana, previous) && is(Katakana, current)
}

// WB13a implements https://unicode.org/reports/tr29/#WB13a
func (sc *Scanner) WB13a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return (is(AHLetter, previous) || is(Numeric, previous) || is(Katakana, previous) || is(ExtendNumLet, previous)) && is(ExtendNumLet, current)
}

// WB13b implements https://unicode.org/reports/tr29/#WB13b
func (sc *Scanner) WB13b(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ExtendNumLet, previous) && (is(AHLetter, current) || is(Numeric, current) || is(Katakana, current))
}

func (sc *Scanner) token() string {
	if len(sc.buffer) == 0 {
		return ""
	}

	s := string(sc.buffer)
	sc.buffer = sc.buffer[:0]

	return s
}

func (sc *Scanner) accept(r rune) {
	sc.buffer = append(sc.buffer, r)
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

func (sc *Scanner) unreadRune() error {
	err := sc.incoming.UnreadRune()

	if err != nil {
		return err
	}

	return nil
}

// peekRune peeks the next rune, without advancing the reader
func (sc *Scanner) peekRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	// Unread ASAP!
	err = sc.incoming.UnreadRune()
	if err != nil {
		return r, false, err
	}

	return r, false, nil
}
<|endoftext|>


<commit_before>/******************************************************************************
pageviewsync Source Code
Copyright (C) 2013 Lumen LLC.

This file is part of the pageviewsync Source Code.

pageviewsync is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

pageviewsync is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with pageviewsync.  If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/

package worker

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"lumenlearning.com/pageviews/canvas"
	"net/http"
	"sync"
)

type Worker struct {
	UserID string
	DBInfo DBConnectInfo
	APIInfo APIConnectInfo

	WorkGrp chan bool
	WaitGrp *sync.WaitGroup
	Logger chan interface{}
}

type DBConnectInfo struct {
	User string
	Pass string
	Schema string
	Table string
}

func (d *DBConnectInfo) String() string {
	return "User:"+d.User+",Pass:"+d.Pass+",Schema:"+d.Schema+",Table:"+d.Table
}

type APIConnectInfo struct {
	Host string
	Auth string
	Client *http.Client
}

func (a *APIConnectInfo) String() string {
	return "Host:"+a.Host+",Auth:"+a.Auth+",Client:"+fmt.Sprint(a.Client)
}

// Indicate that this goroutine is finished.
// Free up space for another worker to start.
func (w *Worker) Done() {
	w.WaitGrp.Done()
	<- w.WorkGrp
}

func (w *Worker) Log(format string, args... interface{}) {
	w.Logger <- fmt.Sprintf("%v: %v", w.UserID, fmt.Sprintf(format, args...))
}

func (w *Worker) RunPageviewUpdate() error {
	// Find out what we need to do to get this user up-to-date.
	// We'll find out what the last timestamp we have recorded is,
	// if any.
	lastRequestID, lastTimestamp, err := GetUpdateReqs(w.UserID, w.DBInfo)
	if err != nil {
		w.Done()
		w.Log("Terminating. GetUpdateReqs says: %v", err.Error())
		return err
	}
	var fullUpdate bool = (lastTimestamp == "" || lastRequestID == "")

	w.Log("lastRequestID=%v, lastRequestID=%v, fullUpdate=%v", lastRequestID, lastTimestamp, fullUpdate)

	// Get the information needed to update the local DB
	url := "https://"+w.APIInfo.Host+"/api/v1/users/"+w.UserID+"/page_views?per_page=1000"
	w.Log("First URL is %v", url)

	// Go get as many pages as we need
	var pvs []canvas.Pageview
	keepGoing := true
	for keepGoing {
		w.Log("Fetching %v", url)
		nextLink, results, err := ParsePage(url, w.APIInfo)
		if err != nil {
			w.Done()
			w.Log("Terminating. ParsePage says: %v", err.Error())
			return err
		}

		// Update url to be the next link
		w.Log("Next page: %v", nextLink)
		url = nextLink

		if !fullUpdate {
			// Find out if we need to keep going
			for _, r := range results {
				ts, _ := r["created_at"]
				id, _ := r["request_id"]
				w.Log("Pageview at %v (%v)", ts, id)

				tISO, err := canvas.TimeFromISO8601(ts.(string))
				if err != nil {
					w.Done()
					w.Log("Terminated. TimeFromISO8601 says: %v", err)
					return err
				}
				tUnix, err := canvas.UnixFromTime(tISO)
				if err != nil {
					w.Done()
					w.Log("Termianted. UnixFromTime says: %v", err)
					return err
				}

				if id.(string) == lastRequestID || tUnix < lastTimestamp {
					keepGoing = false
					break
				} else {
					pvs = append(pvs, r)
				}
			}
		} else {
			// Add everything from this page
			pvs = append(pvs, results...)
		}
		// Is there anything left to grab?
		if nextLink == "" {
			keepGoing = false
			break
		}
	}

	//// send the results to the local database ////
	// make the connection
	con, err := sql.Open("mysql", w.DBInfo.User+":"+w.DBInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+w.DBInfo.Schema)
	defer con.Close()
	if err != nil {
		w.Done()
		w.Log("Terminating. sql.Open says: %v", err.Error())
		return err
	}

	// start a transaction
	tx, err := con.Begin()
	if err != nil {
		w.Done()
		w.Log("Terminating. con.Begin says: %v", err.Error())
		return err
	}

	// Keep track of total pageview records updated
	var insertCount int64

	// convert each canvas.Pageview to an insert statement
	for _, pv := range pvs {
		ins := "INSERT INTO "+w.DBInfo.Table+" ("
		val := " VALUES ("
		i := 0
		for k, v := range pv {
			// Get the string representation of the value
			strVal := fmt.Sprint(v)
			if strVal == "<nil>" {
				strVal = "NULL"
			}

			// build insert and value strings for this item
			insI, valI := BuildInsertAndValues(k, strVal, i)
			ins = ins + insI
			val = val + valI
			i += 1

			// If it's the "created_at" field, we also need to make a copy
			//   in YYYY-mm-dd HH:MM:SS format.  It's NOT "Unix" time format,
			//   but that's what I called it for lack of a better term.
			// Also for the "updated_at" field.
			if k == "created_at" {
				insI, valI, err := GetDateTimeValue("created_at_datetime", strVal, i)
				if err != nil {
					w.Done()
					w.Log("Terminating. GetDateTimeValue says: %v", err.Error())
					return err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}
			if k == "updated_at" {
				insI, valI, err := GetDateTimeValue("updated_at_datetime", strVal, i)
				if err != nil {
					w.Done()
					w.Log("Terminating. GetDateTimeValue says: %v", err.Error())
					return err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}

		}

		// Finish up the query bits
		ins = ins + ")"
		val = val + ")"

		// add each insert to the transaction
		qu := ins + " " + val
		w.Log("Query: %v", qu)
		res, err := tx.Exec(qu)
		if err != nil {
			err1 := err
			err := tx.Rollback()
			w.Done()
			w.Log("Terminating. tx.Exec says: %v", err.Error())
			return errors.New(err1.Error()+"\n"+err.Error())
		}

		// Record number of rows inserted
		rowsAffected, err := res.(sql.Result).RowsAffected()
		if err != nil {
			w.Done()
			w.Log("Terminating. res.RowsAffected says: %v", err.Error())
			return err
		}
		insertCount += rowsAffected
	}

	// commit the transaction
	err = tx.Commit()
	if err != nil {
		w.Done()
		w.Log("Terminating. tx.Commit says: %v", err.Error())
		return err
	}

	// If we got to this point, no errors!
	w.Done()
	w.Log("Updated with %v rows.", insertCount)
	return nil
}

func GetDateTimeValue(k, strVal string, i int) (string, string, error) {
	t, err := canvas.TimeFromISO8601(strVal)
	if err != nil {
		return "", "", err
	}

	tu, err := canvas.UnixFromTime(t)
	if err != nil {
		return "", "", err
	}

	insI, valI := BuildInsertAndValues(k, tu, i)
	return insI, valI, nil
}

func BuildInsertAndValues(k, strVal string, i int) (string, string) {
	ins := ""
	val := ""

	// Change the quote character to handle NULLs, which shouldn't be in quotes.
	qc := "\""
	if strVal == "NULL" {
		qc = ""
	}

	if i > 0 {
		ins = fmt.Sprintf(",`%v`", k)
		val  = fmt.Sprintf(",%v%v%v", qc, strVal, qc)
	} else {
		ins = fmt.Sprintf("`%v`", k)
		val = fmt.Sprintf("%v%v%v", qc, strVal, qc)
	}

	return ins, val
}

// Call the API and parse the results
// Return  the "next" link
// Return a []canvas.Pageview
func ParsePage(url string, apiInfo APIConnectInfo) (string, []canvas.Pageview, error) {
	// Call the API and get the response
	resp, body, err := GetResponse(url, apiInfo)
	if err != nil {
		return "", nil, err
	}

	// Get the Link header, check for rel="next"
	nextLink, err := canvas.GetNextLink(resp)
	if err != nil {
		return "", nil, err
	}

	// Get a map representation of the data returned
	obj, err := canvas.GetObjFromJSON(body)
	if err != nil {
		return "", nil, err
	}

	var results []interface{}

	switch vt := (*obj).(type) {
	case []interface{}:
		results = (*obj).([]interface{})
	default:
		return "", nil, errors.New(fmt.Sprint("Expecting an array (received ", vt, ")"))
	}

	var pvs []canvas.Pageview

	for _, v := range results {
		switch v.(type) {
		case map[string]interface{}:
			pageviewResult := v.(map[string]interface{})
			var pv canvas.Pageview = make(canvas.Pageview)
			for k, v := range pageviewResult {
				if k == "created_at" {

				}
				pv[k] = v
			}
			pvs = append(pvs, pv)
		}
	}

	return nextLink, pvs, nil
}

func GetResponse(url string, apiInfo APIConnectInfo) (*http.Response, *[]byte, error) {
	resp, _, err := canvas.AuthorizedCall(url, apiInfo.Auth, apiInfo.Client)
	if err != nil {
		return nil, nil, err
	}

	body, _, err := canvas.ReadResponse(resp)
	if err != nil {
		return nil, nil, err
	}

	return resp, body, nil
}

func GetUpdateReqs(userID string, dbInfo DBConnectInfo) (string, string, error) {
	// Connect to the DB and the appropriate schema
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return "", "", err
	}

	// Build the query
	query := "SELECT request_id, created_at FROM pageviews WHERE user_id = '"+userID+"' ORDER BY created_at DESC LIMIT 1"

	// Find out the last pageview timestamp for this userID
	row := con.QueryRow(query)
	var request_id string
	var created_at string
	err = row.Scan(&request_id, &created_at)

	if err != nil {
		if err == sql.ErrNoRows {
			request_id = ""
			created_at = ""
		} else {
			return "", "", err
		}
	}

	return request_id, created_at, nil
}
<commit_msg>Move DB update code into its own function.<commit_after>/******************************************************************************
pageviewsync Source Code
Copyright (C) 2013 Lumen LLC.

This file is part of the pageviewsync Source Code.

pageviewsync is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

pageviewsync is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with pageviewsync.  If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/

package worker

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"lumenlearning.com/pageviews/canvas"
	"net/http"
	"sync"
)

type Worker struct {
	UserID string
	DBInfo DBConnectInfo
	APIInfo APIConnectInfo

	WorkGrp chan bool
	WaitGrp *sync.WaitGroup
	Logger chan interface{}
}

type DBConnectInfo struct {
	User string
	Pass string
	Schema string
	Table string
}

func (d *DBConnectInfo) String() string {
	return "User:"+d.User+",Pass:"+d.Pass+",Schema:"+d.Schema+",Table:"+d.Table
}

type APIConnectInfo struct {
	Host string
	Auth string
	Client *http.Client
}

func (a *APIConnectInfo) String() string {
	return "Host:"+a.Host+",Auth:"+a.Auth+",Client:"+fmt.Sprint(a.Client)
}

// Indicate that this goroutine is finished.
// Free up space for another worker to start.
func (w *Worker) Done() {
	w.WaitGrp.Done()
	<- w.WorkGrp
}

func (w *Worker) Log(format string, args... interface{}) {
	w.Logger <- fmt.Sprintf("%v: %v", w.UserID, fmt.Sprintf(format, args...))
}

func (w *Worker) RunPageviewUpdate() error {
	// Find out what we need to do to get this user up-to-date.
	// We'll find out what the last timestamp we have recorded is,
	// if any.
	lastRequestID, lastTimestamp, err := GetUpdateReqs(w.UserID, w.DBInfo)
	if err != nil {
		w.Done()
		w.Log("Terminating. GetUpdateReqs says: %v", err.Error())
		return err
	}
	var fullUpdate bool = (lastTimestamp == "" || lastRequestID == "")

	w.Log("lastRequestID=%v, lastRequestID=%v, fullUpdate=%v", lastRequestID, lastTimestamp, fullUpdate)

	// Get the information needed to update the local DB
	url := "https://"+w.APIInfo.Host+"/api/v1/users/"+w.UserID+"/page_views?per_page=1000"
	w.Log("First URL is %v", url)

	// Go get as many pages as we need
	var pvs []canvas.Pageview
	keepGoing := true
	for keepGoing {
		w.Log("Fetching %v", url)
		nextLink, results, err := ParsePage(url, w.APIInfo)
		if err != nil {
			w.Done()
			w.Log("Terminating. ParsePage says: %v", err.Error())
			return err
		}

		// Update url to be the next link
		w.Log("Next page: %v", nextLink)
		url = nextLink

		if !fullUpdate {
			// Find out if we need to keep going
			for _, r := range results {
				ts, _ := r["created_at"]
				id, _ := r["request_id"]
				w.Log("Pageview at %v (%v)", ts, id)

				tISO, err := canvas.TimeFromISO8601(ts.(string))
				if err != nil {
					w.Done()
					w.Log("Terminated. TimeFromISO8601 says: %v", err)
					return err
				}
				tUnix, err := canvas.UnixFromTime(tISO)
				if err != nil {
					w.Done()
					w.Log("Termianted. UnixFromTime says: %v", err)
					return err
				}

				if id.(string) == lastRequestID || tUnix < lastTimestamp {
					keepGoing = false
					break
				} else {
					pvs = append(pvs, r)
				}
			}
		} else {
			// Add everything from this page
			pvs = append(pvs, results...)
		}
		// Is there anything left to grab?
		if nextLink == "" {
			keepGoing = false
			break
		}
	}

	insertCount, errorFunc, err := UpdateDB(&w.DBInfo, &pvs)
	if err != nil {
		w.Done()
		w.Log("Terminated. UpdateDB/%v says: %v", errorFunc, err.Error())
		return err
	}

	// If we got to this point, no errors!
	w.Done()
	w.Log("Updated with %v rows.", insertCount)
	return nil
}

func UpdateDB(dbInfo *DBConnectInfo, pvs *[]canvas.Pageview) (int64, string, error) {
	//// send the results to the local database ////
	// make the connection
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return 0, "sql.Open", err
	}

	// start a transaction
	tx, err := con.Begin()
	if err != nil {
		return 0, "sql.Begin", err
	}

	// Keep track of total pageview records updated
	var insertCount int64

	// convert each canvas.Pageview to an insert statement
	for _, pv := range *pvs {
		ins := "INSERT INTO "+dbInfo.Table+" ("
		val := " VALUES ("
		i := 0
		for k, v := range pv {
			// Get the string representation of the value
			strVal := fmt.Sprint(v)
			if strVal == "<nil>" {
				strVal = "NULL"
			}

			// build insert and value strings for this item
			insI, valI := BuildInsertAndValues(k, strVal, i)
			ins = ins + insI
			val = val + valI
			i += 1

			// If it's the "created_at" field, we also need to make a copy
			//   in YYYY-mm-dd HH:MM:SS format.  It's NOT "Unix" time format,
			//   but that's what I called it for lack of a better term.
			// Also for the "updated_at" field.
			if k == "created_at" {
				insI, valI, err := GetDateTimeValue("created_at_datetime", strVal, i)
				if err != nil {
					return 0, "GetDateTimeValue", err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}
			if k == "updated_at" {
				insI, valI, err := GetDateTimeValue("updated_at_datetime", strVal, i)
				if err != nil {
					return 0, "GetDateTimeValue", err
				}
				ins = ins + insI
				val = val + valI
				i += 1
			}

		}

		// Finish up the query bits
		ins = ins + ")"
		val = val + ")"

		// add each insert to the transaction
		qu := ins + " " + val
		res, err := tx.Exec(qu)
		if err != nil {
			err1 := err
			err := tx.Rollback()
			return 0, "tx.Exec", errors.New(err1.Error()+"\n"+err.Error())
		}

		// Record number of rows inserted
		rowsAffected, err := res.(sql.Result).RowsAffected()
		if err != nil {
			return 0, "res.RowsAffected", err
		}
		insertCount += rowsAffected
	}

	// commit the transaction
	err = tx.Commit()
	if err != nil {
		return 0, "tx.Commit", err
	}

	return insertCount, "", nil
}

func GetDateTimeValue(k, strVal string, i int) (string, string, error) {
	t, err := canvas.TimeFromISO8601(strVal)
	if err != nil {
		return "", "", err
	}

	tu, err := canvas.UnixFromTime(t)
	if err != nil {
		return "", "", err
	}

	insI, valI := BuildInsertAndValues(k, tu, i)
	return insI, valI, nil
}

func BuildInsertAndValues(k, strVal string, i int) (string, string) {
	ins := ""
	val := ""

	// Change the quote character to handle NULLs, which shouldn't be in quotes.
	qc := "\""
	if strVal == "NULL" {
		qc = ""
	}

	if i > 0 {
		ins = fmt.Sprintf(",`%v`", k)
		val  = fmt.Sprintf(",%v%v%v", qc, strVal, qc)
	} else {
		ins = fmt.Sprintf("`%v`", k)
		val = fmt.Sprintf("%v%v%v", qc, strVal, qc)
	}

	return ins, val
}

// Call the API and parse the results
// Return  the "next" link
// Return a []canvas.Pageview
func ParsePage(url string, apiInfo APIConnectInfo) (string, []canvas.Pageview, error) {
	// Call the API and get the response
	resp, body, err := GetResponse(url, apiInfo)
	if err != nil {
		return "", nil, err
	}

	// Get the Link header, check for rel="next"
	nextLink, err := canvas.GetNextLink(resp)
	if err != nil {
		return "", nil, err
	}

	// Get a map representation of the data returned
	obj, err := canvas.GetObjFromJSON(body)
	if err != nil {
		return "", nil, err
	}

	var results []interface{}

	switch vt := (*obj).(type) {
	case []interface{}:
		results = (*obj).([]interface{})
	default:
		return "", nil, errors.New(fmt.Sprint("Expecting an array (received ", vt, ")"))
	}

	var pvs []canvas.Pageview

	for _, v := range results {
		switch v.(type) {
		case map[string]interface{}:
			pageviewResult := v.(map[string]interface{})
			var pv canvas.Pageview = make(canvas.Pageview)
			for k, v := range pageviewResult {
				if k == "created_at" {

				}
				pv[k] = v
			}
			pvs = append(pvs, pv)
		}
	}

	return nextLink, pvs, nil
}

func GetResponse(url string, apiInfo APIConnectInfo) (*http.Response, *[]byte, error) {
	resp, _, err := canvas.AuthorizedCall(url, apiInfo.Auth, apiInfo.Client)
	if err != nil {
		return nil, nil, err
	}

	body, _, err := canvas.ReadResponse(resp)
	if err != nil {
		return nil, nil, err
	}

	return resp, body, nil
}

func GetUpdateReqs(userID string, dbInfo DBConnectInfo) (string, string, error) {
	// Connect to the DB and the appropriate schema
	con, err := sql.Open("mysql", dbInfo.User+":"+dbInfo.Pass+"@unix(/var/run/mysqld/mysqld.sock)"+"/"+dbInfo.Schema)
	defer con.Close()
	if err != nil {
		return "", "", err
	}

	// Build the query
	query := "SELECT request_id, created_at FROM pageviews WHERE user_id = '"+userID+"' ORDER BY created_at DESC LIMIT 1"

	// Find out the last pageview timestamp for this userID
	row := con.QueryRow(query)
	var request_id string
	var created_at string
	err = row.Scan(&request_id, &created_at)

	if err != nil {
		if err == sql.ErrNoRows {
			request_id = ""
			created_at = ""
		} else {
			return "", "", err
		}
	}

	return request_id, created_at, nil
}
<|endoftext|>


<commit_before>package worker

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/events"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/storage"
	"github.com/ohsu-comp-bio/funnel/util"
	"github.com/ohsu-comp-bio/funnel/version"
)

// DefaultWorker is the default task worker, which follows a basic,
// sequential process of task initialization, execution, finalization,
// and logging.
type DefaultWorker struct {
	Conf        config.Worker
	Store       storage.Storage
	TaskReader  TaskReader
	EventWriter events.Writer
}

// Run runs the Worker.
// TODO document behavior of slow consumer of task log updates
func (r *DefaultWorker) Run(pctx context.Context, taskID string) (runerr error) {

	// The code here is verbose, but simple; mainly loops and simple error checking.
	//
	// The steps are:
	// - prepare the working directory
	// - map the task files to the working directory
	// - log the IP address
	// - set up the storage configuration
	// - validate input and output files
	// - download inputs
	// - run the steps (docker)
	// - upload the outputs
	var event *events.TaskWriter
	var mapper *FileMapper
	var run helper
	var task *tes.Task

	// set up task specific utilities
	event = events.NewTaskWriter(taskID, 0, r.EventWriter)
	mapper = NewFileMapper(filepath.Join(r.Conf.WorkDir, taskID))

	event.Info("Version", version.LogFields()...)
	event.State(tes.State_INITIALIZING)
	event.StartTime(time.Now())

	if name, err := os.Hostname(); err == nil {
		event.Metadata(map[string]string{"hostname": name})
	}

	task, run.syserr = r.TaskReader.Task(pctx, taskID)

	// Run the final logging/state steps in a deferred function
	// to ensure they always run, even if there's a missed error.
	defer func() {
		event.EndTime(time.Now())

		switch {
		case run.taskCanceled:
			// The task was canceled.
			event.Info("Canceled")
			event.State(tes.State_CANCELED)
			runerr = fmt.Errorf("task canceled")
		case run.syserr != nil:
			// Something else failed
			event.Error("System error", "error", run.syserr)
			event.State(tes.State_SYSTEM_ERROR)
			runerr = run.syserr
		case run.execerr != nil:
			// One of the executors failed
			event.Error("Exec error", "error", run.execerr)
			event.State(tes.State_EXECUTOR_ERROR)
			runerr = run.execerr
		default:
			event.State(tes.State_COMPLETE)
		}

		// cleanup workdir
		if !r.Conf.LeaveWorkDir {
			mapper.Cleanup()
		}
	}()

	// Recover from panics
	defer handlePanic(func(e error) {
		fmt.Printf("%#v", e)
		run.syserr = e
	})

	ctx := r.pollForCancel(pctx, taskID, func() { run.taskCanceled = true })
	run.ctx = ctx

	// Prepare file mapper, which maps task file URLs to host filesystem paths
	if run.ok() {
		run.syserr = mapper.MapTask(task)
	}

	if run.ok() {
		run.syserr = r.validateInputs(mapper)
	}

	if run.ok() {
		run.syserr = r.validateOutputs(mapper)
	}

	// Download inputs
	downloadErrs := make(util.MultiError, len(mapper.Inputs))
	downloadCtx, cancelDownloadCtx := context.WithCancel(ctx)
	defer cancelDownloadCtx()
	wg := &sync.WaitGroup{}
	wg.Add(len(mapper.Inputs))
	if run.ok() {
		for i, input := range mapper.Inputs {
			go func(input *tes.Input, i int) {
				defer wg.Done()
				event.Info("Download started", "url", input.Url)
				err := r.Store.Get(downloadCtx, input.Url, input.Path, input.Type)
				if err != nil {
					if err == storage.ErrEmptyDirectory {
						event.Warn("Download finished with warning", "url", input.Url, "warning", err)
					} else {
						downloadErrs[i] = err
						event.Error("Download failed", "url", input.Url, "error", err)
						cancelDownloadCtx()
					}
				} else {
					event.Info("Download finished", "url", input.Url)
				}
				return
			}(input, i)
		}
	}
	wg.Wait()
	if !downloadErrs.IsNil() {
		run.syserr = fmt.Errorf("%s", downloadErrs.Error())
	}

	if run.ok() {
		event.State(tes.State_RUNNING)
	}

	// Run steps
	if run.ok() {
		for i, d := range task.GetExecutors() {
			s := &stepWorker{
				Conf:  r.Conf,
				Event: event.NewExecutorWriter(uint32(i)),
				Command: &DockerCommand{
					Image:         d.Image,
					Command:       d.Command,
					Env:           d.Env,
					Volumes:       mapper.Volumes,
					Workdir:       d.Workdir,
					ContainerName: fmt.Sprintf("%s-%d", task.Id, i),
					// TODO make RemoveContainer configurable
					RemoveContainer: true,
					Event:           event.NewExecutorWriter(uint32(i)),
				},
			}

			// Opens stdin/out/err files and updates those fields on "cmd".
			if run.ok() {
				run.syserr = r.openStepLogs(mapper, s, d)
			}

			if run.ok() {
				run.execerr = s.Run(ctx)
			}
		}
	}

	// Upload outputs
	uploadErrs := make(util.MultiError, len(mapper.Outputs))
	outputs := make([][]*tes.OutputFileLog, len(mapper.Outputs))
	wg = &sync.WaitGroup{}
	wg.Add(len(mapper.Outputs))
	if run.ok() {
		for i, output := range mapper.Outputs {
			go func(output *tes.Output, i int) {
				defer wg.Done()
				event.Info("Upload started", "url", output.Url)
				r.fixLinks(mapper, output.Path)
				out, err := r.Store.Put(ctx, output.Url, output.Path, output.Type)
				if err != nil {
					if err == storage.ErrEmptyDirectory {
						event.Warn("Upload finished with warning", "url", output.Url, "warning", err)
					} else {
						uploadErrs[i] = err
						event.Error("Upload failed", "url", output.Url, "error", err)
					}
				} else {
					event.Info("Upload finished", "url", output.Url)
				}
				outputs[i] = out
				return
			}(output, i)
		}
	}
	wg.Wait()
	outputLog := []*tes.OutputFileLog{}
	for _, out := range outputs {
		outputLog = append(outputLog, out...)
	}
	if !uploadErrs.IsNil() {
		run.syserr = fmt.Errorf("%s", uploadErrs.Error())
	}

	// unmap paths for OutputFileLog
	for _, o := range outputLog {
		o.Path = mapper.ContainerPath(o.Path)
	}

	if len(outputLog) > 0 {
		event.Outputs(outputLog)
	}

	return
}

// fixLinks walks the output paths, fixing cases where a symlink is
// broken because it's pointing to a path inside a container volume.
func (r *DefaultWorker) fixLinks(mapper *FileMapper, basepath string) {
	filepath.Walk(basepath, func(p string, f os.FileInfo, err error) error {
		if err != nil {
			// There's an error, so be safe and give up on this file
			return nil
		}

		// Only bother to check symlinks
		if f.Mode()&os.ModeSymlink != 0 {
			// Test if the file can be opened because it doesn't exist
			fh, rerr := os.Open(p)
			fh.Close()

			if rerr != nil && os.IsNotExist(rerr) {

				// Get symlink source path
				src, err := os.Readlink(p)
				if err != nil {
					return nil
				}
				// Map symlink source (possible container path) to host path
				mapped, err := mapper.HostPath(src)
				if err != nil {
					return nil
				}

				// Check whether the mapped path exists
				fh, err := os.Open(mapped)
				fh.Close()

				// If the mapped path exists, fix the symlink
				if err == nil {
					err := os.Remove(p)
					if err != nil {
						return nil
					}
					os.Symlink(mapped, p)
				}
			}
		}
		return nil
	})
}

// openLogs opens/creates the logs files for a step and updates those fields.
func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.Executor) error {

	// Find the path for task stdin
	var err error
	if d.Stdin != "" {
		s.Command.Stdin, err = mapper.OpenHostFile(d.Stdin)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	// Create file for task stdout
	if d.Stdout != "" {
		s.Command.Stdout, err = mapper.CreateHostFile(d.Stdout)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	// Create file for task stderr
	if d.Stderr != "" {
		s.Command.Stderr, err = mapper.CreateHostFile(d.Stderr)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	return nil
}

// Validate the input downloads
func (r *DefaultWorker) validateInputs(mapper *FileMapper) error {
	for _, input := range mapper.Inputs {
		err := r.Store.SupportsGet(input.Url, input.Type)
		if err != nil {
			return fmt.Errorf("Input download not supported by storage: %v", err)
		}
	}
	return nil
}

// Validate the output uploads
func (r *DefaultWorker) validateOutputs(mapper *FileMapper) error {
	for _, output := range mapper.Outputs {
		err := r.Store.SupportsPut(output.Url, output.Type)
		if err != nil {
			return fmt.Errorf("Output upload not supported by storage: %v", err)
		}
	}
	return nil
}

func (r *DefaultWorker) pollForCancel(pctx context.Context, taskID string, cancelCallback func()) context.Context {
	taskctx, cancel := context.WithCancel(pctx)

	// Start a goroutine that polls the server to watch for a canceled state.
	// If a cancel state is found, "taskctx" is canceled.
	go func() {
		ticker := time.NewTicker(r.Conf.PollingRate)
		defer ticker.Stop()

		for {
			select {
			case <-taskctx.Done():
				return
			case <-ticker.C:
				state, _ := r.TaskReader.State(taskctx, taskID)
				if tes.TerminalState(state) {
					cancel()
					cancelCallback()
				}
			}
		}
	}()
	return taskctx
}
<commit_msg>fix wait group issue<commit_after>package worker

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/events"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/storage"
	"github.com/ohsu-comp-bio/funnel/util"
	"github.com/ohsu-comp-bio/funnel/version"
)

// DefaultWorker is the default task worker, which follows a basic,
// sequential process of task initialization, execution, finalization,
// and logging.
type DefaultWorker struct {
	Conf        config.Worker
	Store       storage.Storage
	TaskReader  TaskReader
	EventWriter events.Writer
}

// Run runs the Worker.
// TODO document behavior of slow consumer of task log updates
func (r *DefaultWorker) Run(pctx context.Context, taskID string) (runerr error) {

	// The code here is verbose, but simple; mainly loops and simple error checking.
	//
	// The steps are:
	// - prepare the working directory
	// - map the task files to the working directory
	// - log the IP address
	// - set up the storage configuration
	// - validate input and output files
	// - download inputs
	// - run the steps (docker)
	// - upload the outputs
	var event *events.TaskWriter
	var mapper *FileMapper
	var run helper
	var task *tes.Task

	// set up task specific utilities
	event = events.NewTaskWriter(taskID, 0, r.EventWriter)
	mapper = NewFileMapper(filepath.Join(r.Conf.WorkDir, taskID))

	event.Info("Version", version.LogFields()...)
	event.State(tes.State_INITIALIZING)
	event.StartTime(time.Now())

	if name, err := os.Hostname(); err == nil {
		event.Metadata(map[string]string{"hostname": name})
	}

	task, run.syserr = r.TaskReader.Task(pctx, taskID)

	// Run the final logging/state steps in a deferred function
	// to ensure they always run, even if there's a missed error.
	defer func() {
		event.EndTime(time.Now())

		switch {
		case run.taskCanceled:
			// The task was canceled.
			event.Info("Canceled")
			event.State(tes.State_CANCELED)
			runerr = fmt.Errorf("task canceled")
		case run.syserr != nil:
			// Something else failed
			event.Error("System error", "error", run.syserr)
			event.State(tes.State_SYSTEM_ERROR)
			runerr = run.syserr
		case run.execerr != nil:
			// One of the executors failed
			event.Error("Exec error", "error", run.execerr)
			event.State(tes.State_EXECUTOR_ERROR)
			runerr = run.execerr
		default:
			event.State(tes.State_COMPLETE)
		}

		// cleanup workdir
		if !r.Conf.LeaveWorkDir {
			mapper.Cleanup()
		}
	}()

	// Recover from panics
	defer handlePanic(func(e error) {
		fmt.Printf("%#v", e)
		run.syserr = e
	})

	ctx := r.pollForCancel(pctx, taskID, func() { run.taskCanceled = true })
	run.ctx = ctx

	// Prepare file mapper, which maps task file URLs to host filesystem paths
	if run.ok() {
		run.syserr = mapper.MapTask(task)
	}

	if run.ok() {
		run.syserr = r.validateInputs(mapper)
	}

	if run.ok() {
		run.syserr = r.validateOutputs(mapper)
	}

	// Download inputs
	downloadErrs := make(util.MultiError, len(mapper.Inputs))
	downloadCtx, cancelDownloadCtx := context.WithCancel(ctx)
	defer cancelDownloadCtx()
	wg := &sync.WaitGroup{}
	if run.ok() {
		for i, input := range mapper.Inputs {
			wg.Add(1)
			go func(input *tes.Input, i int) {
				defer wg.Done()
				event.Info("Download started", "url", input.Url)
				err := r.Store.Get(downloadCtx, input.Url, input.Path, input.Type)
				if err != nil {
					if err == storage.ErrEmptyDirectory {
						event.Warn("Download finished with warning", "url", input.Url, "warning", err)
					} else {
						downloadErrs[i] = err
						event.Error("Download failed", "url", input.Url, "error", err)
						cancelDownloadCtx()
					}
				} else {
					event.Info("Download finished", "url", input.Url)
				}
				return
			}(input, i)
		}
	}
	wg.Wait()
	if !downloadErrs.IsNil() {
		run.syserr = downloadErrs.ToError()
	}

	if run.ok() {
		event.State(tes.State_RUNNING)
	}

	// Run steps
	if run.ok() {
		for i, d := range task.GetExecutors() {
			s := &stepWorker{
				Conf:  r.Conf,
				Event: event.NewExecutorWriter(uint32(i)),
				Command: &DockerCommand{
					Image:         d.Image,
					Command:       d.Command,
					Env:           d.Env,
					Volumes:       mapper.Volumes,
					Workdir:       d.Workdir,
					ContainerName: fmt.Sprintf("%s-%d", task.Id, i),
					// TODO make RemoveContainer configurable
					RemoveContainer: true,
					Event:           event.NewExecutorWriter(uint32(i)),
				},
			}

			// Opens stdin/out/err files and updates those fields on "cmd".
			if run.ok() {
				run.syserr = r.openStepLogs(mapper, s, d)
			}

			if run.ok() {
				run.execerr = s.Run(ctx)
			}
		}
	}

	// Upload outputs
	uploadErrs := make(util.MultiError, len(mapper.Outputs))
	outputs := make([][]*tes.OutputFileLog, len(mapper.Outputs))
	wg = &sync.WaitGroup{}
	if run.ok() {
		for i, output := range mapper.Outputs {
			wg.Add(1)
			go func(output *tes.Output, i int) {
				defer wg.Done()
				event.Info("Upload started", "url", output.Url)
				r.fixLinks(mapper, output.Path)
				out, err := r.Store.Put(ctx, output.Url, output.Path, output.Type)
				if err != nil {
					if err == storage.ErrEmptyDirectory {
						event.Warn("Upload finished with warning", "url", output.Url, "warning", err)
					} else {
						uploadErrs[i] = err
						event.Error("Upload failed", "url", output.Url, "error", err)
					}
				} else {
					event.Info("Upload finished", "url", output.Url)
				}
				outputs[i] = out
				return
			}(output, i)
		}
	}
	wg.Wait()
	outputLog := []*tes.OutputFileLog{}
	for _, out := range outputs {
		outputLog = append(outputLog, out...)
	}
	if !uploadErrs.IsNil() {
		run.syserr = uploadErrs.ToError()
	}

	// unmap paths for OutputFileLog
	for _, o := range outputLog {
		o.Path = mapper.ContainerPath(o.Path)
	}

	if len(outputLog) > 0 {
		event.Outputs(outputLog)
	}

	return
}

// fixLinks walks the output paths, fixing cases where a symlink is
// broken because it's pointing to a path inside a container volume.
func (r *DefaultWorker) fixLinks(mapper *FileMapper, basepath string) {
	filepath.Walk(basepath, func(p string, f os.FileInfo, err error) error {
		if err != nil {
			// There's an error, so be safe and give up on this file
			return nil
		}

		// Only bother to check symlinks
		if f.Mode()&os.ModeSymlink != 0 {
			// Test if the file can be opened because it doesn't exist
			fh, rerr := os.Open(p)
			fh.Close()

			if rerr != nil && os.IsNotExist(rerr) {

				// Get symlink source path
				src, err := os.Readlink(p)
				if err != nil {
					return nil
				}
				// Map symlink source (possible container path) to host path
				mapped, err := mapper.HostPath(src)
				if err != nil {
					return nil
				}

				// Check whether the mapped path exists
				fh, err := os.Open(mapped)
				fh.Close()

				// If the mapped path exists, fix the symlink
				if err == nil {
					err := os.Remove(p)
					if err != nil {
						return nil
					}
					os.Symlink(mapped, p)
				}
			}
		}
		return nil
	})
}

// openLogs opens/creates the logs files for a step and updates those fields.
func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.Executor) error {

	// Find the path for task stdin
	var err error
	if d.Stdin != "" {
		s.Command.Stdin, err = mapper.OpenHostFile(d.Stdin)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	// Create file for task stdout
	if d.Stdout != "" {
		s.Command.Stdout, err = mapper.CreateHostFile(d.Stdout)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	// Create file for task stderr
	if d.Stderr != "" {
		s.Command.Stderr, err = mapper.CreateHostFile(d.Stderr)
		if err != nil {
			s.Event.Error("Couldn't prepare log files", err)
			return err
		}
	}

	return nil
}

// Validate the input downloads
func (r *DefaultWorker) validateInputs(mapper *FileMapper) error {
	for _, input := range mapper.Inputs {
		err := r.Store.SupportsGet(input.Url, input.Type)
		if err != nil {
			return fmt.Errorf("Input download not supported by storage: %v", err)
		}
	}
	return nil
}

// Validate the output uploads
func (r *DefaultWorker) validateOutputs(mapper *FileMapper) error {
	for _, output := range mapper.Outputs {
		err := r.Store.SupportsPut(output.Url, output.Type)
		if err != nil {
			return fmt.Errorf("Output upload not supported by storage: %v", err)
		}
	}
	return nil
}

func (r *DefaultWorker) pollForCancel(pctx context.Context, taskID string, cancelCallback func()) context.Context {
	taskctx, cancel := context.WithCancel(pctx)

	// Start a goroutine that polls the server to watch for a canceled state.
	// If a cancel state is found, "taskctx" is canceled.
	go func() {
		ticker := time.NewTicker(r.Conf.PollingRate)
		defer ticker.Stop()

		for {
			select {
			case <-taskctx.Done():
				return
			case <-ticker.C:
				state, _ := r.TaskReader.State(taskctx, taskID)
				if tes.TerminalState(state) {
					cancel()
					cancelCallback()
				}
			}
		}
	}()
	return taskctx
}
<|endoftext|>


<commit_before>package workflow

import (
	"context"
	"fmt"
	"strings"

	"time"

	"github.com/zmj/sfslack/sharefile"
	"github.com/zmj/sfslack/slack"
)

type sendWorkflow struct {
	*wfBase
	downloadShare *sharefile.Share
}

func newSend(host Host) Workflow {
	return &sendWorkflow{
		wfBase: newBase(host),
	}
}

func (wf *sendWorkflow) Setup() error {
	if wf.err != nil {
		return fmt.Errorf("Workflow already error: %v", wf.err)
	}
	folder, err := wf.createWorkflowFolder()
	if err != nil {
		wf.err = fmt.Errorf("Failed to create workflow folder: %v", err)
		return wf.err
	}

	c := make(chan struct{})
	go func() {
		err = wf.subscribe(folder)
		if err != nil {
			wf.err = fmt.Errorf("Failed to subscribe to workflow folder: %v", err)
		}
		c <- struct{}{}
	}()

	requestShare, err := wf.sf.CreateRequestShare(context.TODO(), folder.ID)
	if err != nil {
		wf.err = fmt.Errorf("Failed to create request share: %v", err)
		return wf.err // cancel sub - check done / shutdown called?
	}

	<-c
	if wf.err != nil {
		return wf.err
	}

	uploadURL := requestShare.URI
	wf.RedirectOrReply(uploadURL, uploadMessage(uploadURL))

	return nil
}

func (wf *sendWorkflow) Listen() error {
	if wf.err != nil {
		return fmt.Errorf("Workflow already error: %v", wf.err)
	}

	var notify <-chan time.Time
	done := time.After(10 * time.Minute)
	for {
		select {
		case <-wf.events:
			notify = time.After(1 * time.Second)
		case <-notify:
			notify = nil
			done = time.After(5 * time.Minute)
			newFiles, err := wf.getNewFiles()
			if err != nil {
				wf.err = fmt.Errorf("Failed to get info for upload notification: %v", err)
				return wf.err
			}
			wf.downloadShare, err = addToShare(wf.sf, wf.downloadShare, newFiles)
			if err != nil {
				wf.err = fmt.Errorf("Failed to make share for notification: %v", err)
				return wf.err
			}
			accepted := wf.Host.Reply(wf.downloadMessage())
			if !accepted {
				return nil
			}
		case <-done:
			return nil
		}
	}
}

func uploadMessage(uploadURL string) slack.Message {
	return slack.Message{
		Text: fmt.Sprintf("Upload your files: %v", uploadURL),
	}
}

func (wf *sendWorkflow) downloadMessage() slack.Message {
	user := wf.Host.User()
	files := wf.files // all files - new only instead?
	downloadURL := wf.sf.Account().DownloadAllURL(*wf.downloadShare)

	msg := slack.Message{ResponseType: slack.ResponseTypeInChannel}
	if len(files) == 1 {
		msg.Text = fmt.Sprintf("%v has shared %v: %v", user, files[0].FileName, downloadURL)
	} else {
		msg.Text = fmt.Sprintf("%v has shared %v files: %v", user, len(files), downloadURL)
		var fileNames []string
		for _, file := range files {
			fileNames = append(fileNames, file.FileName)
		}
		msg.Attachments = []slack.Attachment{
			slack.Attachment{
				Text:     strings.Join(fileNames, "\n"),
				Fallback: strings.Join(fileNames, " "),
			},
		}
	}
	return msg
}
<commit_msg>increase send batch window to 2 seconds<commit_after>package workflow

import (
	"context"
	"fmt"
	"strings"

	"time"

	"github.com/zmj/sfslack/sharefile"
	"github.com/zmj/sfslack/slack"
)

type sendWorkflow struct {
	*wfBase
	downloadShare *sharefile.Share
}

func newSend(host Host) Workflow {
	return &sendWorkflow{
		wfBase: newBase(host),
	}
}

func (wf *sendWorkflow) Setup() error {
	if wf.err != nil {
		return fmt.Errorf("Workflow already error: %v", wf.err)
	}
	folder, err := wf.createWorkflowFolder()
	if err != nil {
		wf.err = fmt.Errorf("Failed to create workflow folder: %v", err)
		return wf.err
	}

	c := make(chan struct{})
	go func() {
		err = wf.subscribe(folder)
		if err != nil {
			wf.err = fmt.Errorf("Failed to subscribe to workflow folder: %v", err)
		}
		c <- struct{}{}
	}()

	requestShare, err := wf.sf.CreateRequestShare(context.TODO(), folder.ID)
	if err != nil {
		wf.err = fmt.Errorf("Failed to create request share: %v", err)
		return wf.err // cancel sub - check done / shutdown called?
	}

	<-c
	if wf.err != nil {
		return wf.err
	}

	uploadURL := requestShare.URI
	wf.RedirectOrReply(uploadURL, uploadMessage(uploadURL))

	return nil
}

func (wf *sendWorkflow) Listen() error {
	if wf.err != nil {
		return fmt.Errorf("Workflow already error: %v", wf.err)
	}

	var notify <-chan time.Time
	done := time.After(10 * time.Minute)
	for {
		select {
		case <-wf.events:
			notify = time.After(2 * time.Second)
		case <-notify:
			notify = nil
			done = time.After(5 * time.Minute)
			newFiles, err := wf.getNewFiles()
			if err != nil {
				wf.err = fmt.Errorf("Failed to get info for upload notification: %v", err)
				return wf.err
			}
			wf.downloadShare, err = addToShare(wf.sf, wf.downloadShare, newFiles)
			if err != nil {
				wf.err = fmt.Errorf("Failed to make share for notification: %v", err)
				return wf.err
			}
			accepted := wf.Host.Reply(wf.downloadMessage())
			if !accepted {
				return nil
			}
		case <-done:
			return nil
		}
	}
}

func uploadMessage(uploadURL string) slack.Message {
	return slack.Message{
		Text: fmt.Sprintf("Upload your files: %v", uploadURL),
	}
}

func (wf *sendWorkflow) downloadMessage() slack.Message {
	user := wf.Host.User()
	files := wf.files // all files - new only instead?
	downloadURL := wf.sf.Account().DownloadAllURL(*wf.downloadShare)

	msg := slack.Message{ResponseType: slack.ResponseTypeInChannel}
	if len(files) == 1 {
		msg.Text = fmt.Sprintf("%v has shared %v: %v", user, files[0].FileName, downloadURL)
	} else {
		msg.Text = fmt.Sprintf("%v has shared %v files: %v", user, len(files), downloadURL)
		var fileNames []string
		for _, file := range files {
			fileNames = append(fileNames, file.FileName)
		}
		msg.Attachments = []slack.Attachment{
			slack.Attachment{
				Text:     strings.Join(fileNames, "\n"),
				Fallback: strings.Join(fileNames, " "),
			},
		}
	}
	return msg
}
<|endoftext|>


<commit_before>package mixpanel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
)

// The official base URL
const MixpanelBaseURL = "https://data.mixpanel.com/api/2.0/export"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// EventData is a representation of each individual JSON record spit out of the
// export process.
type EventData map[string]interface{}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithURL(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL. (I doubt this will ever be useful but there you go)
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
//
// Algorithm:
// - join key=value pairs
// - sort the pairs alphabetically
// - appending a secret
// - take MD5 hex digest.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()

	var params []string
	for k, vs := range *args {
		for _, v := range vs {
			params = append(params, fmt.Sprintf("%s=%s", k, v))
		}
	}

	sort.StringSlice(params).Sort()

	io.WriteString(hash, strings.Join(params, "")+m.Secret)

	args.Set("sig", fmt.Sprintf("%x", hash.Sum(nil)))
}

// Generate the initial, base arguments that all Mixpanel API requests use.
func (m *Mixpanel) makeArgs() url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", fmt.Sprintf("%d", time.Now().Unix()+10000))

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, outChan chan<- EventData, moreArgs *url.Values) {
	args := m.makeArgs()

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	day := date.Format("2006-01-02")

	args.Set("from_date", day)
	args.Set("to_date", day)

	m.addSignature(&args)

	resp, err := http.Get(fmt.Sprintf("%s?%s", m.BaseURL, args.Encode()))
	defer resp.Body.Close()

	if err != nil {
		log.Fatalf("%s: XXX handle this: download failed: %s", m.Product, err)
	}

	decoder := json.NewDecoder(resp.Body)

	// Don't default all numeric values to float
	decoder.UseNumber()

	for {
		var ev struct {
			Error      *string
			Event      string
			Properties map[string]interface{}
		}

		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			log.Fatalf("%s: Failed to parse JSON: %s", m.Product, err)
		} else if ev.Error != nil {
			log.Fatalf("%s: Hit API error: %s", m.Product, *ev.Error)
		}

		// TODO: handle distinct_id not being present instead of skipping
		if _, ok := ev.Properties["distinct_id"]; !ok {
			continue
		}

		ev.Properties["product"] = m.Product
		ev.Properties["event"] = ev.Event

		outChan <- ev.Properties

	}

	defer close(outChan)
}
<commit_msg>Split out JSON transformation code to new function<commit_after>package mixpanel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
)

// The official base URL
const MixpanelBaseURL = "https://data.mixpanel.com/api/2.0/export"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// EventData is a representation of each individual JSON record spit out of the
// export process.
type EventData map[string]interface{}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithURL(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL. (I doubt this will ever be useful but there you go)
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
//
// Algorithm:
// - join key=value pairs
// - sort the pairs alphabetically
// - appending a secret
// - take MD5 hex digest.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()

	var params []string
	for k, vs := range *args {
		for _, v := range vs {
			params = append(params, fmt.Sprintf("%s=%s", k, v))
		}
	}

	sort.StringSlice(params).Sort()

	io.WriteString(hash, strings.Join(params, "")+m.Secret)

	args.Set("sig", fmt.Sprintf("%x", hash.Sum(nil)))
}

// Generate the initial, base arguments that all Mixpanel API requests use.
func (m *Mixpanel) makeArgs(date time.Time) url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", fmt.Sprintf("%d", time.Now().Unix()+10000))

	day := date.Format("2006-01-02")

	args.Set("from_date", day)
	args.Set("to_date", day)

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, output chan<- EventData, moreArgs *url.Values) {
	defer close(output)

	args := m.makeArgs(date)

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	m.addSignature(&args)

	resp, err := http.Get(fmt.Sprintf("%s?%s", m.BaseURL, args.Encode()))
	defer resp.Body.Close()

	if err != nil {
		log.Fatalf("%s: XXX handle this: download failed: %s", m.Product, err)
	}

	m.TransformEventData(resp.Body, output)
}

// TransformEventData reads JSON objects line by line from `input`, performs a
// simple translation, and pipes the result back out through the `output` chan.
//
// The transformation effectively folds the properties map into the top level
// and attaches product information.
//
// Input : `{"event": "...", "properties": {"k": "v"}}`
// Output: `{"event": "...", "product: "...", "k": "v", ...}`
func (m *Mixpanel) TransformEventData(input io.Reader, output chan<- EventData) {
	decoder := json.NewDecoder(input)

	// Don't default all numeric values to float
	decoder.UseNumber()

	for {
		var ev struct {
			Error      *string
			Event      string
			Properties map[string]interface{}
		}

		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			log.Fatalf("%s: Failed to parse JSON: %s", m.Product, err)
		} else if ev.Error != nil {
			log.Fatalf("%s: Hit API error: %s", m.Product, *ev.Error)
		}

		// TODO: handle distinct_id not being present instead of skipping
		if _, ok := ev.Properties["distinct_id"]; !ok {
			continue
		}

		ev.Properties["product"] = m.Product
		ev.Properties["event"] = ev.Event

		output <- ev.Properties
	}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"sort"
)

type CellList []*Cell

type IntSlice []int

func getRow(cell *Cell) int {
	return cell.Row
}

func getCol(cell *Cell) int {
	return cell.Col
}

func getBlock(cell *Cell) int {
	return cell.Block
}

func (self CellList) SameRow() bool {
	return self.CollectNums(getRow).Same()
}

func (self CellList) SameCol() bool {
	return self.CollectNums(getCol).Same()
}

func (self CellList) SameBlock() bool {
	return self.CollectNums(getBlock).Same()
}

func (self CellList) Row() int {
	//Will return the row of a random item.
	if len(self) == 0 {
		return 0
	}
	return self[0].Row
}

func (self CellList) Col() int {
	if len(self) == 0 {
		return 0
	}
	return self[0].Col
}

func (self CellList) Block() int {
	if len(self) == 0 {
		return 0
	}
	return self[0].Block
}

func (self CellList) AddExclude(exclude int) {
	mapper := func(cell *Cell) {
		cell.setExcluded(exclude, true)
	}
	self.Map(mapper)
}

func (self CellList) FilterByPossible(possible int) CellList {
	//TODO: test this
	filter := func(cell *Cell) bool {
		return cell.Possible(possible)
	}
	return self.Filter(filter)
}

func (self CellList) FilterByNumPossibilities(target int) CellList {
	//TODO: test this
	filter := func(cell *Cell) bool {
		return len(cell.Possibilities()) == target
	}
	return self.Filter(filter)
}

func (self CellList) FilterByHasPossibilities() CellList {
	//Returns a list of cells that have possibilities.
	//TODO: test this.
	filter := func(cell *Cell) bool {
		return len(cell.Possibilities()) > 0
	}
	return self.Filter(filter)
}

func (self CellList) RemoveCells(targets CellList) CellList {
	//TODO: test this.
	targetCells := make(map[*Cell]bool)
	for _, cell := range targets {
		targetCells[cell] = true
	}
	filterFunc := func(cell *Cell) bool {
		return !targetCells[cell]
	}
	return self.Filter(filterFunc)
}

func (self CellList) PossibilitiesUnion() IntSlice {
	//Returns an IntSlice of the union of all possibilities.
	set := make(map[int]bool)

	for _, cell := range self {
		for _, possibility := range cell.Possibilities() {
			set[possibility] = true
		}
	}

	result := make(IntSlice, len(set))

	i := 0
	for possibility, _ := range set {
		result[i] = possibility
		i++
	}

	return result
}

func (self CellList) Subset(indexes IntSlice) CellList {
	result := make(CellList, len(indexes))
	max := len(self)
	for i, index := range indexes {
		if index > max {
			continue
		}
		result[i] = self[index]
	}
	return result
}

func (self CellList) CollectNums(fetcher func(*Cell) int) IntSlice {
	var result IntSlice
	for _, cell := range self {
		result = append(result, fetcher(cell))
	}
	return result
}

func (self CellList) Filter(filter func(*Cell) bool) CellList {
	var result CellList
	for _, cell := range self {
		if filter(cell) {
			result = append(result, cell)
		}
	}
	return result
}

func (self CellList) Map(mapper func(*Cell)) {
	for _, cell := range self {
		mapper(cell)
	}
}

func (self IntSlice) Same() bool {
	if len(self) == 0 {
		return true
	}
	target := self[0]
	for _, num := range self {
		if target != num {
			return false
		}
	}
	return true
}

func (self IntSlice) SameContentAs(otherSlice IntSlice) bool {
	//Same as SameAs, but doesn't care about order.
	var selfToUse IntSlice
	if sort.IntsAreSorted(self) {
		selfToUse = self
	} else {
		selfToUse := make(IntSlice, len(self))
		sort.IntSlice(selfToUse).Sort()
	}

	var otherToUse IntSlice

	if sort.IntsAreSorted(otherSlice) {
		otherToUse = otherSlice
	} else {
		otherToUse := make(IntSlice, len(otherToUse))
		sort.IntSlice(otherToUse).Sort()
	}

	return selfToUse.SameAs(otherToUse)
}

func (self IntSlice) SameAs(other IntSlice) bool {
	//TODO: test this.
	if len(self) != len(other) {
		return false
	}
	for i, num := range self {
		if other[i] != num {
			return false
		}
	}
	return true
}
<commit_msg>Fixed an off by one error in the stack-crashing detection for Subset.<commit_after>package sudoku

import (
	"sort"
)

type CellList []*Cell

type IntSlice []int

func getRow(cell *Cell) int {
	return cell.Row
}

func getCol(cell *Cell) int {
	return cell.Col
}

func getBlock(cell *Cell) int {
	return cell.Block
}

func (self CellList) SameRow() bool {
	return self.CollectNums(getRow).Same()
}

func (self CellList) SameCol() bool {
	return self.CollectNums(getCol).Same()
}

func (self CellList) SameBlock() bool {
	return self.CollectNums(getBlock).Same()
}

func (self CellList) Row() int {
	//Will return the row of a random item.
	if len(self) == 0 {
		return 0
	}
	return self[0].Row
}

func (self CellList) Col() int {
	if len(self) == 0 {
		return 0
	}
	return self[0].Col
}

func (self CellList) Block() int {
	if len(self) == 0 {
		return 0
	}
	return self[0].Block
}

func (self CellList) AddExclude(exclude int) {
	mapper := func(cell *Cell) {
		cell.setExcluded(exclude, true)
	}
	self.Map(mapper)
}

func (self CellList) FilterByPossible(possible int) CellList {
	//TODO: test this
	filter := func(cell *Cell) bool {
		return cell.Possible(possible)
	}
	return self.Filter(filter)
}

func (self CellList) FilterByNumPossibilities(target int) CellList {
	//TODO: test this
	filter := func(cell *Cell) bool {
		return len(cell.Possibilities()) == target
	}
	return self.Filter(filter)
}

func (self CellList) FilterByHasPossibilities() CellList {
	//Returns a list of cells that have possibilities.
	//TODO: test this.
	filter := func(cell *Cell) bool {
		return len(cell.Possibilities()) > 0
	}
	return self.Filter(filter)
}

func (self CellList) RemoveCells(targets CellList) CellList {
	//TODO: test this.
	targetCells := make(map[*Cell]bool)
	for _, cell := range targets {
		targetCells[cell] = true
	}
	filterFunc := func(cell *Cell) bool {
		return !targetCells[cell]
	}
	return self.Filter(filterFunc)
}

func (self CellList) PossibilitiesUnion() IntSlice {
	//Returns an IntSlice of the union of all possibilities.
	set := make(map[int]bool)

	for _, cell := range self {
		for _, possibility := range cell.Possibilities() {
			set[possibility] = true
		}
	}

	result := make(IntSlice, len(set))

	i := 0
	for possibility, _ := range set {
		result[i] = possibility
		i++
	}

	return result
}

func (self CellList) Subset(indexes IntSlice) CellList {
	result := make(CellList, len(indexes))
	max := len(self)
	for i, index := range indexes {
		if index >= max {
			//This probably is indicative of a larger problem.
			continue
		}
		result[i] = self[index]
	}
	return result
}

func (self CellList) CollectNums(fetcher func(*Cell) int) IntSlice {
	var result IntSlice
	for _, cell := range self {
		result = append(result, fetcher(cell))
	}
	return result
}

func (self CellList) Filter(filter func(*Cell) bool) CellList {
	var result CellList
	for _, cell := range self {
		if filter(cell) {
			result = append(result, cell)
		}
	}
	return result
}

func (self CellList) Map(mapper func(*Cell)) {
	for _, cell := range self {
		mapper(cell)
	}
}

func (self IntSlice) Same() bool {
	if len(self) == 0 {
		return true
	}
	target := self[0]
	for _, num := range self {
		if target != num {
			return false
		}
	}
	return true
}

func (self IntSlice) SameContentAs(otherSlice IntSlice) bool {
	//Same as SameAs, but doesn't care about order.
	var selfToUse IntSlice
	if sort.IntsAreSorted(self) {
		selfToUse = self
	} else {
		selfToUse := make(IntSlice, len(self))
		sort.IntSlice(selfToUse).Sort()
	}

	var otherToUse IntSlice

	if sort.IntsAreSorted(otherSlice) {
		otherToUse = otherSlice
	} else {
		otherToUse := make(IntSlice, len(otherToUse))
		sort.IntSlice(otherToUse).Sort()
	}

	return selfToUse.SameAs(otherToUse)
}

func (self IntSlice) SameAs(other IntSlice) bool {
	//TODO: test this.
	if len(self) != len(other) {
		return false
	}
	for i, num := range self {
		if other[i] != num {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package conf

import (
	"errors"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Config contains the application configuration settings
type Config struct {
	// Port gets the port number under which the site is being hosted.
	Port int

	// UploadDriver is used to select which backend data store is used for file uploads.
	// Supported drivers: fs, s3
	UploadDriver string

	// UploadPath gets the path (full or relative) under which to store uploads.
	// When using Amazon S3, this should be set to the bucket name.
	UploadPath string

	// IsDevelopment defines whether to run the application in "development mode".
	// Development mode turns on additional features, such as logging, that may
	// not be desirable in a production environment.
	IsDevelopment bool

	// SecureKey is used for session authentication. Recommended to be 32 or 64 ASCII characters.
	// Multiple keys can be separated by commas.
	SecureKeys []string

	// ApplicationTitle is used where the application name (title) is displayed on screen.
	ApplicationTitle string

	// HomeTitle is an optional heading displayed at the top of the gome screen.
	HomeTitle string

	// HomeImage is an optional heading image displayed beneath the HomeTitle.
	HomeImage string

	// DatabaseDriver gets which database/sql driver to use.
	// Supported drivers: postgres
	DatabaseDriver string

	// DatabaseUrl gets the url (or path, connection string, etc) to use with the associated
	// database driver when opening the database connection.
	DatabaseURL string

	// DatabaseMaxConnections gets the maximum number of open database connection to maintain.
	// This value is fed to sql.DB.SetMaxOpenConns(N). The default is 0 (unlimited).
	DatabaseMaxConnections int
}

// Load reads the configuration file from the specified path
func Load(path string) *Config {
	c := Config{
		Port:                   4000,
		UploadDriver:           "fs",
		UploadPath:             filepath.Join("data", "uploads"),
		IsDevelopment:          false,
		SecureKeys:             nil,
		ApplicationTitle:       "GOMP: Go Meal Planner",
		HomeTitle:              "",
		HomeImage:              "",
		DatabaseDriver:         "postgres",
		DatabaseURL:            "",
		DatabaseMaxConnections: 0,
	}

	// If environment variables are set, use them.
	loadEnv("PORT", &c.Port)
	loadEnv("GOMP_UPLOAD_DRIVER", &c.UploadDriver)
	loadEnv("GOMP_UPLOAD_PATH", &c.UploadPath)
	loadEnv("GOMP_IS_DEVELOPMENT", &c.IsDevelopment)
	loadEnv("SECURE_KEY", &c.SecureKeys)
	loadEnv("GOMP_APPLICATION_TITLE", &c.ApplicationTitle)
	loadEnv("GOMP_HOME_TITLE", &c.HomeTitle)
	loadEnv("GOMP_HOME_IMAGE", &c.HomeImage)
	loadEnv("DATABASE_DRIVER", &c.DatabaseDriver)
	loadEnv("DATABASE_URL", &c.DatabaseURL)
	loadEnv("DATABASE_MAX_CONNS", &c.DatabaseMaxConnections)

	if c.IsDevelopment {
		log.Printf("[config] Port=%d", c.Port)
		log.Printf("[config] UploadDriver=%s", c.UploadDriver)
		log.Printf("[config] UploadPath=%s", c.UploadPath)
		log.Printf("[config] IsDevelopment=%t", c.IsDevelopment)
		log.Printf("[config] SecureKeys=%s", c.SecureKeys)
		log.Printf("[config] ApplicationTitle=%s", c.ApplicationTitle)
		log.Printf("[config] HomeTitle=%s", c.HomeTitle)
		log.Printf("[config] HomeImage=%s", c.HomeImage)
		log.Printf("[config] DatabaseDriver=%s", c.DatabaseDriver)
		log.Printf("[config] DatabaseURL=%s", c.DatabaseURL)
		log.Printf("[config] DatabaseMaxConnections=%d", c.DatabaseMaxConnections)
	}

	return &c
}

// Validate checks whether the current configuration settings are valid.
func (c *Config) Validate() error {
	if c.Port <= 0 {
		return errors.New("PORT must be a positive integer")
	}

	if c.UploadDriver != "fs" && c.UploadDriver != "s3" {
		return errors.New("UPLOAD_DRIVER must be one of ('fs', 's3')")
	}

	if c.UploadPath == "" {
		return errors.New("UPLOAD_PATH must be specified")
	}

	if c.SecureKeys == nil || len(c.SecureKeys) < 1 {
		return errors.New("SECURE_KEY must be specified with 1 or more keys separated by a comma")
	}

	if c.ApplicationTitle == "" {
		return errors.New("GOMP_APPLICATION_TITLE must be specified")
	}

	if c.DatabaseDriver != "postgres" {
		return errors.New("DATABASE_DRIVER must be one of ('postgres')")
	}

	if c.DatabaseURL == "" {
		return errors.New("DATABASE_URL must be specified")
	}

	if _, err := url.Parse(c.DatabaseURL); err != nil {
		return errors.New("DATABASE_URL is invalid")
	}

	if c.DatabaseMaxConnections < 0 {
		return errors.New("DATABASE_MAX_CONNS must be a non-negative integer")
	}

	return nil
}

func loadEnv(name string, dest interface{}) {
	var err error
	if envStr := os.Getenv(name); envStr != "" {
		switch dest := dest.(type) {
		case *string:
			*dest = envStr
		case *[]string:
			*dest = strings.Split(envStr, ",")
		case *int:
			if *dest, err = strconv.Atoi(envStr); err != nil {
				log.Fatalf("[config] Failed to convert %s environment variable to an integer. Value = %s, Error = %s",
					name, envStr, err)
			}
		case *bool:
			*dest = envStr != "0"
		}
	}
}
<commit_msg>Fixed a small typo<commit_after>package conf

import (
	"errors"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Config contains the application configuration settings
type Config struct {
	// Port gets the port number under which the site is being hosted.
	Port int

	// UploadDriver is used to select which backend data store is used for file uploads.
	// Supported drivers: fs, s3
	UploadDriver string

	// UploadPath gets the path (full or relative) under which to store uploads.
	// When using Amazon S3, this should be set to the bucket name.
	UploadPath string

	// IsDevelopment defines whether to run the application in "development mode".
	// Development mode turns on additional features, such as logging, that may
	// not be desirable in a production environment.
	IsDevelopment bool

	// SecureKeys is used for session authentication. Recommended to be 32 or 64 ASCII characters.
	// Multiple keys can be separated by commas.
	SecureKeys []string

	// ApplicationTitle is used where the application name (title) is displayed on screen.
	ApplicationTitle string

	// HomeTitle is an optional heading displayed at the top of the gome screen.
	HomeTitle string

	// HomeImage is an optional heading image displayed beneath the HomeTitle.
	HomeImage string

	// DatabaseDriver gets which database/sql driver to use.
	// Supported drivers: postgres
	DatabaseDriver string

	// DatabaseUrl gets the url (or path, connection string, etc) to use with the associated
	// database driver when opening the database connection.
	DatabaseURL string

	// DatabaseMaxConnections gets the maximum number of open database connection to maintain.
	// This value is fed to sql.DB.SetMaxOpenConns(N). The default is 0 (unlimited).
	DatabaseMaxConnections int
}

// Load reads the configuration file from the specified path
func Load(path string) *Config {
	c := Config{
		Port:                   4000,
		UploadDriver:           "fs",
		UploadPath:             filepath.Join("data", "uploads"),
		IsDevelopment:          false,
		SecureKeys:             nil,
		ApplicationTitle:       "GOMP: Go Meal Planner",
		HomeTitle:              "",
		HomeImage:              "",
		DatabaseDriver:         "postgres",
		DatabaseURL:            "",
		DatabaseMaxConnections: 0,
	}

	// If environment variables are set, use them.
	loadEnv("PORT", &c.Port)
	loadEnv("GOMP_UPLOAD_DRIVER", &c.UploadDriver)
	loadEnv("GOMP_UPLOAD_PATH", &c.UploadPath)
	loadEnv("GOMP_IS_DEVELOPMENT", &c.IsDevelopment)
	loadEnv("SECURE_KEY", &c.SecureKeys)
	loadEnv("GOMP_APPLICATION_TITLE", &c.ApplicationTitle)
	loadEnv("GOMP_HOME_TITLE", &c.HomeTitle)
	loadEnv("GOMP_HOME_IMAGE", &c.HomeImage)
	loadEnv("DATABASE_DRIVER", &c.DatabaseDriver)
	loadEnv("DATABASE_URL", &c.DatabaseURL)
	loadEnv("DATABASE_MAX_CONNS", &c.DatabaseMaxConnections)

	if c.IsDevelopment {
		log.Printf("[config] Port=%d", c.Port)
		log.Printf("[config] UploadDriver=%s", c.UploadDriver)
		log.Printf("[config] UploadPath=%s", c.UploadPath)
		log.Printf("[config] IsDevelopment=%t", c.IsDevelopment)
		log.Printf("[config] SecureKeys=%s", c.SecureKeys)
		log.Printf("[config] ApplicationTitle=%s", c.ApplicationTitle)
		log.Printf("[config] HomeTitle=%s", c.HomeTitle)
		log.Printf("[config] HomeImage=%s", c.HomeImage)
		log.Printf("[config] DatabaseDriver=%s", c.DatabaseDriver)
		log.Printf("[config] DatabaseURL=%s", c.DatabaseURL)
		log.Printf("[config] DatabaseMaxConnections=%d", c.DatabaseMaxConnections)
	}

	return &c
}

// Validate checks whether the current configuration settings are valid.
func (c *Config) Validate() error {
	if c.Port <= 0 {
		return errors.New("PORT must be a positive integer")
	}

	if c.UploadDriver != "fs" && c.UploadDriver != "s3" {
		return errors.New("UPLOAD_DRIVER must be one of ('fs', 's3')")
	}

	if c.UploadPath == "" {
		return errors.New("UPLOAD_PATH must be specified")
	}

	if c.SecureKeys == nil || len(c.SecureKeys) < 1 {
		return errors.New("SECURE_KEY must be specified with 1 or more keys separated by a comma")
	}

	if c.ApplicationTitle == "" {
		return errors.New("GOMP_APPLICATION_TITLE must be specified")
	}

	if c.DatabaseDriver != "postgres" {
		return errors.New("DATABASE_DRIVER must be one of ('postgres')")
	}

	if c.DatabaseURL == "" {
		return errors.New("DATABASE_URL must be specified")
	}

	if _, err := url.Parse(c.DatabaseURL); err != nil {
		return errors.New("DATABASE_URL is invalid")
	}

	if c.DatabaseMaxConnections < 0 {
		return errors.New("DATABASE_MAX_CONNS must be a non-negative integer")
	}

	return nil
}

func loadEnv(name string, dest interface{}) {
	var err error
	if envStr := os.Getenv(name); envStr != "" {
		switch dest := dest.(type) {
		case *string:
			*dest = envStr
		case *[]string:
			*dest = strings.Split(envStr, ",")
		case *int:
			if *dest, err = strconv.Atoi(envStr); err != nil {
				log.Fatalf("[config] Failed to convert %s environment variable to an integer. Value = %s, Error = %s",
					name, envStr, err)
			}
		case *bool:
			*dest = envStr != "0"
		}
	}
}
<|endoftext|>