<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package capnslog

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"runtime"
	"strings"
	"time"
)

type Formatter interface {
	Format(pkg string, level LogLevel, depth int, entries ...interface{})
	Flush()
}

func NewStringFormatter(w io.Writer) Formatter {
	return &StringFormatter{
		w: bufio.NewWriter(w),
	}
}

type StringFormatter struct {
	w *bufio.Writer
}

func (s *StringFormatter) Format(pkg string, l LogLevel, i int, entries ...interface{}) {
	now := time.Now().UTC()
	s.w.WriteString(now.Format(time.RFC3339))
	s.w.WriteByte(' ')
	writeEntries(s.w, pkg, l, i, entries...)
	s.Flush()
}

func writeEntries(w *bufio.Writer, pkg string, _ LogLevel, _ int, entries ...interface{}) {
	if pkg != "" {
		w.WriteString(pkg + ": ")
	}
	str := fmt.Sprint(entries...)
	endsInNL := strings.HasSuffix(str, "\n")
	w.WriteString(str)
	if !endsInNL {
		w.WriteString("\n")
	}
}

func (s *StringFormatter) Flush() {
	s.w.Flush()
}

func NewPrettyFormatter(w io.Writer, debug bool) Formatter {
	return &PrettyFormatter{
		w:     bufio.NewWriter(w),
		debug: debug,
	}
}

type PrettyFormatter struct {
	w     *bufio.Writer
	debug bool
}

func (c *PrettyFormatter) Format(pkg string, l LogLevel, depth int, entries ...interface{}) {
	now := time.Now()
	ts := now.Format("2006-01-02 15:04:05")
	c.w.WriteString(ts)
	ms := now.Nanosecond() / 1000
	c.w.WriteString(fmt.Sprintf(".%06d", ms))
	if c.debug {
		_, file, line, ok := runtime.Caller(depth) // It's always the same number of frames to the user's call.
		if !ok {
			file = "???"
			line = 1
		} else {
			slash := strings.LastIndex(file, "/")
			if slash >= 0 {
				file = file[slash+1:]
			}
		}
		if line < 0 {
			line = 0 // not a real line number
		}
		c.w.WriteString(fmt.Sprintf(" [%s:%d]", file, line))
	}
	c.w.WriteString(fmt.Sprint(" ", l.Char(), " | "))
	writeEntries(c.w, pkg, l, depth, entries...)
	c.Flush()
}

func (c *PrettyFormatter) Flush() {
	c.w.Flush()
}

// LogFormatter emulates the form of the traditional built-in logger.
type LogFormatter struct {
	logger *log.Logger
	prefix string
}

// NewLogFormatter is a helper to produce a new LogFormatter struct. It uses the
// golang log package to actually do the logging work so that logs look similar.
func NewLogFormatter(w io.Writer, prefix string, flag int) Formatter {
	return &LogFormatter{
		logger: log.New(w, "", flag), // don't use prefix here
		prefix: prefix,               // save it instead
	}
}

// Format builds a log message for the LogFormatter. The LogLevel is ignored.
func (lf *LogFormatter) Format(pkg string, _ LogLevel, _ int, entries ...interface{}) {
	str := fmt.Sprint(entries...)
	prefix := lf.prefix
	if pkg != "" {
		prefix = fmt.Sprintf("%s%s: ", prefix, pkg)
	}
	lf.logger.Output(5, fmt.Sprintf("%s%v", prefix, str)) // call depth is 5
}

// Flush is included so that the interface is complete, but is a no-op.
func (lf *LogFormatter) Flush() {
	// noop
}
<commit_msg>capnslog: Add NilFormatter for silencing unwanted log messages<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package capnslog

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"runtime"
	"strings"
	"time"
)

type Formatter interface {
	Format(pkg string, level LogLevel, depth int, entries ...interface{})
	Flush()
}

func NewStringFormatter(w io.Writer) Formatter {
	return &StringFormatter{
		w: bufio.NewWriter(w),
	}
}

type StringFormatter struct {
	w *bufio.Writer
}

func (s *StringFormatter) Format(pkg string, l LogLevel, i int, entries ...interface{}) {
	now := time.Now().UTC()
	s.w.WriteString(now.Format(time.RFC3339))
	s.w.WriteByte(' ')
	writeEntries(s.w, pkg, l, i, entries...)
	s.Flush()
}

func writeEntries(w *bufio.Writer, pkg string, _ LogLevel, _ int, entries ...interface{}) {
	if pkg != "" {
		w.WriteString(pkg + ": ")
	}
	str := fmt.Sprint(entries...)
	endsInNL := strings.HasSuffix(str, "\n")
	w.WriteString(str)
	if !endsInNL {
		w.WriteString("\n")
	}
}

func (s *StringFormatter) Flush() {
	s.w.Flush()
}

func NewPrettyFormatter(w io.Writer, debug bool) Formatter {
	return &PrettyFormatter{
		w:     bufio.NewWriter(w),
		debug: debug,
	}
}

type PrettyFormatter struct {
	w     *bufio.Writer
	debug bool
}

func (c *PrettyFormatter) Format(pkg string, l LogLevel, depth int, entries ...interface{}) {
	now := time.Now()
	ts := now.Format("2006-01-02 15:04:05")
	c.w.WriteString(ts)
	ms := now.Nanosecond() / 1000
	c.w.WriteString(fmt.Sprintf(".%06d", ms))
	if c.debug {
		_, file, line, ok := runtime.Caller(depth) // It's always the same number of frames to the user's call.
		if !ok {
			file = "???"
			line = 1
		} else {
			slash := strings.LastIndex(file, "/")
			if slash >= 0 {
				file = file[slash+1:]
			}
		}
		if line < 0 {
			line = 0 // not a real line number
		}
		c.w.WriteString(fmt.Sprintf(" [%s:%d]", file, line))
	}
	c.w.WriteString(fmt.Sprint(" ", l.Char(), " | "))
	writeEntries(c.w, pkg, l, depth, entries...)
	c.Flush()
}

func (c *PrettyFormatter) Flush() {
	c.w.Flush()
}

// LogFormatter emulates the form of the traditional built-in logger.
type LogFormatter struct {
	logger *log.Logger
	prefix string
}

// NewLogFormatter is a helper to produce a new LogFormatter struct. It uses the
// golang log package to actually do the logging work so that logs look similar.
func NewLogFormatter(w io.Writer, prefix string, flag int) Formatter {
	return &LogFormatter{
		logger: log.New(w, "", flag), // don't use prefix here
		prefix: prefix,               // save it instead
	}
}

// Format builds a log message for the LogFormatter. The LogLevel is ignored.
func (lf *LogFormatter) Format(pkg string, _ LogLevel, _ int, entries ...interface{}) {
	str := fmt.Sprint(entries...)
	prefix := lf.prefix
	if pkg != "" {
		prefix = fmt.Sprintf("%s%s: ", prefix, pkg)
	}
	lf.logger.Output(5, fmt.Sprintf("%s%v", prefix, str)) // call depth is 5
}

// Flush is included so that the interface is complete, but is a no-op.
func (lf *LogFormatter) Flush() {
	// noop
}

// NilFormatter is a no-op log formatter that does nothing.
type NilFormatter struct {
}

// NewNilFormatter is a helper to produce a new LogFormatter struct. It logs no
// messages so that you can cause part of your logging to be silent.
func NewNilFormatter() Formatter {
	return &NilFormatter{}
}

// Format does nothing.
func (_ *NilFormatter) Format(_ string, _ LogLevel, _ int, _ ...interface{}) {
	// noop
}

// Flush is included so that the interface is complete, but is a no-op.
func (_ *NilFormatter) Flush() {
	// noop
}
<|endoftext|>


<commit_before>package main

import "strings"
import "strconv"
import "fmt"
import "errors"
import "net"
import "net/http"
import "net/url"
import "time"
import "html"
import "log"
import "io"
import "bufio"
import "bytes"
import "flag"
import "encoding/xml"
import "golang.org/x/tools/blog/atom"

const versionMajor = 0
const versionMinor = 1
const versionBugfix = 0
const versionGit = true

const bugzillaDateFormat = "2006-01-02 15:04:05 -0700"
const userAgentName = "bugzillatoatom"

// Maximum size a request for a bug-xml is read in byte.
var maxBugRequestRead int64

// Maximum number of requests per second. Set to something negative to disable
var maxRequestsPerSecond int

// Channel to block on during too many requests in a second
var tooManyRequestsBlocker chan bool = make(chan bool)

// returns the minimum of the given values
func min(a int, b int) int {
    if a <= b {
        return a
    } else {
        return b
    }
}

// Returns the current version as string in the form major.minor.bugfix(-git)
func getVersion() string {
    if versionGit {
        return strconv.Itoa(versionMajor) + "." + strconv.Itoa(versionMinor) + "." + strconv.Itoa(versionBugfix) + "-git"
    } else {
        return strconv.Itoa(versionMajor) + "." + strconv.Itoa(versionMinor) + "." + strconv.Itoa(versionBugfix)
    }
}

// Read from r until given string is found. Appends toAppend afterwards if string
// is found and in any case returns the result
func readUntilString(r io.Reader, until string, toAppend string) (string, error) {
    var buffer bytes.Buffer
    eofReached := false
    rs := bufio.NewReader(io.LimitReader(r, maxBugRequestRead))

    for !eofReached {
        str, err := rs.ReadString('\n')

        if err == io.EOF {
            eofReached = true
        } else if err != nil {
            return "", errors.New(fmt.Sprintf("Error during reading from url: %s", err))
        }

        index := strings.Index(str, until)

        if index == -1 {
            buffer.WriteString(str)
        } else {
            buffer.WriteString(str[:index])
            buffer.WriteString(toAppend)
            break
        }
    }

    return buffer.String(), nil
}

// Requests the bug from given url
func doRequest(target *url.URL) (string, error) {
    request := http.Request{
        Method: http.MethodGet,
        URL: target,
        Header: http.Header{
            "User-Agent": { userAgentName + "/" + getVersion() },
        },
    }

    resp, err := http.DefaultClient.Do(&request)

    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during GET to url \"%s\": %s", target, err))
    }

    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return "", errors.New(fmt.Sprintf("Request returned status code %d (%s).", resp.StatusCode, http.StatusText(resp.StatusCode)))
    }

    // TODO: maybe we should search for something more clever to abort. <attachment could be given by a user in a report
    return readUntilString(resp.Body, "<attachment", "</bug></bugzilla>")
}

// converts the given xml string into an atom feed
func convertXmlToAtom(inXml string) (string, error) {
    type Who struct {
        Name string `xml:",chardata"`
        RealName string `xml:"name,attr"`
    }

    getFormatedName := func(w Who) string {
        if w.RealName == "" {
            return w.Name
        } else {
            return w.RealName + " (" + w.Name + ")"
        }
    }

    type Comment struct {
        CommentId int `xml:"commentid"`
        CommentCount int `xml:"comment_count"`
        AttachmentID int `xml:"attachid"`
        Who Who `xml:"who"`
        When string `xml:"bug_when"`
        Text string `xml:"thetext"`
    }

    type InResult struct {
        Urlbase string `xml:"urlbase,attr"`
        BugId int `xml:"bug>bug_id"`
        Description string `xml:"bug>short_desc"`
        Comments []Comment `xml:"bug>long_desc"`
    }

    inResult := InResult{}
    err := xml.Unmarshal([]byte(inXml), &inResult)
    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during unmarshalling the xml: %s", err))
    } else if len(inResult.Comments) == 0 {
        // One comment, the initial one, should always be available
        err := errors.New("Zero comments in bug. There should be at least the initial one.")
        return "", err
    }

    updateTime, err := time.Parse(bugzillaDateFormat, inResult.Comments[len(inResult.Comments)-1].When)
    if err != nil {
        return "", errors.New(fmt.Sprintf("Couldn't parse updateTime in initial comment: %s", err))
    }

    inUrl := fmt.Sprintf("%s/show_bug.cgi?id=%d", inResult.Urlbase, inResult.BugId)
    attachmentUrl := fmt.Sprintf("%s/attachment.cgi?id=", inResult.Urlbase)

    feed := &atom.Feed{
        Title: inResult.Description,
        ID: inUrl,
        Link: []atom.Link{atom.Link{Href: inUrl, Rel: "alternate"}},
        Updated: atom.Time(updateTime),
        Author: &atom.Person{Name: getFormatedName(inResult.Comments[0].Who)},
        Entry: make([]*atom.Entry, 0, len(inResult.Comments)),
    }

    for i, comment := range inResult.Comments {
        creationTime, err := time.Parse(bugzillaDateFormat, comment.When)
        if err != nil {
            return "", errors.New(fmt.Sprintf("Couldn't parse updateTime in comment %d: %s", i, err))
        }

        links := []atom.Link{atom.Link{Href: inUrl + "#c" + strconv.Itoa(comment.CommentCount), Rel: "alternate"}}
        if comment.AttachmentID != 0 {
            links = append(links, atom.Link{Href: attachmentUrl + strconv.Itoa(comment.AttachmentID), Rel: "enclosure"})
        }

        entry := &atom.Entry{
            Title: getFormatedName(comment.Who) + ": " + comment.Text[:min(100, len(comment.Text))],
            ID: inUrl + "#c" + strconv.Itoa(comment.CommentCount),
            Link: links,
            Published: atom.Time(creationTime),
            Author: &atom.Person{Name: getFormatedName(comment.Who)},
            Content: &atom.Text{Type: "html", Body: strings.Replace(html.EscapeString(comment.Text), "\n", "
", -1)},
        }

        feed.Entry = append(feed.Entry, entry)
    }

    atom, err := xml.MarshalIndent(feed, "", "\t")
    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during creating the atom feed: %s", err))
    }

    return xml.Header + string(atom), nil
}

// Filters not allowed targets, defined by the given networks.
// Returns false if target is not allowed, otherwise true.
func checkTargetAllowed(target string, forbiddenNetworks []*net.IPNet) (bool, error) {
    if forbiddenNetworks == nil {
        return true, nil
    }

    ips, err := net.LookupIP(target)

    if err != nil {
        return false, err
    }

    for _, ip := range ips {
        for _, ipnet := range forbiddenNetworks {
            if ipnet.Contains(ip) {
                log.Printf("Blocked target \"%s\" since it's IP %s is contained in blocked network %s.\n", target, ip, ipnet)
                return false, nil
            }
        }
    }

    return true, nil
}

func handleConvert(w http.ResponseWriter, r *http.Request, forbiddenNetworks []*net.IPNet) {
    // Block during too many requests in the last second
    if maxRequestsPerSecond >= 0 {
        <-tooManyRequestsBlocker
    }

    // Check for a possible recursive call
    if r.Header != nil {
        for _, agent := range r.Header["User-Agent"] {
            if strings.Contains(agent, userAgentName) {
                log.Printf("Blocked request by %s due to User-Agent \"%s\".\n", r.RemoteAddr, agent)
                errStr := fmt.Sprintf("User-Agent \"%s\" blocked.", r.Header["User-Agent"])
                http.Error(w, errStr, http.StatusForbidden)
                return
            }
        }
    }

    formValueUrl := r.FormValue("url")

    // if the user didn't give a protocol simply assume http
    if !(strings.HasPrefix(formValueUrl, "http://") || strings.HasPrefix(formValueUrl, "https://")) {
        formValueUrl = "http://" + formValueUrl
    }

    target, err := url.Parse(formValueUrl)

    if err != nil {
        log.Printf("Error occurred during parsing the url \"%s\": %s.\n", r.FormValue("url"), err.Error())
        errStr := fmt.Sprintf("Error occurred during parsing the url \"%s\": %s\nAre you sure the url is correct?", r.FormValue("url"), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    parsedQuery := target.Query()
    parsedQuery.Set("ctype", "xml")
    target.RawQuery = parsedQuery.Encode()
    target.Fragment = ""

    portPosition := strings.Index(target.Host, ":")
    var hostWithoutPort string
    if portPosition >= 0 {
        hostWithoutPort = target.Host[:portPosition]
    } else {
        hostWithoutPort = target.Host
    }

    if hostWithoutPort == "" {
        log.Printf("Error occurred during parsing the url \"%s\": No host recognized.\n", formValueUrl)
        errStr := fmt.Sprintf("Error occurred during parsing the url \"%s\": No host recognized.\nAre you sure the url is correct?", formValueUrl)
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    // Theoretically a circumvention is possible if DNS is compromised.
    // First return a harmless IP for the check and another one
    // later for the actual request.
    allowed, err := checkTargetAllowed(hostWithoutPort, forbiddenNetworks)
    if err != nil {
        log.Printf("Error occurred during checking whether the host \"%s\" is blocked: %s.", hostWithoutPort, err.Error())
        errStr := fmt.Sprintf("Error occurred during checking whether the host \"%s\" is blocked: %s.\nAre you sure the url is correct?", hostWithoutPort, err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    if !allowed {
        // No log.Printf here, as it is reported further in with more information
        errStr := fmt.Sprintf("Host \"%s\" of url \"%s\" is blocked.", hostWithoutPort, formValueUrl)
        http.Error(w, errStr, http.StatusForbidden)
        return
    }

    inXml, err := doRequest(target)
    if err != nil {
        log.Printf("Error occurred during fetching the url \"%s\": %s\n", target.String(), err.Error())
        errStr := fmt.Sprintf("Error occurred during fetching the url \"%s\": %s\nAre you sure the url is correct?", target.String(), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    atom, err := convertXmlToAtom(inXml)
    if err != nil {
        log.Printf("Error occurred during conversion of the url \"%s\" to atom: %s\n", target.String(), err.Error())
        errStr := fmt.Sprintf("Error occurred during conversion of the url \"%s\" to atom: %s\nAre you sure the url is correct?", target.String(), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    fmt.Fprintf(w, "%s", atom)
}

func handleMain(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "%s", `

<title>bugzillatoatom</title>

<form action=convert>
    Convert a Bugzilla bug entry into an Atom feed. Enter an url:
    <input type="text" name="url">

`)
}

// Parses a given IP or CIDR into a CIDR. IPs are treated as CIDRs with full bitmask
func parseIPOrCIDR(str string) (*net.IPNet, error) {

    if !strings.Contains(str, "/") {
        if strings.Contains(str, ":") {
            str = str + "/128"
        } else {
            str = str + "/32"
        }
    }

    _, ipnet, err := net.ParseCIDR(str)
    return ipnet, err
}

// To allow forbiddenNetworks to be parsed as argument
type CIDRList []*net.IPNet

func (forbiddenNetworks *CIDRList) String() string {
    strs := []string{}

    for _, ipnet := range *forbiddenNetworks {
        strs = append(strs, ipnet.String())
    }

    return strings.Join(strs, ", ")
}

func (forbiddenNetworks *CIDRList) Set(value string) error {
    ipnet, err := parseIPOrCIDR(value)

    if err == nil {
        *forbiddenNetworks = append(*forbiddenNetworks, ipnet)
    }

    return err
}

func main() {
    version := flag.Bool("version", false, "Print the current version and exit")
    port := flag.Uint64("p", 9080, "Port to bind to")
    maxBugRequestReadFlag := flag.Uint64("requestsize", 1 * 1024 * 1024, "Maximum number of bytes to read during a request to another server.") // 1MiB per default
    flag.IntVar(&maxRequestsPerSecond, "persecond", 5, "Maximum number of requests to another server per second. Set to -1 to disable.")
    forbiddenNetworks := CIDRList{}
    flag.Var(&forbiddenNetworks, "b", "IP or Network in CIDR format to block. If a host is available under any blocked IP it will be blocked. Can be given multiple times.\n\tYou probably want to exclude localhost or local networks both on IPv4 and IPv6.")
    flag.Parse()

    if *version {
        log.Fatalln(getVersion())
    }

    if *maxBugRequestReadFlag & (1 << 63) != 0 {
        log.Fatalln("Too large requestsize")
    } else {
        maxBugRequestRead = int64(*maxBugRequestReadFlag)
    }

    // Add a timeout for the default http.Get() in case something goes wrong
    // on the oher side.
    http.DefaultClient = &http.Client{Timeout: time.Second * 30}

    for i := 0; i < maxRequestsPerSecond; i++ {
        go func() {
            for {
                tooManyRequestsBlocker <- true
                time.Sleep(time.Second)
            }
        }()
    }

    http.HandleFunc("/convert", func(w http.ResponseWriter, r *http.Request) { handleConvert(w, r, forbiddenNetworks) })
    http.HandleFunc("/", handleMain)

    log.Fatal(http.ListenAndServe(":" + strconv.FormatUint(*port, 10), nil))
}<commit_msg>correctly filter out blocked hosts<commit_after>package main

import "strings"
import "strconv"
import "fmt"
import "errors"
import "net"
import "net/http"
import "net/url"
import "context"
import "time"
import "html"
import "log"
import "io"
import "bufio"
import "bytes"
import "flag"
import "encoding/xml"
import "golang.org/x/tools/blog/atom"

const versionMajor = 0
const versionMinor = 1
const versionBugfix = 0
const versionGit = true

const bugzillaDateFormat = "2006-01-02 15:04:05 -0700"
const userAgentName = "bugzillatoatom"

// Maximum size a request for a bug-xml is read in byte.
var maxBugRequestRead int64

// Maximum number of requests per second. Set to something negative to disable
var maxRequestsPerSecond int

// Channel to block on during too many requests in a second
var tooManyRequestsBlocker chan bool = make(chan bool)

// returns the minimum of the given values
func min(a int, b int) int {
    if a <= b {
        return a
    } else {
        return b
    }
}

// Returns the current version as string in the form major.minor.bugfix(-git)
func getVersion() string {
    if versionGit {
        return strconv.Itoa(versionMajor) + "." + strconv.Itoa(versionMinor) + "." + strconv.Itoa(versionBugfix) + "-git"
    } else {
        return strconv.Itoa(versionMajor) + "." + strconv.Itoa(versionMinor) + "." + strconv.Itoa(versionBugfix)
    }
}

// Read from r until given string is found. Appends toAppend afterwards if string
// is found and in any case returns the result
func readUntilString(r io.Reader, until string, toAppend string) (string, error) {
    var buffer bytes.Buffer
    eofReached := false
    rs := bufio.NewReader(io.LimitReader(r, maxBugRequestRead))

    for !eofReached {
        str, err := rs.ReadString('\n')

        if err == io.EOF {
            eofReached = true
        } else if err != nil {
            return "", errors.New(fmt.Sprintf("Error during reading from url: %s", err))
        }

        index := strings.Index(str, until)

        if index == -1 {
            buffer.WriteString(str)
        } else {
            buffer.WriteString(str[:index])
            buffer.WriteString(toAppend)
            break
        }
    }

    return buffer.String(), nil
}

// Requests the bug from given url
func doRequest(target *url.URL) (string, error) {
    request := http.Request{
        Method: http.MethodGet,
        URL: target,
        Header: http.Header{
            "User-Agent": { userAgentName + "/" + getVersion() },
        },
    }

    resp, err := http.DefaultClient.Do(&request)

    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during GET to url \"%s\": %s", target, err))
    }

    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return "", errors.New(fmt.Sprintf("Request returned status code %d (%s).", resp.StatusCode, http.StatusText(resp.StatusCode)))
    }

    // TODO: maybe we should search for something more clever to abort. <attachment could be given by a user in a report
    return readUntilString(resp.Body, "<attachment", "</bug></bugzilla>")
}

// converts the given xml string into an atom feed
func convertXmlToAtom(inXml string) (string, error) {
    type Who struct {
        Name string `xml:",chardata"`
        RealName string `xml:"name,attr"`
    }

    getFormatedName := func(w Who) string {
        if w.RealName == "" {
            return w.Name
        } else {
            return w.RealName + " (" + w.Name + ")"
        }
    }

    type Comment struct {
        CommentId int `xml:"commentid"`
        CommentCount int `xml:"comment_count"`
        AttachmentID int `xml:"attachid"`
        Who Who `xml:"who"`
        When string `xml:"bug_when"`
        Text string `xml:"thetext"`
    }

    type InResult struct {
        Urlbase string `xml:"urlbase,attr"`
        BugId int `xml:"bug>bug_id"`
        Description string `xml:"bug>short_desc"`
        Comments []Comment `xml:"bug>long_desc"`
    }

    inResult := InResult{}
    err := xml.Unmarshal([]byte(inXml), &inResult)
    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during unmarshalling the xml: %s", err))
    } else if len(inResult.Comments) == 0 {
        // One comment, the initial one, should always be available
        err := errors.New("Zero comments in bug. There should be at least the initial one.")
        return "", err
    }

    updateTime, err := time.Parse(bugzillaDateFormat, inResult.Comments[len(inResult.Comments)-1].When)
    if err != nil {
        return "", errors.New(fmt.Sprintf("Couldn't parse updateTime in initial comment: %s", err))
    }

    inUrl := fmt.Sprintf("%s/show_bug.cgi?id=%d", inResult.Urlbase, inResult.BugId)
    attachmentUrl := fmt.Sprintf("%s/attachment.cgi?id=", inResult.Urlbase)

    feed := &atom.Feed{
        Title: inResult.Description,
        ID: inUrl,
        Link: []atom.Link{atom.Link{Href: inUrl, Rel: "alternate"}},
        Updated: atom.Time(updateTime),
        Author: &atom.Person{Name: getFormatedName(inResult.Comments[0].Who)},
        Entry: make([]*atom.Entry, 0, len(inResult.Comments)),
    }

    for i, comment := range inResult.Comments {
        creationTime, err := time.Parse(bugzillaDateFormat, comment.When)
        if err != nil {
            return "", errors.New(fmt.Sprintf("Couldn't parse updateTime in comment %d: %s", i, err))
        }

        links := []atom.Link{atom.Link{Href: inUrl + "#c" + strconv.Itoa(comment.CommentCount), Rel: "alternate"}}
        if comment.AttachmentID != 0 {
            links = append(links, atom.Link{Href: attachmentUrl + strconv.Itoa(comment.AttachmentID), Rel: "enclosure"})
        }

        entry := &atom.Entry{
            Title: getFormatedName(comment.Who) + ": " + comment.Text[:min(100, len(comment.Text))],
            ID: inUrl + "#c" + strconv.Itoa(comment.CommentCount),
            Link: links,
            Published: atom.Time(creationTime),
            Author: &atom.Person{Name: getFormatedName(comment.Who)},
            Content: &atom.Text{Type: "html", Body: strings.Replace(html.EscapeString(comment.Text), "\n", "
", -1)},
        }

        feed.Entry = append(feed.Entry, entry)
    }

    atom, err := xml.MarshalIndent(feed, "", "\t")
    if err != nil {
        return "", errors.New(fmt.Sprintf("Error during creating the atom feed: %s", err))
    }

    return xml.Header + string(atom), nil
}

// Checks for not allowed targets, defined by the given networks.
// Returns a list of IPs if target is allowed, otherwise
// throws an error
func targetAllowedIps(target string, forbiddenNetworks []*net.IPNet) ([]net.IP, error) {
    ips, err := net.LookupIP(target)

    if forbiddenNetworks == nil {
        return ips, nil
    }

    if err != nil {
        return []net.IP{}, err
    }

    for _, ip := range ips {
        for _, ipnet := range forbiddenNetworks {
            if ipnet.Contains(ip) {
                log.Printf("Blocked target \"%s\" since it's IP %s is contained in blocked network %s.\n", target, ip, ipnet)
                errStr := fmt.Sprintf("Target \"%s\" blocked.", target)
                return []net.IP{}, errors.New(errStr)
            }
        }
    }

    return ips, nil
}

func handleConvert(w http.ResponseWriter, r *http.Request) {
    // Block during too many requests in the last second
    if maxRequestsPerSecond >= 0 {
        <-tooManyRequestsBlocker
    }

    // Check for a possible recursive call
    if r.Header != nil {
        for _, agent := range r.Header["User-Agent"] {
            if strings.Contains(agent, userAgentName) {
                log.Printf("Blocked request by %s due to User-Agent \"%s\".\n", r.RemoteAddr, agent)
                errStr := fmt.Sprintf("User-Agent \"%s\" blocked.", r.Header["User-Agent"])
                http.Error(w, errStr, http.StatusForbidden)
                return
            }
        }
    }

    formValueUrl := r.FormValue("url")

    // if the user didn't give a protocol simply assume http
    if !(strings.HasPrefix(formValueUrl, "http://") || strings.HasPrefix(formValueUrl, "https://")) {
        formValueUrl = "http://" + formValueUrl
    }

    target, err := url.Parse(formValueUrl)

    if err != nil {
        log.Printf("Error occurred during parsing the url \"%s\": %s.\n", r.FormValue("url"), err.Error())
        errStr := fmt.Sprintf("Error occurred during parsing the url \"%s\": %s\nAre you sure the url is correct?", r.FormValue("url"), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    parsedQuery := target.Query()
    parsedQuery.Set("ctype", "xml")
    target.RawQuery = parsedQuery.Encode()
    target.Fragment = ""

    inXml, err := doRequest(target)
    if err != nil {
        log.Printf("Error occurred during fetching the url \"%s\": %s\n", target.String(), err.Error())
        errStr := fmt.Sprintf("Error occurred during fetching the url \"%s\": %s\nAre you sure the url is correct?", target.String(), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    atom, err := convertXmlToAtom(inXml)
    if err != nil {
        log.Printf("Error occurred during conversion of the url \"%s\" to atom: %s\n", target.String(), err.Error())
        errStr := fmt.Sprintf("Error occurred during conversion of the url \"%s\" to atom: %s\nAre you sure the url is correct?", target.String(), err.Error())
        http.Error(w, errStr, http.StatusInternalServerError)
        return
    }

    fmt.Fprintf(w, "%s", atom)
}

func handleMain(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "%s", `

<title>bugzillatoatom</title>

<form action=convert>
    Convert a Bugzilla bug entry into an Atom feed. Enter an url:
    <input type="text" name="url">

`)
}

// Parses a given IP or CIDR into a CIDR. IPs are treated as CIDRs with full bitmask
func parseIPOrCIDR(str string) (*net.IPNet, error) {

    if !strings.Contains(str, "/") {
        if strings.Contains(str, ":") {
            str = str + "/128"
        } else {
            str = str + "/32"
        }
    }

    _, ipnet, err := net.ParseCIDR(str)
    return ipnet, err
}

// To allow forbiddenNetworks to be parsed as argument
type CIDRList []*net.IPNet

func (forbiddenNetworks *CIDRList) String() string {
    strs := []string{}

    for _, ipnet := range *forbiddenNetworks {
        strs = append(strs, ipnet.String())
    }

    return strings.Join(strs, ", ")
}

func (forbiddenNetworks *CIDRList) Set(value string) error {
    ipnet, err := parseIPOrCIDR(value)

    if err == nil {
        *forbiddenNetworks = append(*forbiddenNetworks, ipnet)
    }

    return err
}

// Sets the http.DefaultClient to a client with timeout which
// blocks connections to a host with an IP in forbiddenNetworks
func setHttpDefaultClient(forbiddenNetworks []*net.IPNet) {
    // Copy the original DefaultTransport and add our dialer
    httpDialer := &net.Dialer{
       Timeout:   30 * time.Second,
       KeepAlive: 30 * time.Second,
    }

    httpTransport := *(http.DefaultTransport.(*http.Transport))
    httpTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
        port := ""
        portPosition := strings.Index(addr, ":")

        if portPosition != -1 {
            port = addr[portPosition:]
            addr = addr[:portPosition]
        }

        ips, err := targetAllowedIps(addr, forbiddenNetworks)

        if err != nil {
            return nil, err
        }

        for _, ip := range ips {
            var ipstr string

            if len(ip.To4()) == net.IPv4len {
                ipstr = ip.String() + port
            } else {
                ipstr = "[" + ip.String() + "]" + port
            }

            conn, err := httpDialer.DialContext(ctx, network, ipstr)

            if err == nil {
                return conn, nil
            }
        }

        return nil, errors.New("Unable to connect to " + addr)
    }

    // Add a timeout for the default http.Get() in case something goes wrong
    // on the other side.
    http.DefaultClient = &http.Client{
        Timeout: time.Second * 30,
        Transport: &httpTransport,
    }
}

func main() {
    version := flag.Bool("version", false, "Print the current version and exit")
    port := flag.Uint64("p", 9080, "Port to bind to")
    maxBugRequestReadFlag := flag.Uint64("requestsize", 1 * 1024 * 1024, "Maximum number of bytes to read during a request to another server.") // 1MiB per default
    flag.IntVar(&maxRequestsPerSecond, "persecond", 5, "Maximum number of requests to another server per second. Set to -1 to disable.")
    forbiddenNetworks := CIDRList{}
    flag.Var(&forbiddenNetworks, "b", "IP or Network in CIDR format to block. If a host is available under any blocked IP it will be blocked. Can be given multiple times.\n\tYou probably want to exclude localhost or local networks both on IPv4 and IPv6.")
    flag.Parse()

    if *version {
        log.Fatalln(getVersion())
    }

    if *maxBugRequestReadFlag & (1 << 63) != 0 {
        log.Fatalln("Too large requestsize")
    } else {
        maxBugRequestRead = int64(*maxBugRequestReadFlag)
    }

    setHttpDefaultClient(forbiddenNetworks)

    for i := 0; i < maxRequestsPerSecond; i++ {
        go func() {
            for {
                tooManyRequestsBlocker <- true
                time.Sleep(time.Second)
            }
        }()
    }

    http.HandleFunc("/convert", handleConvert)
    http.HandleFunc("/", handleMain)

    log.Fatal(http.ListenAndServe(":" + strconv.FormatUint(*port, 10), nil))
}<|endoftext|>


<commit_before>package restruct

import (
	"encoding/binary"
	"fmt"
	"reflect"
)

// Field represents a structure field, similar to reflect.StructField.
type Field struct {
	Name    string
	CanSet  bool
	Type    reflect.Type
	DefType reflect.Type
	Order   binary.ByteOrder
}

// FieldsFromStruct returns a slice of fields for binary packing and unpacking.
func FieldsFromStruct(typ reflect.Type) (result []Field) {
	if typ.Kind() != reflect.Struct {
		panic(fmt.Errorf("tried to get fields from non-struct type %s", typ.Kind().String()))
	}

	count := typ.NumField()
	for i := 0; i < count; i++ {
		val := typ.Field(i)

		// Parse struct tag
		opts := MustParseTag(val.Tag.Get("struct"))
		if opts.Ignore {
			continue
		}

		// Derive type
		ftyp := val.Type
		if opts.Type != nil {
			ftyp = opts.Type
		}

		result = append(result, Field{
			Name:    val.Name,
			CanSet:  true,
			Type:    ftyp,
			DefType: val.Type,
			Order:   opts.Order,
		})
	}

	return result
}

// IsTypeTrivial determines if a given type is constant-size.
func IsTypeTrivial(typ reflect.Type) bool {
	switch typ.Kind() {
	case reflect.Bool,
		reflect.Int,
		reflect.Int8,
		reflect.Int16,
		reflect.Int32,
		reflect.Int64,
		reflect.Uint,
		reflect.Uint8,
		reflect.Uint16,
		reflect.Uint32,
		reflect.Uint64,
		reflect.Uintptr,
		reflect.Float32,
		reflect.Float64,
		reflect.Complex64,
		reflect.Complex128:
		return true
	case reflect.Array, reflect.Ptr:
		return IsTypeTrivial(typ.Elem())
	case reflect.Struct:
		for _, field := range FieldsFromStruct(typ) {
			if !IsTypeTrivial(field.Type) {
				return false
			}
		}
		return true
	default:
		return true
	}
}

// Elem synthesizes a field from an element of a slice, array, or pointer.
// If Field is not a slice, array, or pointer, FieldFromArray panics.
func (f *Field) Elem() *Field {
	return &Field{
		Name:    "*" + f.Name,
		CanSet:  f.CanSet,
		Type:    f.Type.Elem(),
		DefType: f.Type.Elem(),
		Order:   f.Order,
	}
}

// SizeOf determines what the binary size of the field should be.
func (f *Field) SizeOf(val reflect.Value) {

}
<commit_msg>Remove premature "Elem" function<commit_after>package restruct

import (
	"encoding/binary"
	"fmt"
	"reflect"
)

// Field represents a structure field, similar to reflect.StructField.
type Field struct {
	Name    string
	CanSet  bool
	Type    reflect.Type
	DefType reflect.Type
	Order   binary.ByteOrder
}

// FieldsFromStruct returns a slice of fields for binary packing and unpacking.
func FieldsFromStruct(typ reflect.Type) (result []Field) {
	if typ.Kind() != reflect.Struct {
		panic(fmt.Errorf("tried to get fields from non-struct type %s", typ.Kind().String()))
	}

	count := typ.NumField()
	for i := 0; i < count; i++ {
		val := typ.Field(i)

		// Parse struct tag
		opts := MustParseTag(val.Tag.Get("struct"))
		if opts.Ignore {
			continue
		}

		// Derive type
		ftyp := val.Type
		if opts.Type != nil {
			ftyp = opts.Type
		}

		result = append(result, Field{
			Name:    val.Name,
			CanSet:  true,
			Type:    ftyp,
			DefType: val.Type,
			Order:   opts.Order,
		})
	}

	return result
}

// IsTypeTrivial determines if a given type is constant-size.
func IsTypeTrivial(typ reflect.Type) bool {
	switch typ.Kind() {
	case reflect.Bool,
		reflect.Int,
		reflect.Int8,
		reflect.Int16,
		reflect.Int32,
		reflect.Int64,
		reflect.Uint,
		reflect.Uint8,
		reflect.Uint16,
		reflect.Uint32,
		reflect.Uint64,
		reflect.Uintptr,
		reflect.Float32,
		reflect.Float64,
		reflect.Complex64,
		reflect.Complex128:
		return true
	case reflect.Array, reflect.Ptr:
		return IsTypeTrivial(typ.Elem())
	case reflect.Struct:
		for _, field := range FieldsFromStruct(typ) {
			if !IsTypeTrivial(field.Type) {
				return false
			}
		}
		return true
	default:
		return false
	}
}

// SizeOf determines what the binary size of the field should be.
func (f *Field) SizeOf(val reflect.Value) {

}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"github.com/blackbass1988/access_logs_stats/core/output"
	_ "github.com/blackbass1988/access_logs_stats/core/output/console"
	_ "github.com/blackbass1988/access_logs_stats/core/output/zabbix"
	"sort"
	"strconv"
	"strings"
	"sync"
)

type Sender struct {
	//настройка конкретного фильтра
	filter *Filter

	//ссылка на конфиг приложения
	config *Config

	//настроенный отправлятор, реализации настраиваются в конфиге "outputs"
	output *output.Output

	//мап флоатов с реализацией агрегирующих фунций
	floatData map[string]*Float64Data

	//закешированное кол-во секунд в периоде, указанному в "period" конфигурации
	periodInSeconds float64

	//здесь хранятся числа по полям, указанные в "aggregates" конфигурации
	floatsForAggregates map[string][]float64

	//здесь хранятся счетчики уникальных значений по полям, указанные в "counts" конфигурации
	//хранится по схеме поле.уник_значение.кол-во
	//вывод происходит по схеме - кол-во в 1 секунду
	counts map[string]map[string]uint64
}

func (s *Sender) resetData() {
	//locks?
	s.floatsForAggregates = make(map[string][]float64)
	s.floatData = make(map[string]*Float64Data)
	s.counts = make(map[string]map[string]uint64)
}

func (s *Sender) stringMatched(row *RowEntry) bool {

	//micro optimization
	if s.filter.Filter == ".+" || s.filter.Filter == ".*" {
		return true
	}

	return s.filter.FilterRex.MatchString(row.Raw)
}

func (s *Sender) appendIfOk(row *RowEntry) (err error) {

	if s.stringMatched(row) {

		for field, val := range row.Fields {

			if _, ok := s.config.Aggregates[field]; ok {
				valFloat, err := strconv.ParseFloat(val, 10)
				check(err)
				s.floatsForAggregates[field] = append(s.floatsForAggregates[field], valFloat)
			}

			//в конфиге указано поле, как поле, по которому считаются
			// суммы по уникальным значениям
			if _, ok := s.config.Counts[field]; ok {
				if s.counts[field] == nil {
					s.counts[field] = make(map[string]uint64)
				}
				s.counts[field][val]++
			}
		}
	}

	return err
}

func (s *Sender) sendStats() (err error) {

	for _, metricsOfField := range s.filter.Items {

		for _, metric := range metricsOfField.Metrics {
			s.appendToOutput(metricsOfField.Field, metric)
		}
	}
	s.output.Send()

	return err
}

func (s *Sender) appendToOutput(field string, metric string) {
	var (
		cnt    uint64
		key    string
		ok     bool
		result float64
	)

	key = fmt.Sprintf("%s_%v", field, metric)

	switch {
	case metric == "min":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Min()))
	case metric == "max":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Max()))
	case metric == "len":
		s.output.AddMessage(key, fmt.Sprintf("%d", s.getFloatData(field).Len()))
	case metric == "avg":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Avg()))
	case metric == "sum":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Sum()))
	case metric == "sum_ps":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Sum()/s.getPeriodInSeconds()))
	case metric == "ips":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).ItemsPerSeconds(s.getPeriodInSeconds())))
	case strings.Contains(metric, "cent_"):
		cent := strings.Split(metric, "_")
		centFloat, err := strconv.ParseFloat(cent[1], 10)
		check(err)
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Percentile(centFloat)))

	case metric == "uniq":
		s.output.AddMessage(key, fmt.Sprintf("%d", s.getUniqCnt(field)))
	case metric == "uniq_ps":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", float64(s.getUniqCnt(field))/s.getPeriodInSeconds()))
	case strings.Contains(metric, "cps_"):
		metrics := strings.Split(metric, "_")
		metric = metrics[1]
		if _, ok = s.counts[field]; !ok {
			cnt = 0
		} else if cnt, ok = s.counts[field][metric]; !ok {
			cnt = 0
		}
		s.output.AddMessage(key, fmt.Sprintf("%.3f", float64(cnt)/s.getPeriodInSeconds()))
	case strings.Contains(metric, "percentage_"):
		metrics := strings.Split(metric, "_")
		metric = metrics[1]

		total := s.getTotalCountByField(field)

		result = 0
		if cnt, ok = s.counts[field][metric]; ok && total > 0 {
			result = float64(cnt * 100 / total)
		}

		s.output.AddMessage(key, fmt.Sprintf("%.3f", result))
	}
}

func (s *Sender) getTotalCountByField(field string) uint64 {
	var (
		ok  bool
		cnt uint64
	)
	if _, ok = s.counts[field]; !ok {
		return 0
	}

	cnt = 0
	for _, c := range s.counts[field] {
		cnt += c
	}

	return cnt
}

func (s *Sender) getUniqCnt(field string) uint64 {
	var (
		cnt uint64
		ok  bool
	)
	cnt = 0
	if _, ok = s.counts[field]; ok {
		cnt = uint64(len(s.counts[field]))
	}
	return cnt
}
func (s *Sender) getFloatData(field string) *Float64Data {
	//кешируем флоатдату
	if _, ok := s.floatData[field]; !ok {
		f := Float64Data(s.floatsForAggregates[field])
		s.floatData[field] = &f
		sort.Sort(s.floatData[field])
	}
	return s.floatData[field]
}

func (s *Sender) getPeriodInSeconds() float64 {
	if s.periodInSeconds == 0 {
		s.periodInSeconds = s.config.Period.Seconds()
	}
	return s.periodInSeconds
}

func NewSender(filter *Filter, config *Config) (*Sender, error) {
	sender := new(Sender)
	sender.filter = filter
	sender.config = config

	sender.output = new(output.Output)

	if len(filter.Prefix) > 0 {
		sender.output.SetPrefix(filter.Prefix)
	}

	for _, s := range config.Outputs {
		sender.output.Init(s.Type, s.Settings)
	}

	return sender, nil
}

type SenderCollection struct {
	procs  []*Sender
	config *Config
}

func NewSenderCollection(config *Config) *SenderCollection {
	subProcesses := new(SenderCollection)

	processes := []*Sender{}
	for _, f := range config.Filters {
		sp, err := NewSender(f, config)
		check(err)
		processes = append(processes, sp)
	}

	subProcesses.procs = processes
	subProcesses.config = config
	subProcesses.resetData()
	return subProcesses
}

func (s *SenderCollection) resetData() {
	for _, proc := range s.procs {
		proc.resetData()
	}
}

func (s *SenderCollection) appendData(row *RowEntry) error {
	var err error

	for _, proc := range s.procs {
		proc.appendIfOk(row)
	}

	return err
}

func (s *SenderCollection) sendStats() {
	var wg sync.WaitGroup
	for _, proc := range s.procs {
		wg.Add(1)
		go func(proc *Sender) {
			defer wg.Done()
			proc.sendStats()
		}(proc)
	}
	wg.Wait()
}
<commit_msg>bla bla<commit_after>package core

import (
	"fmt"
	"github.com/blackbass1988/access_logs_stats/core/output"
	_ "github.com/blackbass1988/access_logs_stats/core/output/console"
	_ "github.com/blackbass1988/access_logs_stats/core/output/zabbix"
	"sort"
	"strconv"
	"strings"
	"sync"
)

type Sender struct {
	//настройка конкретного фильтра
	filter *Filter

	//ссылка на конфиг приложения
	config *Config

	//настроенный отправлятор, реализации настраиваются в конфиге "outputs"
	output *output.Output

	//мап флоатов с реализацией агрегирующих фунций
	floatData map[string]*Float64Data

	//закешированное кол-во секунд в периоде, указанному в "period" конфигурации
	periodInSeconds float64

	//здесь хранятся числа по полям, указанные в "aggregates" конфигурации
	floatsForAggregates map[string][]float64

	//здесь хранятся счетчики уникальных значений по полям, указанные в "counts" конфигурации
	//хранится по схеме поле.уник_значение.кол-во
	//вывод происходит по схеме - кол-во в 1 секунду
	counts map[string]map[string]uint64
}

func (s *Sender) resetData() {
	//locks?
	s.floatsForAggregates = make(map[string][]float64)
	s.floatData = make(map[string]*Float64Data)
	s.counts = make(map[string]map[string]uint64)
}

func (s *Sender) stringMatched(row *RowEntry) bool {

	//micro optimization
	if s.filter.Filter == ".+" || s.filter.Filter == ".*" {
		return true
	}

	return s.filter.FilterRex.MatchString(row.Raw)
}

func (s *Sender) appendIfOk(row *RowEntry) (err error) {

	if s.stringMatched(row) {

		for field, val := range row.Fields {

			if _, ok := s.config.Aggregates[field]; ok {
				valFloat, err := strconv.ParseFloat(val, 10)
				check(err)
				s.floatsForAggregates[field] = append(s.floatsForAggregates[field], valFloat)
			}

			//в конфиге указано поле, как поле, по которому считаются
			// суммы по уникальным значениям
			if _, ok := s.config.Counts[field]; ok {
				if s.counts[field] == nil {
					s.counts[field] = make(map[string]uint64)
				}
				s.counts[field][val]++
			}
		}
	}

	return err
}

func (s *Sender) sendStats() (err error) {

	for _, metricsOfField := range s.filter.Items {

		for _, metric := range metricsOfField.Metrics {
			s.appendToOutput(metricsOfField.Field, metric)
		}
	}
	s.output.Send()

	return err
}

func (s *Sender) appendToOutput(field string, metric string) {
	var (
		cnt    uint64
		key    string
		ok     bool
		result float64
		periodInSeconds float64
	)

	key = fmt.Sprintf("%s_%v", field, metric)

	switch {
	case metric == "min":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Min()))
	case metric == "max":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Max()))
	case metric == "len":
		s.output.AddMessage(key, fmt.Sprintf("%d", s.getFloatData(field).Len()))
	case metric == "avg":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Avg()))
	case metric == "sum":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Sum()))
	case metric == "sum_ps":

		result := s.getFloatData(field).Sum()
		periodInSeconds = s.getPeriodInSeconds()
		if periodInSeconds == 0 {
			result = 0
		} else {
			result = s.getFloatData(field).Sum()/s.getPeriodInSeconds()
		}

		s.output.AddMessage(key, fmt.Sprintf("%.3f", result))
	case metric == "ips":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).ItemsPerSeconds(s.getPeriodInSeconds())))
	case strings.Contains(metric, "cent_"):
		cent := strings.Split(metric, "_")
		centFloat, err := strconv.ParseFloat(cent[1], 10)
		check(err)
		s.output.AddMessage(key, fmt.Sprintf("%.3f", s.getFloatData(field).Percentile(centFloat)))

	case metric == "uniq":
		s.output.AddMessage(key, fmt.Sprintf("%d", s.getUniqCnt(field)))
	case metric == "uniq_ps":
		s.output.AddMessage(key, fmt.Sprintf("%.3f", float64(s.getUniqCnt(field))/s.getPeriodInSeconds()))
	case strings.Contains(metric, "cps_"):
		metrics := strings.Split(metric, "_")
		metric = metrics[1]
		if _, ok = s.counts[field]; !ok {
			cnt = 0
		} else if cnt, ok = s.counts[field][metric]; !ok {
			cnt = 0
		}
		s.output.AddMessage(key, fmt.Sprintf("%.3f", float64(cnt)/s.getPeriodInSeconds()))
	case strings.Contains(metric, "percentage_"):
		metrics := strings.Split(metric, "_")
		metric = metrics[1]

		total := s.getTotalCountByField(field)

		result = 0
		if cnt, ok = s.counts[field][metric]; ok && total > 0 {
			result = float64(cnt * 100 / total)
		}

		s.output.AddMessage(key, fmt.Sprintf("%.3f", result))
	}
}

func (s *Sender) getTotalCountByField(field string) uint64 {
	var (
		ok  bool
		cnt uint64
	)
	if _, ok = s.counts[field]; !ok {
		return 0
	}

	cnt = 0
	for _, c := range s.counts[field] {
		cnt += c
	}

	return cnt
}

func (s *Sender) getUniqCnt(field string) uint64 {
	var (
		cnt uint64
		ok  bool
	)
	cnt = 0
	if _, ok = s.counts[field]; ok {
		cnt = uint64(len(s.counts[field]))
	}
	return cnt
}
func (s *Sender) getFloatData(field string) *Float64Data {
	//кешируем флоатдату
	if _, ok := s.floatData[field]; !ok {
		f := Float64Data(s.floatsForAggregates[field])
		s.floatData[field] = &f
		sort.Sort(s.floatData[field])
	}
	return s.floatData[field]
}

func (s *Sender) getPeriodInSeconds() float64 {
	if s.periodInSeconds == 0 {
		s.periodInSeconds = s.config.Period.Seconds()
	}
	return s.periodInSeconds
}

func NewSender(filter *Filter, config *Config) (*Sender, error) {
	sender := new(Sender)
	sender.filter = filter
	sender.config = config

	sender.output = new(output.Output)

	if len(filter.Prefix) > 0 {
		sender.output.SetPrefix(filter.Prefix)
	}

	for _, s := range config.Outputs {
		sender.output.Init(s.Type, s.Settings)
	}

	return sender, nil
}

type SenderCollection struct {
	procs  []*Sender
	config *Config
}

func NewSenderCollection(config *Config) *SenderCollection {
	subProcesses := new(SenderCollection)

	processes := []*Sender{}
	for _, f := range config.Filters {
		sp, err := NewSender(f, config)
		check(err)
		processes = append(processes, sp)
	}

	subProcesses.procs = processes
	subProcesses.config = config
	subProcesses.resetData()
	return subProcesses
}

func (s *SenderCollection) resetData() {
	for _, proc := range s.procs {
		proc.resetData()
	}
}

func (s *SenderCollection) appendData(row *RowEntry) error {
	var err error

	for _, proc := range s.procs {
		proc.appendIfOk(row)
	}

	return err
}

func (s *SenderCollection) sendStats() {
	var wg sync.WaitGroup
	for _, proc := range s.procs {
		wg.Add(1)
		go func(proc *Sender) {
			defer wg.Done()
			proc.sendStats()
		}(proc)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package weedharvester

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

// Filer represents a filer for SeaweedFS
type Filer struct {
	url string
}

// Directory represents a directory mapped from the seaweed filer
type Directory struct {
	Directory      string         `json:"Directory"`
	Files          []file         `json:"Files"`
	Subdirectories []subdirectory `json:"Subdirectories"`
}

type subdirectory struct {
	Name string `json:"Name"`
	ID   int    `json:"Id"`
}

type file struct {
	Name string `json:"name"`
	Fid  string `json:"fid"`
}

// Ping checks wether the filer with the given url can be reached or not
func (f *Filer) Ping() error {
	completeURL := addSlashIfNeeded(f.url) + "testUrlWeedharvester"
	response, err := http.Get(completeURL)

	if err != nil {
		return err
	}
	if response.StatusCode > 300 {
		return fmt.Errorf("Bad Statuscode %d while trying to ping %s", response.StatusCode, completeURL)
	}

	return nil
}

// Create creates a new file with the given content name and under the given path
func (f *Filer) Create(content io.Reader, filename string, path string) error {
	var b bytes.Buffer
	writer, err := createMultipartForm(&content, &b)

	if err != nil {
		return err
	}

	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), filename)
	} else {
		url = fmt.Sprintf("%s%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), filename)
	}

	return sendMultipartFormData(writer, &b, url)
}

func (f *Filer) Read(filename string, path string) (io.Reader, error) {
	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), filename)
	} else {
		url = fmt.Sprintf("%s%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), filename)
	}

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 300 {
		return nil, fmt.Errorf("Bad StatusCode reading: %s/%s", filename, path)
	}

	return resp.Body, nil
}

// ReadDirectory returns all files contained in a given directory
func (f *Filer) ReadDirectory(path string, lastFileName string) (*Directory, error) {
	var url string
	if len(lastFileName) != 0 {
		url = fmt.Sprintf("%s%s?lastFileName=%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), lastFileName)
	} else {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path))
	}
	client := http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Add("accept", "application/json")
	resp, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	directory := Directory{}
	err = decodeJSON(resp.Body, &directory)

	if err != nil {
		return nil, err
	}
	return &directory, err
}
<commit_msg>Bugfix for filer ping<commit_after>package weedharvester

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

// Filer represents a filer for SeaweedFS
type Filer struct {
	url string
}

// Directory represents a directory mapped from the seaweed filer
type Directory struct {
	Directory      string         `json:"Directory"`
	Files          []file         `json:"Files"`
	Subdirectories []subdirectory `json:"Subdirectories"`
}

type subdirectory struct {
	Name string `json:"Name"`
	ID   int    `json:"Id"`
}

type file struct {
	Name string `json:"name"`
	Fid  string `json:"fid"`
}

// Ping checks wether the filer with the given url can be reached or not
func (f *Filer) Ping() error {
	completeURL := addSlashIfNeeded(f.url) + "testUrlWeedharvester"
	response, err := http.Get(completeURL)

	if err != nil {
		return err
	}
	if response.StatusCode >= 500 {
		return fmt.Errorf("Bad Statuscode %d while trying to ping %s", response.StatusCode, completeURL)
	}

	return nil
}

// Create creates a new file with the given content name and under the given path
func (f *Filer) Create(content io.Reader, filename string, path string) error {
	var b bytes.Buffer
	writer, err := createMultipartForm(&content, &b)

	if err != nil {
		return err
	}

	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), filename)
	} else {
		url = fmt.Sprintf("%s%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), filename)
	}

	return sendMultipartFormData(writer, &b, url)
}

func (f *Filer) Read(filename string, path string) (io.Reader, error) {
	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), filename)
	} else {
		url = fmt.Sprintf("%s%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), filename)
	}

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 300 {
		return nil, fmt.Errorf("Bad StatusCode reading: %s/%s", filename, path)
	}

	return resp.Body, nil
}

// ReadDirectory returns all files contained in a given directory
func (f *Filer) ReadDirectory(path string, lastFileName string) (*Directory, error) {
	var url string
	if len(lastFileName) != 0 {
		url = fmt.Sprintf("%s%s?lastFileName=%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path), lastFileName)
	} else {
		url = fmt.Sprintf("%s%s", addSlashIfNeeded(f.url), addSlashIfNeeded(path))
	}
	client := http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Add("accept", "application/json")
	resp, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	directory := Directory{}
	err = decodeJSON(resp.Body, &directory)

	if err != nil {
		return nil, err
	}
	return &directory, err
}
<|endoftext|>


<commit_before>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "ADDR"
	TLLHeader    = "TLL"
	DeviceHeader = "DEVICE"
	SingedHeader = "SINGED"

	QuietFormat    = "{{.Singed}}"
	JwtTableFormat = "table {{.Addr}}\t{{.TTL}}\t{{.Device}}\t{{.Singed}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Singed: {{.Singed}}`
		} else {
			ctx.Template = `Addr: {{.Addr}}\nTTL: {{.TTL}}\nDevice: {{.Device}}\nSinged: {{.Singed}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) TLL() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'f', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Singed() string {
	j.AddHeader(SingedHeader)
	return j.jwt.Singed
}
<commit_msg>debug<commit_after>package formatter

import (
	"github.com/BluePecker/JwtAuth/dialog/client/formatter/context"
	"github.com/BluePecker/JwtAuth/dialog/server/parameter/jwt/response"
	"bytes"
	"strconv"
)

const (
	AddrHeader   = "ADDR"
	TLLHeader    = "TLL"
	DeviceHeader = "DEVICE"
	SingedHeader = "SINGED"

	QuietFormat    = "{{.Singed}}"
	JwtTableFormat = "table {{.Addr}}\t{{.TTL}}\t{{.Device}}\t{{.Singed}}"
)

type (
	JsonWebToken struct {
		context.BaseSubjectContext
		truncate bool
		jwt      response.JsonWebToken
	}

	JsonWebTokenContext struct {
		context.Context
		JsonWebTokens []response.JsonWebToken
	}
)

func (ctx JsonWebTokenContext) Write() {
	switch ctx.Template {
	case context.RawKey:
		if ctx.Quiet {
			ctx.Template = `Singed: {{.Singed}}`
		} else {
			//ctx.Template = `Addr: {{.Addr}}\nTTL: {{.TTL}}\nDevice: {{.Device}}\nSinged: {{.Singed}}\n`
			ctx.Template = `Addr: {{.Addr}}\nDevice: {{.Device}}\nSinged: {{.Singed}}\n`
		}
	case context.TableKey:
		if ctx.Quiet {
			ctx.Template = QuietFormat
		} else {
			ctx.Template = JwtTableFormat
		}
	}

	ctx.Buffer = bytes.NewBufferString("")
	ctx.PreFormat()

	tpl, err := ctx.Parser()
	if err != nil {
		return
	}

	for _, jwt := range ctx.JsonWebTokens {
		jwtCtx := &JsonWebToken{
			truncate: ctx.Truncate,
			jwt:      jwt,
		}
		err = ctx.ContextFormat(tpl, jwtCtx)
		if err != nil {
			return
		}
	}

	ctx.FormFormat(tpl, &JsonWebToken{})
}

func (j *JsonWebToken) Addr() string {
	j.AddHeader(AddrHeader)
	return j.jwt.Addr
}

func (j *JsonWebToken) TLL() string {
	j.AddHeader(TLLHeader)
	return strconv.FormatFloat(j.jwt.TTL, 'f', -1, 64)
}

func (j *JsonWebToken) Device() string {
	j.AddHeader(DeviceHeader)
	return j.jwt.Device
}

func (j *JsonWebToken) Singed() string {
	j.AddHeader(SingedHeader)
	return j.jwt.Singed
}
<|endoftext|>


<commit_before>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   uint

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries.
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
}

// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
// and custom min and max size boundaries.
func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	// go guarantees the expected behavior for bit shifts even for shift counts
	// larger than the value width. Bounding the value of polShift allows the compiler
	// to optimize the code for 'digest >> polShift'
	if polShift > 53-8 {
		return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
	}
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			// limit wpos before to elide array bound checks
			wpos = wpos % windowSize
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos++

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++
			if add < minSize {
				continue
			}

			if (digest&c.splitmask) == 0 || add >= maxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos % windowSize

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<commit_msg>Only check for minimum chunk size after finding a split point<commit_after>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   uint

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries.
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
}

// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
// and custom min and max size boundaries.
func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	// go guarantees the expected behavior for bit shifts even for shift counts
	// larger than the value width. Bounding the value of polShift allows the compiler
	// to optimize the code for 'digest >> polShift'
	if polShift > 53-8 {
		return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
	}
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			// limit wpos before to elide array bound checks
			wpos = wpos % windowSize
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos++

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++

			if (digest&c.splitmask) == 0 || add >= maxSize {
				if add < minSize {
					continue
				}

				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos % windowSize

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<|endoftext|>


<commit_before>// Copyright 2014 Benjamin Savs.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math32

import "math"

func Floor(x float32) float32

func floor(x float32) float32 {
	return float32(math.Floor(float64(x)))
}

func Floor2(x, y float32) (float32, float32)

func PlainFloor2(x, y float32) (float32, float32) {
	return Floor(x), Floor(y)
}<commit_msg>Added Ceil().<commit_after>// Copyright 2014 Benjamin Savs.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math32

import "math"

func Floor(x float32) float32

func floor(x float32) float32 {
	return float32(math.Floor(float64(x)))
}

func Floor2(x, y float32) (float32, float32)

func PlainFloor2(x, y float32) (float32, float32) {
	return Floor(x), Floor(y)
}

func Ceil(x float32) float32 {
	return float32(math.Ceil(float64(x)))
}<|endoftext|>


<commit_before>package lendingclub

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const (
	apiVersion  = "v1"
	userAgent   = "Lending Club Go " + version
	version     = "0.1.0"
	contentType = "application/json"
)

const (
	lendingClubAPIURL = "https://api.lendingclub.com/api/investor/" + apiVersion
)

type Client struct {
	*http.Client
	authToken string
	baseURL   string
}

type ErrorResponse struct {
	Errors []APIError `json:"errors"`
}

type APIError struct {
	Field   string `json:"field"`
	Code    string `json:"code"`
	Message string `json:"message"`
}

func NewClient(authToken string, client *http.Client) *Client {
	return newClient(lendingClubAPIURL, authToken, client)
}

func newClient(baseURL, authToken string, client *http.Client) *Client {
	if client == nil {
		client = http.DefaultClient
	}

	return &Client{
		Client:    client,
		baseURL:   baseURL,
		authToken: authToken,
	}
}

func (c *Client) newRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Authorization", c.authToken)
	req.Header.Set("Content-Type", contentType)

	return req, nil
}

func debug(bdy io.Reader) {
	bs, err := ioutil.ReadAll(bdy)
	if err != nil {
		log.Fatal(err)
	}

	log.Fatal(string(bs))
}

func (c *Client) processResponse(res *http.Response, body interface{}) error {
	switch res.StatusCode {
	case http.StatusOK:
		if err := json.NewDecoder(res.Body).Decode(&body); err != nil {
			return err
		}
	case http.StatusBadRequest:
		log.Println("BAD REQUEST")
		debug(res.Body)
	case http.StatusForbidden:
		log.Println("FORBIDDEN")
		debug(res.Body)
	case http.StatusUnauthorized:
		return errors.New("unauthorized")
	case http.StatusNotFound:
		log.Println("Not Found")
		debug(res.Body)
	case http.StatusInternalServerError:
		return errors.New(res.Status)
	default:
		return errors.New("unknown status code " + res.Status)
	}

	return nil
}

type Time struct {
	time.Time
}

const timeFormat = "2006-01-02T15:04:05.999-0700"

func (lct *Time) UnmarshalJSON(b []byte) error {
	if b[0] == '"' && b[len(b)-1] == '"' {
		b = b[1 : len(b)-1]
	}

	t, err := time.Parse(timeFormat, string(b))
	if err != nil {
		return err
	}
	*lct = Time{Time: t}

	return nil
}

func (lct Time) MarshalJSON() ([]byte, error) {
	ts := fmt.Sprintf("%q", lct.Format(timeFormat))
	return []byte(ts), nil
}
<commit_msg>add package documentation<commit_after>/*
Go wrapper for the Lending Club API.

See https://www.lendingclub.com/developers/lc-api.action for more details on the API.
*/
package lendingclub

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const (
	apiVersion  = "v1"
	userAgent   = "Lending Club Go " + version
	version     = "0.1.0"
	contentType = "application/json"
)

const (
	lendingClubAPIURL = "https://api.lendingclub.com/api/investor/" + apiVersion
)

type Client struct {
	*http.Client
	authToken string
	baseURL   string
}

type ErrorResponse struct {
	Errors []APIError `json:"errors"`
}

type APIError struct {
	Field   string `json:"field"`
	Code    string `json:"code"`
	Message string `json:"message"`
}

func NewClient(authToken string, client *http.Client) *Client {
	return newClient(lendingClubAPIURL, authToken, client)
}

func newClient(baseURL, authToken string, client *http.Client) *Client {
	if client == nil {
		client = http.DefaultClient
	}

	return &Client{
		Client:    client,
		baseURL:   baseURL,
		authToken: authToken,
	}
}

func (c *Client) newRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Authorization", c.authToken)
	req.Header.Set("Content-Type", contentType)

	return req, nil
}

func debug(bdy io.Reader) {
	bs, err := ioutil.ReadAll(bdy)
	if err != nil {
		log.Fatal(err)
	}

	log.Fatal(string(bs))
}

func (c *Client) processResponse(res *http.Response, body interface{}) error {
	switch res.StatusCode {
	case http.StatusOK:
		if err := json.NewDecoder(res.Body).Decode(&body); err != nil {
			return err
		}
	case http.StatusBadRequest:
		log.Println("BAD REQUEST")
		debug(res.Body)
	case http.StatusForbidden:
		log.Println("FORBIDDEN")
		debug(res.Body)
	case http.StatusUnauthorized:
		return errors.New("unauthorized")
	case http.StatusNotFound:
		log.Println("Not Found")
		debug(res.Body)
	case http.StatusInternalServerError:
		return errors.New(res.Status)
	default:
		return errors.New("unknown status code " + res.Status)
	}

	return nil
}

type Time struct {
	time.Time
}

const timeFormat = "2006-01-02T15:04:05.999-0700"

func (lct *Time) UnmarshalJSON(b []byte) error {
	if b[0] == '"' && b[len(b)-1] == '"' {
		b = b[1 : len(b)-1]
	}

	t, err := time.Parse(timeFormat, string(b))
	if err != nil {
		return err
	}
	*lct = Time{Time: t}

	return nil
}

func (lct Time) MarshalJSON() ([]byte, error) {
	ts := fmt.Sprintf("%q", lct.Format(timeFormat))
	return []byte(ts), nil
}
<|endoftext|>


<commit_before>package position

import (
	"errors"
	"github.com/driusan/de/demodel"
	"unicode"
)

var Invalid error = errors.New("Invalid Position")

// DotStart can be used as a demodel.Position argument
// to actions
func DotStart(buff demodel.CharBuffer) (uint, error) {
	return buff.Dot.Start, nil
}

// DotEnd can be used as a demodel.Position argument
// to actions
func DotEnd(buff demodel.CharBuffer) (uint, error) {
	return buff.Dot.End, nil
}

func TagDotStart(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return buff.Tagline.Dot.Start, nil
}
func TagDotEnd(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return buff.Tagline.Dot.End, nil
}

func BuffStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) > 0 {
		return 0, nil
	}
	return 0, Invalid
}
func BuffEnd(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) > 0 {
		return uint(len(buff.Buffer) - 1), nil
	}
	return 0, Invalid
}

func PrevChar(buff demodel.CharBuffer) (uint, error) {
	if buff.Dot.Start == 0 {
		return 0, Invalid
	}

	// BUG: This doesn't deal with multibyte UTF-8 runes.
	return buff.Dot.Start - 1, nil
}
func NextChar(buff demodel.CharBuffer) (uint, error) {
	if buff.Dot.End >= uint(len(buff.Buffer)) {
		return uint(len(buff.Buffer)) - 1, Invalid
	}
	return buff.Dot.End + 1, nil
}

func PrevLine(buff demodel.CharBuffer) (uint, error) {
	// how far into the current line is the current character?
	var lineIdx int = -1

	// where does the previous line start, so that we can index
	// lineIdx into it at the end?
	var prevLineStart, curLineStart int = -1, -1
	for i := buff.Dot.Start; i > 0; i-- {
		if uint(len(buff.Buffer)) <= i {
			return buff.Dot.End, Invalid
		}
		if buff.Buffer[i] == '\n' {
			if lineIdx == -1 {
				lineIdx = int(buff.Dot.Start) - int(i)
				curLineStart = int(i)
			} else {
				prevLineStart = int(i)
				// the current line was shorter than lineIdx, so move to the
				// end of the line instead of lineIdx into it.
				if prevLineStart+lineIdx > curLineStart {
					return uint(curLineStart), nil
				}
				break
			}
		}
	}

	// it was the first line, so return the first character
	if prevLineStart < 0 || lineIdx < 0 {
		return 0, nil
	}

	return uint(prevLineStart + lineIdx), nil
}

func NextLine(buff demodel.CharBuffer) (uint, error) {
	// how far into the current line is the current character?
	var lineIdx int = -1

	// calculate how far we are into the current line.
	var nextLineStart, subsequentLine int = -1, -1
	for i := buff.Dot.End; i > 0; i-- {
		if uint(len(buff.Buffer)) <= i {
			return buff.Dot.End, Invalid
		}
		if buff.Buffer[i] == '\n' {
			if lineIdx == -1 {
				lineIdx = int(buff.Dot.End - i)
				break
			}
		}
	}
	if lineIdx < 0 {
		// must be the on the first line, which means Dot
		// is actually the index.
		lineIdx = int(buff.Dot.End)
	}

	// now find the next line start, so we can add to it, and the line
	// after that, so we can check if nextLine+idx goes too far.
	for i := buff.Dot.End + 1; int(i) < len(buff.Buffer); i++ {
		if buff.Buffer[i] == '\n' {
			if nextLineStart == -1 {
				nextLineStart = int(i)
			} else {
				subsequentLine = int(i)
				break
			}
		}
	}

	// must be on the last line, so go to the end of the buffer
	if nextLineStart < 0 {
		return uint(len(buff.Buffer)) - 1, nil
	}

	// calculate the position
	pos := uint(nextLineStart + lineIdx)
	if subsequentLine >= 0 && pos > uint(subsequentLine) {
		// went too far, so return the end of the line
		return uint(subsequentLine) - 1, nil
	}

	if pos >= uint(len(buff.Buffer)) {
		// overflowed the buffer somehow, so return the end of the buffer
		return uint(len(buff.Buffer)) - 1, Invalid
	}

	return pos, nil
}

func EndOfLine(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}

	if buff.Buffer[buff.Dot.End] == '\n' {
		return buff.Dot.End, nil
	}
	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if buff.Buffer[i] == '\n' {
			return i - 1, nil
		}
	}
	return uint(len(buff.Buffer) - 1), nil
}
func StartOfLine(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}
	if buff.Dot.Start == 0 {
		return 0, nil
	}

	for i := buff.Dot.Start - 1; i > 0; i-- {
		if buff.Buffer[i] == '\n' {
			return i + 1, nil
		}
	}
	return 0, nil
}

func CurWordStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}
	if buff.Dot.Start == 0 {
		return 0, nil
	}
	for i := buff.Dot.Start - 1; i > 0; i-- {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			return i + 1, nil
		}
		// some non-space word borders. Note that '.'
		// isn't considered a word border so that opening
		// files by right clicking or hitting enter works.
		switch buff.Buffer[i] {
		case '(', ')', '"', '\'', ',', '/', '[', ']', ';':
			return i + 1, nil
		}

	}
	// no word boundaries found, so the first word is the start..
	return 0, nil
}

func CurWordEnd(buff demodel.CharBuffer) (uint, error) {
	switch len(buff.Buffer) {
	case 0:
		return 0, Invalid
	case 1:
		return 0, nil
	}

	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			return i - 1, nil
		}
		switch buff.Buffer[i] {
		case '(', ')', '"', '\'', ',', '/', '[', ']', ';':
			return i - 1, nil
		}
	}

	// no words found, so the end of the buffer is the end of the word.
	return uint(len(buff.Buffer)) - 1, nil

}

func CurTagWordStart(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return CurWordStart(*buff.Tagline)
}

func CurTagWordEnd(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return CurWordEnd(*buff.Tagline)

}
func NextWordStart(buff demodel.CharBuffer) (uint, error) {
	foundSpace := false
	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			foundSpace = true
			continue
		} else {
			if foundSpace == true {
				return i, nil
			}
		}
	}
	return buff.Dot.End, Invalid
}

func PrevWordStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}

	foundSpace := false
	foundNonSpaceBeforeSpace := false
	for i := buff.Dot.Start; i >= 0; i-- {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			if foundNonSpaceBeforeSpace == true {
				return i + 1, nil
			}
			foundSpace = true
		} else if foundSpace == true {
			foundNonSpaceBeforeSpace = true
		}

	}
	// Got to the start of the buffer, so this must be the start.
	return 0, nil
}
<commit_msg>Fixed panic when pushing b to move back a word at the start of the buffer<commit_after>package position

import (
	"errors"
	"github.com/driusan/de/demodel"
	"unicode"
)

var Invalid error = errors.New("Invalid Position")

// DotStart can be used as a demodel.Position argument
// to actions
func DotStart(buff demodel.CharBuffer) (uint, error) {
	return buff.Dot.Start, nil
}

// DotEnd can be used as a demodel.Position argument
// to actions
func DotEnd(buff demodel.CharBuffer) (uint, error) {
	return buff.Dot.End, nil
}

func TagDotStart(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return buff.Tagline.Dot.Start, nil
}
func TagDotEnd(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return buff.Tagline.Dot.End, nil
}

func BuffStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) > 0 {
		return 0, nil
	}
	return 0, Invalid
}
func BuffEnd(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) > 0 {
		return uint(len(buff.Buffer) - 1), nil
	}
	return 0, Invalid
}

func PrevChar(buff demodel.CharBuffer) (uint, error) {
	if buff.Dot.Start == 0 {
		return 0, Invalid
	}

	// BUG: This doesn't deal with multibyte UTF-8 runes.
	return buff.Dot.Start - 1, nil
}
func NextChar(buff demodel.CharBuffer) (uint, error) {
	if buff.Dot.End >= uint(len(buff.Buffer)) {
		return uint(len(buff.Buffer)) - 1, Invalid
	}
	return buff.Dot.End + 1, nil
}

func PrevLine(buff demodel.CharBuffer) (uint, error) {
	// how far into the current line is the current character?
	var lineIdx int = -1

	// where does the previous line start, so that we can index
	// lineIdx into it at the end?
	var prevLineStart, curLineStart int = -1, -1
	for i := buff.Dot.Start; i > 0; i-- {
		if uint(len(buff.Buffer)) <= i {
			return buff.Dot.End, Invalid
		}
		if buff.Buffer[i] == '\n' {
			if lineIdx == -1 {
				lineIdx = int(buff.Dot.Start) - int(i)
				curLineStart = int(i)
			} else {
				prevLineStart = int(i)
				// the current line was shorter than lineIdx, so move to the
				// end of the line instead of lineIdx into it.
				if prevLineStart+lineIdx > curLineStart {
					return uint(curLineStart), nil
				}
				break
			}
		}
	}

	// it was the first line, so return the first character
	if prevLineStart < 0 || lineIdx < 0 {
		return 0, nil
	}

	return uint(prevLineStart + lineIdx), nil
}

func NextLine(buff demodel.CharBuffer) (uint, error) {
	// how far into the current line is the current character?
	var lineIdx int = -1

	// calculate how far we are into the current line.
	var nextLineStart, subsequentLine int = -1, -1
	for i := buff.Dot.End; i > 0; i-- {
		if uint(len(buff.Buffer)) <= i {
			return buff.Dot.End, Invalid
		}
		if buff.Buffer[i] == '\n' {
			if lineIdx == -1 {
				lineIdx = int(buff.Dot.End - i)
				break
			}
		}
	}
	if lineIdx < 0 {
		// must be the on the first line, which means Dot
		// is actually the index.
		lineIdx = int(buff.Dot.End)
	}

	// now find the next line start, so we can add to it, and the line
	// after that, so we can check if nextLine+idx goes too far.
	for i := buff.Dot.End + 1; int(i) < len(buff.Buffer); i++ {
		if buff.Buffer[i] == '\n' {
			if nextLineStart == -1 {
				nextLineStart = int(i)
			} else {
				subsequentLine = int(i)
				break
			}
		}
	}

	// must be on the last line, so go to the end of the buffer
	if nextLineStart < 0 {
		return uint(len(buff.Buffer)) - 1, nil
	}

	// calculate the position
	pos := uint(nextLineStart + lineIdx)
	if subsequentLine >= 0 && pos > uint(subsequentLine) {
		// went too far, so return the end of the line
		return uint(subsequentLine) - 1, nil
	}

	if pos >= uint(len(buff.Buffer)) {
		// overflowed the buffer somehow, so return the end of the buffer
		return uint(len(buff.Buffer)) - 1, Invalid
	}

	return pos, nil
}

func EndOfLine(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}

	if buff.Buffer[buff.Dot.End] == '\n' {
		return buff.Dot.End, nil
	}
	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if buff.Buffer[i] == '\n' {
			return i - 1, nil
		}
	}
	return uint(len(buff.Buffer) - 1), nil
}
func StartOfLine(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}
	if buff.Dot.Start == 0 {
		return 0, nil
	}

	for i := buff.Dot.Start - 1; i > 0; i-- {
		if buff.Buffer[i] == '\n' {
			return i + 1, nil
		}
	}
	return 0, nil
}

func CurWordStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}
	if buff.Dot.Start == 0 {
		return 0, nil
	}
	for i := buff.Dot.Start - 1; i > 0; i-- {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			return i + 1, nil
		}
		// some non-space word borders. Note that '.'
		// isn't considered a word border so that opening
		// files by right clicking or hitting enter works.
		switch buff.Buffer[i] {
		case '(', ')', '"', '\'', ',', '/', '[', ']', ';':
			return i + 1, nil
		}

	}
	// no word boundaries found, so the first word is the start..
	return 0, nil
}

func CurWordEnd(buff demodel.CharBuffer) (uint, error) {
	switch len(buff.Buffer) {
	case 0:
		return 0, Invalid
	case 1:
		return 0, nil
	}

	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			return i - 1, nil
		}
		switch buff.Buffer[i] {
		case '(', ')', '"', '\'', ',', '/', '[', ']', ';':
			return i - 1, nil
		}
	}

	// no words found, so the end of the buffer is the end of the word.
	return uint(len(buff.Buffer)) - 1, nil

}

func CurTagWordStart(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return CurWordStart(*buff.Tagline)
}

func CurTagWordEnd(buff demodel.CharBuffer) (uint, error) {
	if buff.Tagline == nil || len(buff.Tagline.Buffer) == 0 {
		return 0, Invalid
	}
	return CurWordEnd(*buff.Tagline)

}
func NextWordStart(buff demodel.CharBuffer) (uint, error) {
	foundSpace := false
	for i := buff.Dot.End; i < uint(len(buff.Buffer)); i++ {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			foundSpace = true
			continue
		} else {
			if foundSpace == true {
				return i, nil
			}
		}
	}
	return buff.Dot.End, Invalid
}

func PrevWordStart(buff demodel.CharBuffer) (uint, error) {
	if len(buff.Buffer) == 0 {
		return 0, Invalid
	}
	if buff.Dot.Start == 0 {
		return 0, nil
	}
	foundSpace := false
	foundNonSpaceBeforeSpace := false
	for i := buff.Dot.Start; i >= 0; i-- {
		if unicode.IsSpace(rune(buff.Buffer[i])) {
			if foundNonSpaceBeforeSpace == true {
				return i + 1, nil
			}
			foundSpace = true
		} else if foundSpace == true {
			foundNonSpaceBeforeSpace = true
		}

	}
	// Got to the start of the buffer, so this must be the start.
	return 0, nil
}
<|endoftext|>