<commit_before>package parser

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/imdario/mergo"
	"github.com/pkg/errors"
)

type loader struct {
	name    string
	baseDir string
	seeds   []string
}

func newLoader(name string) *loader {
	d := &loader{name: name}
	d.detectBaseDir()

	return d
}

func (d *loader) detectBaseDir() {
	abs, err := filepath.Abs(filepath.Dir(d.name))
	if err == nil {
		d.baseDir = abs
	}
}

func (d *loader) partial(name string) string {
	b, err := d.read(name)
	if err != nil {
		return ""
	}

	return string(b)
}

func (d *loader) read(name string) ([]byte, error) {
	fname := filepath.Join(d.baseDir, name)
	return ioutil.ReadFile(fname)
}

func (d *loader) unmarshal(name string) (data map[string]interface{}, err error) {
	if name == "" {
		return
	}

	b, err := d.read(name)
	if err != nil {
		return
	}

	err = json.Unmarshal(b, &data)
	if err != nil {
		return
	}

	return
}

func (d *loader) loadSeeds() (map[string]interface{}, error) {
	z := map[string]interface{}{}

	for _, seed := range d.seeds {
		m, err := d.unmarshal(seed)
		if err != nil {
			return z, errors.Wrap(err, fmt.Sprintf("%s -> %s", d.name, seed))
		}

		mergo.Merge(&z, m)
	}

	return z, nil
}

func (d *loader) convert(s string) string {
	var format string
	var re *regexp.Regexp

	switch {
	case strings.Contains(s, "seed"):
		re = regexp.MustCompile(`<!-- seed\((.+)\) -->`)
		format = "seed"
	case strings.Contains(s, "include"):
		re = regexp.MustCompile(`<!-- include\((.+)\) -->`)
		format = `{{partial "%s"}}`
	case strings.Contains(s, "partial"):
		re = regexp.MustCompile(`<!-- partial\((.+)\) -->`)
		format = `{{partial "%s"}}`
	default:
		re = regexp.MustCompile(`<!-- (.+) -->`)
		format = `{%s}`
	}

	rs := re.FindStringSubmatch(s)

	if len(rs) != 2 {
		return s
	}

	if format == "seed" {
		d.seeds = append(d.seeds, rs[1])
		return ""
	}

	return fmt.Sprintf(format, rs[1])
}

func (d *loader) parse() (string, error) {
	f, err := os.Open(d.name)
	if err != nil {
		return "", errors.Wrap(err, d.name)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	cs := []string{}

	for scanner.Scan() {
		switch {
		case strings.HasPrefix(scanner.Text(), "<!--"):
			cs = append(cs, d.convert(scanner.Text()))
		default:
			cs = append(cs, scanner.Text())
		}
	}

	return strings.Join(cs, "\n"), nil
}

func join(ss []interface{}, s string) string {
	xs := []string{}

	for _, s := range ss {
		if z, ok := s.(string); ok {
			xs = append(xs, z)
		}
	}

	return strings.Join(xs, s)
}

// Read reads API blueprint from file as bytes
func Read(name string) ([]byte, error) {
	d := newLoader(name)

	s, err := d.parse()
	if err != nil {
		return nil, err
	}

	data, err := d.loadSeeds()
	if err != nil {
		return nil, err
	}

	b, _ := process(s, data, template.FuncMap{"partial": d.partial})

	funcMap := sprig.TxtFuncMap()
	funcMap["joinString"] = funcMap["join"]
	funcMap["join"] = join
	funcMap["upcase"] = strings.ToUpper

	b, err = process(string(b), data, funcMap)
	if err != nil {
		return nil, err
	}

	return b, nil
}

func process(s string, data interface{}, funcMap template.FuncMap) ([]byte, error) {
	tmpl, err := template.New("apib").Funcs(funcMap).Parse(s)
	if err != nil {
		return nil, err
	}

	z := bytes.NewBufferString("")

	err = tmpl.Execute(z, data)
	if err != nil {
		return nil, err
	}

	return z.Bytes(), nil
}

// Seeds lists filenames of API blueprint's seeds.
func Seeds(name string) []string {
	d := newLoader(name)

	if _, err := d.parse(); err != nil {
		return []string{}
	}

	return d.seeds
}
<commit_msg>use double processing<commit_after>package parser

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/imdario/mergo"
	"github.com/pkg/errors"
)

type loader struct {
	name    string
	baseDir string
	seeds   []string
}

func newLoader(name string) *loader {
	d := &loader{name: name}
	d.detectBaseDir()

	return d
}

func (d *loader) detectBaseDir() {
	abs, err := filepath.Abs(filepath.Dir(d.name))
	if err == nil {
		d.baseDir = abs
	}
}

func (d *loader) partial(name string) string {
	b, err := d.read(name)
	if err != nil {
		return ""
	}

	return string(b)
}

func (d *loader) read(name string) ([]byte, error) {
	fname := filepath.Join(d.baseDir, name)
	return ioutil.ReadFile(fname)
}

func (d *loader) unmarshal(name string) (data map[string]interface{}, err error) {
	if name == "" {
		return
	}

	b, err := d.read(name)
	if err != nil {
		return
	}

	err = json.Unmarshal(b, &data)
	if err != nil {
		return
	}

	return
}

func (d *loader) loadSeeds() (map[string]interface{}, error) {
	z := map[string]interface{}{}

	for _, seed := range d.seeds {
		m, err := d.unmarshal(seed)
		if err != nil {
			return z, errors.Wrap(err, fmt.Sprintf("%s -> %s", d.name, seed))
		}

		mergo.Merge(&z, m)
	}

	return z, nil
}

func (d *loader) convert(s string) string {
	var format string
	var re *regexp.Regexp

	switch {
	case strings.Contains(s, "seed"):
		re = regexp.MustCompile(`<!-- seed\((.+)\) -->`)
		format = "seed"
	case strings.Contains(s, "include"):
		re = regexp.MustCompile(`<!-- include\((.+)\) -->`)
		format = `{{partial "%s"}}`
	case strings.Contains(s, "partial"):
		re = regexp.MustCompile(`<!-- partial\((.+)\) -->`)
		format = `{{partial "%s"}}`
	default:
		re = regexp.MustCompile(`<!-- (.+) -->`)
		format = `{%s}`
	}

	rs := re.FindStringSubmatch(s)

	if len(rs) != 2 {
		return s
	}

	if format == "seed" {
		d.seeds = append(d.seeds, rs[1])
		return ""
	}

	return fmt.Sprintf(format, rs[1])
}

func (d *loader) parse() (string, error) {
	f, err := os.Open(d.name)
	if err != nil {
		return "", errors.Wrap(err, d.name)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	cs := []string{}

	for scanner.Scan() {
		switch {
		case strings.HasPrefix(scanner.Text(), "<!--"):
			cs = append(cs, d.convert(scanner.Text()))
		default:
			cs = append(cs, scanner.Text())
		}
	}

	return strings.Join(cs, "\n"), nil
}

func join(ss []interface{}, s string) string {
	xs := []string{}

	for _, s := range ss {
		if z, ok := s.(string); ok {
			xs = append(xs, z)
		}
	}

	return strings.Join(xs, s)
}

// Read reads API blueprint from file as bytes
func Read(name string) ([]byte, error) {
	d := newLoader(name)

	s, err := d.parse()
	if err != nil {
		return nil, err
	}

	data, err := d.loadSeeds()
	if err != nil {
		return nil, err
	}

	b, err := process(s, data, template.FuncMap{"partial": d.partial})
	if err != nil {
		return nil, err
	}

	// backward compatible
	funcMap := template.FuncMap{
		"join":   join,
		"upcase": strings.ToUpper,
	}

	b, err = process(string(b), data, funcMap)
	if err != nil {
		return nil, err
	}

	return b, nil
}

func process(s string, data interface{}, funcMap template.FuncMap) ([]byte, error) {
	tmpl, err := template.New("apib").Funcs(sprig.TxtFuncMap()).Funcs(funcMap).Parse(s)
	if err != nil {
		return nil, err
	}

	z := bytes.NewBufferString("")

	err = tmpl.Execute(z, data)
	if err != nil {
		return nil, err
	}

	return z.Bytes(), nil
}

// Seeds lists filenames of API blueprint's seeds.
func Seeds(name string) []string {
	d := newLoader(name)

	if _, err := d.parse(); err != nil {
		return []string{}
	}

	return d.seeds
}
<|endoftext|>


<commit_before>package parser

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
    "encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
    "strings"
	"time"

	"github.com/packetzoom/logslammer/buffer"
)

const (
	ack         = "ACKMSG"
	maxKeyLen   = 100 * 1024 * 1024 // 100 mb
	maxValueLen = 250 * 1024 * 1024 // 250 mb
)

type Receiver interface {
	Send(*buffer.Event)
}

type Parser struct {
	Conn       net.Conn
	Recv       Receiver
	wlen, plen uint32
	buffer     io.Reader
}

func New(c net.Conn, r Receiver) *Parser {
	return &Parser{
		Conn: c,
		Recv: r,
	}
}

// ack acknowledges that the payload was received successfully
func (p *Parser) ack() error {
	if _, err := p.Conn.Write([]byte(ack)); err != nil {
		return err
	}

	return nil
}

// readKV parses key value pairs from within the payload
func (p *Parser) readKV() ([]byte, []byte, error) {
	var klen, vlen uint32

	// Read key len
	binary.Read(p.buffer, binary.BigEndian, &klen)

	if klen > maxKeyLen {
		return nil, nil, fmt.Errorf("key exceeds max len %d, got %d bytes", maxKeyLen, klen)
	}

	// Read key
	key := make([]byte, klen)
	_, err := p.buffer.Read(key)
	if err != nil {
		return nil, nil, err
	}

	// Read value len
	binary.Read(p.buffer, binary.BigEndian, &vlen)
	if vlen > maxValueLen {
		return nil, nil, fmt.Errorf("value exceeds max len %d, got %d bytes", maxValueLen, vlen)
	}

	// Read value
	value := make([]byte, vlen)
	_, err = p.buffer.Read(value)
	if err != nil {
		return nil, nil, err
	}

	return key, value, nil
}

// read parses the compressed data frame
func (p *Parser) read() error {
	var seq, count uint32
	var k, v []byte
	var err error

	r, err := zlib.NewReader(p.Conn)
	if err != nil {
		return err
	}
	defer r.Close()

	// Decompress
	buff := new(bytes.Buffer)
	io.Copy(buff, r)
	p.buffer = buff

	b := make([]byte, 2)
	for i := uint32(0); i < p.wlen; i++ {
		n, err := buff.Read(b)
		if err == io.EOF {
			return err
		}

		if n == 0 {
			continue
		}

		switch string(b) {
		case "2D": // window size
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)

			var ev buffer.Event
			fields := make(map[string]interface{})
			fields["timestamp"] = time.Now().Format(time.RFC3339Nano)

			for j := uint32(0); j < count; j++ {
				if k, v, err = p.readKV(); err != nil {
					return err
				}
				fields[string(k)] = string(v)
			}

			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
			ev.Offset, _ = strconv.ParseInt(fields["offset"].(string), 10, 64)
			ev.Line = uint64(seq)
			t := fields["line"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)
		case "2J": // JSON
			log.Printf("Got JSON data")
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)
			jsonData := make([]byte, count)
			_, err := p.buffer.Read(jsonData)
			log.Printf("Got message: %s", jsonData)

			if err != nil {
				return err
			}

			var ev buffer.Event
            var fields map[string]interface{}
            decoder := json.NewDecoder(strings.NewReader(string(jsonData)))
            decoder.UseNumber()
            err = decoder.Decode(&fields)

            if err != nil {
                return err
            }
			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
            jsonNumber := fields["offset"].(json.Number)
			ev.Offset, _ = jsonNumber.Int64()
			ev.Line = uint64(seq)
			t := fields["message"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)

		default:
			return fmt.Errorf("unknown type: %s", b)
		}
	}

	return nil
}

// Parse initialises the read loop and begins parsing the incoming request
func (p *Parser) Parse() {
	b := make([]byte, 2)

Read:
	for {
		n, err := p.Conn.Read(b)
		switch {
		case err == io.EOF:
			break Read
		case n == 0:
			goto Read
		}

		switch string(b) {
		case "2W": // window length
			binary.Read(p.Conn, binary.BigEndian, &p.wlen)
		case "2C": // frame length
			binary.Read(p.Conn, binary.BigEndian, &p.plen)
			if err := p.read(); err != nil {
				log.Printf("[%s] error parsing %v", p.Conn.RemoteAddr().String(), err)
				break Read
			}

			if err := p.ack(); err != nil {
				log.Printf("[%s] error acking %v", p.Conn.RemoteAddr().String(), err)
				break Read
			}
		default:
			// This really shouldn't happen
			log.Printf("[%s] Received unknown type (%s): %s", p.Conn.RemoteAddr().String(), b, err)
			break Read
		}
	}
}
<commit_msg>go fmt<commit_after>package parser

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/packetzoom/logslammer/buffer"
)

const (
	ack         = "ACKMSG"
	maxKeyLen   = 100 * 1024 * 1024 // 100 mb
	maxValueLen = 250 * 1024 * 1024 // 250 mb
)

type Receiver interface {
	Send(*buffer.Event)
}

type Parser struct {
	Conn       net.Conn
	Recv       Receiver
	wlen, plen uint32
	buffer     io.Reader
}

func New(c net.Conn, r Receiver) *Parser {
	return &Parser{
		Conn: c,
		Recv: r,
	}
}

// ack acknowledges that the payload was received successfully
func (p *Parser) ack() error {
	if _, err := p.Conn.Write([]byte(ack)); err != nil {
		return err
	}

	return nil
}

// readKV parses key value pairs from within the payload
func (p *Parser) readKV() ([]byte, []byte, error) {
	var klen, vlen uint32

	// Read key len
	binary.Read(p.buffer, binary.BigEndian, &klen)

	if klen > maxKeyLen {
		return nil, nil, fmt.Errorf("key exceeds max len %d, got %d bytes", maxKeyLen, klen)
	}

	// Read key
	key := make([]byte, klen)
	_, err := p.buffer.Read(key)
	if err != nil {
		return nil, nil, err
	}

	// Read value len
	binary.Read(p.buffer, binary.BigEndian, &vlen)
	if vlen > maxValueLen {
		return nil, nil, fmt.Errorf("value exceeds max len %d, got %d bytes", maxValueLen, vlen)
	}

	// Read value
	value := make([]byte, vlen)
	_, err = p.buffer.Read(value)
	if err != nil {
		return nil, nil, err
	}

	return key, value, nil
}

// read parses the compressed data frame
func (p *Parser) read() error {
	var seq, count uint32
	var k, v []byte
	var err error

	r, err := zlib.NewReader(p.Conn)
	if err != nil {
		return err
	}
	defer r.Close()

	// Decompress
	buff := new(bytes.Buffer)
	io.Copy(buff, r)
	p.buffer = buff

	b := make([]byte, 2)
	for i := uint32(0); i < p.wlen; i++ {
		n, err := buff.Read(b)
		if err == io.EOF {
			return err
		}

		if n == 0 {
			continue
		}

		switch string(b) {
		case "2D": // window size
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)

			var ev buffer.Event
			fields := make(map[string]interface{})
			fields["timestamp"] = time.Now().Format(time.RFC3339Nano)

			for j := uint32(0); j < count; j++ {
				if k, v, err = p.readKV(); err != nil {
					return err
				}
				fields[string(k)] = string(v)
			}

			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
			ev.Offset, _ = strconv.ParseInt(fields["offset"].(string), 10, 64)
			ev.Line = uint64(seq)
			t := fields["line"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)
		case "2J": // JSON
			log.Printf("Got JSON data")
			binary.Read(buff, binary.BigEndian, &seq)
			binary.Read(buff, binary.BigEndian, &count)
			jsonData := make([]byte, count)
			_, err := p.buffer.Read(jsonData)
			log.Printf("Got message: %s", jsonData)

			if err != nil {
				return err
			}

			var ev buffer.Event
			var fields map[string]interface{}
			decoder := json.NewDecoder(strings.NewReader(string(jsonData)))
			decoder.UseNumber()
			err = decoder.Decode(&fields)

			if err != nil {
				return err
			}
			ev.Source = fmt.Sprintf("lumberjack://%s%s", fields["host"], fields["file"])
			jsonNumber := fields["offset"].(json.Number)
			ev.Offset, _ = jsonNumber.Int64()
			ev.Line = uint64(seq)
			t := fields["message"].(string)
			ev.Text = &t
			ev.Fields = &fields

			// Send to the receiver which is a buffer. We block because...
			p.Recv.Send(&ev)

		default:
			return fmt.Errorf("unknown type: %s", b)
		}
	}

	return nil
}

// Parse initialises the read loop and begins parsing the incoming request
func (p *Parser) Parse() {
	b := make([]byte, 2)

Read:
	for {
		n, err := p.Conn.Read(b)
		switch {
		case err == io.EOF:
			break Read
		case n == 0:
			goto Read
		}

		switch string(b) {
		case "2W": // window length
			binary.Read(p.Conn, binary.BigEndian, &p.wlen)
		case "2C": // frame length
			binary.Read(p.Conn, binary.BigEndian, &p.plen)
			if err := p.read(); err != nil {
				log.Printf("[%s] error parsing %v", p.Conn.RemoteAddr().String(), err)
				break Read
			}

			if err := p.ack(); err != nil {
				log.Printf("[%s] error acking %v", p.Conn.RemoteAddr().String(), err)
				break Read
			}
		default:
			// This really shouldn't happen
			log.Printf("[%s] Received unknown type (%s): %s", p.Conn.RemoteAddr().String(), b, err)
			break Read
		}
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"bytes"
	"go/ast"
	"go/parser"
	"go/token"
	"os/exec"
	"strings"
)

const structComment = "easyjson:json"

type Parser struct {
	PkgPath     string
	PkgName     string
	StructNames []string
	AllStructs  bool
}

type visitor struct {
	*Parser

	name     string
	explicit bool
}

func (p *Parser) needType(comments string) bool {
	for _, v := range strings.Split(comments, "\n") {
		if strings.HasPrefix(v, structComment) {
			return true
		}
	}
	return false
}

func (v *visitor) Visit(n ast.Node) (w ast.Visitor) {
	switch n := n.(type) {
	case *ast.Package:
		return v
	case *ast.File:
		v.PkgName = n.Name.String()
		return v

	case *ast.GenDecl:
		v.explicit = v.needType(n.Doc.Text())

		if !v.explicit && !v.AllStructs {
			return nil
		}
		return v
	case *ast.TypeSpec:
		v.name = n.Name.String()

		// Allow to specify non-structs explicitly independent of '-all' flag.
		if v.explicit {
			v.StructNames = append(v.StructNames, v.name)
			return nil
		}
		return v
	case *ast.StructType:
		v.StructNames = append(v.StructNames, v.name)
		return nil
	}
	return nil
}

func (p *Parser) Parse(fname string, isDir bool) error {
	var err error
	if p.PkgPath, err = getPkgPath(fname, isDir); err != nil {
		return err
	}

	fset := token.NewFileSet()
	if isDir {
		packages, err := parser.ParseDir(fset, fname, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, pckg := range packages {
			ast.Walk(&visitor{Parser: p}, pckg)
		}
	} else {
		f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		ast.Walk(&visitor{Parser: p}, f)
	}
	return nil
}

func getDefaultGoPath() (string, error) {
	output, err := exec.Command("go", "env", "GOPATH").Output()
	return string(bytes.TrimSpace(output)), err
}
<commit_msg>Issue #217: handling of _test.go files<commit_after>package parser

import (
	"bytes"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"os/exec"
	"strings"
)

const structComment = "easyjson:json"

type Parser struct {
	PkgPath     string
	PkgName     string
	StructNames []string
	AllStructs  bool
}

type visitor struct {
	*Parser

	name     string
	explicit bool
}

func (p *Parser) needType(comments string) bool {
	for _, v := range strings.Split(comments, "\n") {
		if strings.HasPrefix(v, structComment) {
			return true
		}
	}
	return false
}

func (v *visitor) Visit(n ast.Node) (w ast.Visitor) {
	switch n := n.(type) {
	case *ast.Package:
		return v
	case *ast.File:
		v.PkgName = n.Name.String()
		return v

	case *ast.GenDecl:
		v.explicit = v.needType(n.Doc.Text())

		if !v.explicit && !v.AllStructs {
			return nil
		}
		return v
	case *ast.TypeSpec:
		v.name = n.Name.String()

		// Allow to specify non-structs explicitly independent of '-all' flag.
		if v.explicit {
			v.StructNames = append(v.StructNames, v.name)
			return nil
		}
		return v
	case *ast.StructType:
		v.StructNames = append(v.StructNames, v.name)
		return nil
	}
	return nil
}

func (p *Parser) Parse(fname string, isDir bool) error {
	var err error
	if p.PkgPath, err = getPkgPath(fname, isDir); err != nil {
		return err
	}

	fset := token.NewFileSet()
	if isDir {
		packages, err := parser.ParseDir(fset, fname, excludeTestFiles, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, pckg := range packages {
			ast.Walk(&visitor{Parser: p}, pckg)
		}
	} else {
		f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
		if err != nil {
			return err
		}

		ast.Walk(&visitor{Parser: p}, f)
	}
	return nil
}

func getDefaultGoPath() (string, error) {
	output, err := exec.Command("go", "env", "GOPATH").Output()
	return string(bytes.TrimSpace(output)), err
}

func excludeTestFiles(fi os.FileInfo) bool {
	return !strings.HasSuffix(fi.Name(), "_test.go")
}
<|endoftext|>


<commit_before>// Package parser parses the given file input
// It displays a paginated version of the file and optionally
// filters the data according to user preferences
// This aspect makes it suitable for inspecting log files of any size
package parser

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

var (
	// Current accepted text types
	// This information is useful to let the parser know
	// how to interpret the ginve file input
	textTypes = []string{
		"plain",
		"json",
	}

	// Color functions to help display meaningful input
	success = color.New(color.FgHiBlack, color.BgHiGreen, color.Bold).SprintFunc()
	fail    = color.New(color.FgHiWhite, color.BgRed, color.Bold).SprintFunc()
	alert   = color.New(color.FgHiYellow, color.Bold).SprintFunc()
	info    = color.New(color.FgHiMagenta, color.Bold).SprintFunc()
)

// Parser type definition
type parser struct {
	file    string
	text    string
	filter  string
	lines   int
	page    int
	noColor bool
	regex   *regexp.Regexp
}

// New returns a new parser object
func New(file, text, filter string, lines, page int, noColor bool, regex *regexp.Regexp) *parser {
	return &parser{
		file:    file,
		text:    text,
		filter:  filter,
		lines:   lines,
		page:    page,
		noColor: noColor,
		regex:   regex,
	}
}

// Parse parses the file and shows the output to the user
func (p *parser) Parse() {
	// Check if a valid lines value was provided
	if p.lines <= 0 {
		exitWithError(fail("Error! Option flag -lines must be strictly positive"))
	}
	// Check if a valid page value was provided
	if p.page <= 0 {
		exitWithError(fail("Error! Option flag -page must be strictly positive"))
	}
	// Check if a valid test type was provided
	if !stringInSlice(p.text, textTypes) {
		exitWithError(fail(fmt.Sprintf("Error! Accepted text types are: %s", strings.Join(textTypes, ", "))))
	}
	// Disables colorized output
	if p.noColor {
		color.NoColor = true
	}
	// Disables regex support for characters of length 1
	// If regex remains enabled strange out is given
	// Also there is no sense in having a regex with length of 1
	if len(p.filter) == 1 {
		p.regex = nil
	}
	// Check if a file path was provided
	if p.file == "" {
		exitWithError(fail("Error! File is required"))
	}
	// Set current page to what the parser gave us
	currentPage := p.page
	// Count all the lines and provide all page offsets
	// These offsets help us navigate to any page instantly
	// This is where all the "magic" happens
	pageOffsets, err := p.countLines()

	if err != nil {
		log.Fatal(err)
	}
	// Determine total number of pages
	numPages := len(pageOffsets)
	// If nothing can be shown tell the user
	if numPages == 0 {
		fmt.Println(info("Sorry. Nothing to show here!"))
		return
	}
	// The current page cannot be greater than the total number of pages
	if currentPage > numPages {
		fmt.Printf("\n%s\n\n", fail(fmt.Sprintf("Error! Page number cannot be greater than %d", numPages)))
		return
	}
	// Get the first page output and print it
	output := p.getFilePage(pageOffsets[currentPage-1])

	fmt.Println(output)
	// If no more pages are to be shown stop here
	// If means we only have 1 page which we already displayed
	if numPages == 1 {
		return
	}
	// Show a message telling the user at which page we are right now and prompt to navigate to whatever page
	fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
	// Start a new input scanner
	in := bufio.NewScanner(os.Stdin)
	// Scan for incoming input
	for in.Scan() {
		// We only accept page numbers here
		// This is how the parser knows where to navigate next
		inputPage, err := strconv.Atoi(in.Text())
		if err != nil {
			fmt.Printf("\n%s\n\n", fail("Error! A valid number is required"))
			fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
		} else {
			if inputPage > numPages {
				fmt.Printf("\n%s\n\n", fail(fmt.Sprintf("Error! Page number cannot be greater than %d", numPages)))
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			} else if inputPage < 1 {
				fmt.Printf("\n%s\n\n", fail("Error! Page number cannot be smaller than 1"))
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			} else {
				currentPage = inputPage
				output := p.getFilePage(pageOffsets[currentPage-1])
				fmt.Printf("\n%s\n", output)
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			}
		}
	}

	if err := in.Err(); err != nil {
		log.Fatal(err)
	}
}

// Gets the output for a new page on the input file
func (p *parser) getFilePage(offset int64) string {
	// Open the file
	f, err := os.Open(p.file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	// Navigate to the given offset
	// This way we skip the part we don't need
	// and avoid parsing unnecesary lines
	_, err = f.Seek(offset, io.SeekStart)
	if err != nil {
		log.Fatal(err)
	}
	// Start a new scanner
	s := bufio.NewScanner(f)
	// Set a larger token size just in case
	s.Buffer(nil, 64*1024*1024)
	// Current line. We need to keep track
	// of all current line number to determine
	// when a page ends
	var current int
	// This will hold the final output to be shown to the user
	// It is reponsable to display only 1 page
	var output bytes.Buffer
	// The text that is obtained by the "getOutput" method
	var text string
	// Scan the file and extract all lines
	// Stop when we reach the number of lines per page
	// that the user specified
	for s.Scan() {
		current++
		// Get the line output and added in the buffer
		text = p.getOutput(s.Text())
		fmt.Fprintf(&output, "%s\n", text)
		if current >= p.lines {
			break
		}
	}

	if err := s.Err(); err != nil {
		log.Fatal(err)
	}

	return output.String()
}

// This is where all the "magic" happens
// Here we count all the file lines
// and extract a slice with all page offsets
func (p *parser) countLines() ([]int64, error) {
	// Open the file
	f, err := os.Open(p.file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	// Get file size the know the position
	// in the file of the last byte
	fi, err := f.Stat()
	if err != nil {
		log.Fatal(err)
	}
	lastPosition := fi.Size()
	// Start a new reader
	r := bufio.NewReader(f)
	// Here we store all page offsets for all pages
	var pageOffsets []int64
	// Here we store page offsets corresponding to filtered text
	var filterOffsets []int64
	// The first line
	// We need to keep track of every line
	// to know when a page ends
	var currentLine int
	// We store the last offset to help us
	// calculate the next offset in the slice
	var lastOffset int64
	// This is the current offset and is
	// calculated line by line
	var offset int64
	// This is used to know if page has been hit
	// by a filter provided by the user
	var pageHit bool
	// We start by adding the first page offset which is 0
	pageOffsets = append(pageOffsets, offset)

	for {
		currentLine++
		// Read the input file line by line
		// This may take a while depending on the file size
		// This is the most time consuming portion of the app
		line, err := r.ReadBytes('\n')
		// If we have a line hit it means we have a page hit
		if p.lineHit(line) {
			pageHit = true
		}
		// Compute the current offset
		offset += int64(len(line))
		if currentLine == p.lines {
			// If we have reached the end of the page
			// Extract the last page offset
			lastOffset = pageOffsets[len(pageOffsets)-1]
			// If we have a page hit (from filtered input)
			// Add it to the filter offsets
			if pageHit {
				filterOffsets = append(filterOffsets, pageOffsets[len(pageOffsets)-1])
			}
			// Add it to the current offset
			// Append this offset to the end of our slice
			offset += lastOffset
			if offset < lastPosition {
				pageOffsets = append(pageOffsets, offset)
			}
			// Reset all counters
			// and also page hit
			currentLine = 0
			offset = 0
			pageHit = false
		}

		switch {
		case err == io.EOF:
			// If we reached the end of file
			// we must take into account also the last lines
			// which were not caught by the page offsets
			if currentLine < p.lines && pageHit {
				filterOffsets = append(filterOffsets, pageOffsets[len(pageOffsets)-1])
			}
			// If the input was filtered return filter page offsets
			// Otherwise return normal page offsets
			if p.filter != "" {
				return filterOffsets, nil
			}
			return pageOffsets, nil

		case err != nil:
			return nil, err
		}
	}
}

// This determines if line matches a given filter
func (p *parser) lineHit(line []byte) bool {
	// If no filter was provided then we do not care about this
	if p.filter == "" {
		return false
	}
	// If regex was enbled, search by regex
	// Otherwise search normally by text bytes
	if p.regex != nil {
		found := p.regex.Find(line)
		return found != nil
	}

	return bytes.Contains(line, []byte(p.filter))
}

// This computes the final output
func (p *parser) getOutput(text string) string {
	// Format input as JSON if needed
	if p.text == "json" {
		re := regexp.MustCompile(`{".*:.*}`)
		jsonMatches := re.FindAll([]byte(text), -1)
		if jsonMatches != nil {
			for _, m := range jsonMatches {
				text = strings.Replace(text, string(m), formatJSON(m), -1)
			}
		}
	}
	// If no filter was provided give the text as it is
	if p.filter == "" {
		return text
	}
	// For regex/normal search highlight filtered input text
	if p.regex != nil {
		// Find all matches
		matches := p.regex.FindAllString(text, -1)
		if matches == nil {
			return text
		}
		// Highlight every match in given input
		for _, m := range matches {
			text = strings.Replace(text, m, success(m), -1)
		}
		return text
	}

	return strings.Replace(text, p.filter, success(p.filter), -1)
}
<commit_msg>Changed coloring on foud text<commit_after>// Package parser parses the given file input
// It displays a paginated version of the file and optionally
// filters the data according to user preferences
// This aspect makes it suitable for inspecting log files of any size
package parser

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

var (
	// Current accepted text types
	// This information is useful to let the parser know
	// how to interpret the ginve file input
	textTypes = []string{
		"plain",
		"json",
	}

	// Color functions to help display meaningful input
	success = color.New(color.FgHiGreen, color.Bold).SprintFunc()
	fail    = color.New(color.FgHiWhite, color.BgRed, color.Bold).SprintFunc()
	alert   = color.New(color.FgHiYellow, color.Bold).SprintFunc()
	info    = color.New(color.FgHiMagenta, color.Bold).SprintFunc()
)

// Parser type definition
type parser struct {
	file    string
	text    string
	filter  string
	lines   int
	page    int
	noColor bool
	regex   *regexp.Regexp
}

// New returns a new parser object
func New(file, text, filter string, lines, page int, noColor bool, regex *regexp.Regexp) *parser {
	return &parser{
		file:    file,
		text:    text,
		filter:  filter,
		lines:   lines,
		page:    page,
		noColor: noColor,
		regex:   regex,
	}
}

// Parse parses the file and shows the output to the user
func (p *parser) Parse() {
	// Check if a valid lines value was provided
	if p.lines <= 0 {
		exitWithError(fail("Error! Option flag -lines must be strictly positive"))
	}
	// Check if a valid page value was provided
	if p.page <= 0 {
		exitWithError(fail("Error! Option flag -page must be strictly positive"))
	}
	// Check if a valid test type was provided
	if !stringInSlice(p.text, textTypes) {
		exitWithError(fail(fmt.Sprintf("Error! Accepted text types are: %s", strings.Join(textTypes, ", "))))
	}
	// Disables colorized output
	if p.noColor {
		color.NoColor = true
	}
	// Disables regex support for characters of length 1
	// If regex remains enabled strange out is given
	// Also there is no sense in having a regex with length of 1
	if len(p.filter) == 1 {
		p.regex = nil
	}
	// Check if a file path was provided
	if p.file == "" {
		exitWithError(fail("Error! File is required"))
	}
	// Set current page to what the parser gave us
	currentPage := p.page
	// Count all the lines and provide all page offsets
	// These offsets help us navigate to any page instantly
	// This is where all the "magic" happens
	pageOffsets, err := p.countLines()

	if err != nil {
		log.Fatal(err)
	}
	// Determine total number of pages
	numPages := len(pageOffsets)
	// If nothing can be shown tell the user
	if numPages == 0 {
		fmt.Println(info("Sorry. Nothing to show here!"))
		return
	}
	// The current page cannot be greater than the total number of pages
	if currentPage > numPages {
		fmt.Printf("\n%s\n\n", fail(fmt.Sprintf("Error! Page number cannot be greater than %d", numPages)))
		return
	}
	// Get the first page output and print it
	output := p.getFilePage(pageOffsets[currentPage-1])

	fmt.Println(output)
	// If no more pages are to be shown stop here
	// If means we only have 1 page which we already displayed
	if numPages == 1 {
		return
	}
	// Show a message telling the user at which page we are right now and prompt to navigate to whatever page
	fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
	// Start a new input scanner
	in := bufio.NewScanner(os.Stdin)
	// Scan for incoming input
	for in.Scan() {
		// We only accept page numbers here
		// This is how the parser knows where to navigate next
		inputPage, err := strconv.Atoi(in.Text())
		if err != nil {
			fmt.Printf("\n%s\n\n", fail("Error! A valid number is required"))
			fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
		} else {
			if inputPage > numPages {
				fmt.Printf("\n%s\n\n", fail(fmt.Sprintf("Error! Page number cannot be greater than %d", numPages)))
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			} else if inputPage < 1 {
				fmt.Printf("\n%s\n\n", fail("Error! Page number cannot be smaller than 1"))
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			} else {
				currentPage = inputPage
				output := p.getFilePage(pageOffsets[currentPage-1])
				fmt.Printf("\n%s\n", output)
				fmt.Print(alert(fmt.Sprintf("Page %d/%d. Enter page number to navigate or press Ctrl+C to quit:", currentPage, numPages)), " ")
			}
		}
	}

	if err := in.Err(); err != nil {
		log.Fatal(err)
	}
}

// Gets the output for a new page on the input file
func (p *parser) getFilePage(offset int64) string {
	// Open the file
	f, err := os.Open(p.file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	// Navigate to the given offset
	// This way we skip the part we don't need
	// and avoid parsing unnecesary lines
	_, err = f.Seek(offset, io.SeekStart)
	if err != nil {
		log.Fatal(err)
	}
	// Start a new scanner
	s := bufio.NewScanner(f)
	// Set a larger token size just in case
	s.Buffer(nil, 64*1024*1024)
	// Current line. We need to keep track
	// of all current line number to determine
	// when a page ends
	var current int
	// This will hold the final output to be shown to the user
	// It is reponsable to display only 1 page
	var output bytes.Buffer
	// The text that is obtained by the "getOutput" method
	var text string
	// Scan the file and extract all lines
	// Stop when we reach the number of lines per page
	// that the user specified
	for s.Scan() {
		current++
		// Get the line output and added in the buffer
		text = p.getOutput(s.Text())
		fmt.Fprintf(&output, "%s\n", text)
		if current >= p.lines {
			break
		}
	}

	if err := s.Err(); err != nil {
		log.Fatal(err)
	}

	return output.String()
}

// This is where all the "magic" happens
// Here we count all the file lines
// and extract a slice with all page offsets
func (p *parser) countLines() ([]int64, error) {
	// Open the file
	f, err := os.Open(p.file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	// Get file size the know the position
	// in the file of the last byte
	fi, err := f.Stat()
	if err != nil {
		log.Fatal(err)
	}
	lastPosition := fi.Size()
	// Start a new reader
	r := bufio.NewReader(f)
	// Here we store all page offsets for all pages
	var pageOffsets []int64
	// Here we store page offsets corresponding to filtered text
	var filterOffsets []int64
	// The first line
	// We need to keep track of every line
	// to know when a page ends
	var currentLine int
	// We store the last offset to help us
	// calculate the next offset in the slice
	var lastOffset int64
	// This is the current offset and is
	// calculated line by line
	var offset int64
	// This is used to know if page has been hit
	// by a filter provided by the user
	var pageHit bool
	// We start by adding the first page offset which is 0
	pageOffsets = append(pageOffsets, offset)

	for {
		currentLine++
		// Read the input file line by line
		// This may take a while depending on the file size
		// This is the most time consuming portion of the app
		line, err := r.ReadBytes('\n')
		// If we have a line hit it means we have a page hit
		if p.lineHit(line) {
			pageHit = true
		}
		// Compute the current offset
		offset += int64(len(line))
		if currentLine == p.lines {
			// If we have reached the end of the page
			// Extract the last page offset
			lastOffset = pageOffsets[len(pageOffsets)-1]
			// If we have a page hit (from filtered input)
			// Add it to the filter offsets
			if pageHit {
				filterOffsets = append(filterOffsets, pageOffsets[len(pageOffsets)-1])
			}
			// Add it to the current offset
			// Append this offset to the end of our slice
			offset += lastOffset
			if offset < lastPosition {
				pageOffsets = append(pageOffsets, offset)
			}
			// Reset all counters
			// and also page hit
			currentLine = 0
			offset = 0
			pageHit = false
		}

		switch {
		case err == io.EOF:
			// If we reached the end of file
			// we must take into account also the last lines
			// which were not caught by the page offsets
			if currentLine < p.lines && pageHit {
				filterOffsets = append(filterOffsets, pageOffsets[len(pageOffsets)-1])
			}
			// If the input was filtered return filter page offsets
			// Otherwise return normal page offsets
			if p.filter != "" {
				return filterOffsets, nil
			}
			return pageOffsets, nil

		case err != nil:
			return nil, err
		}
	}
}

// This determines if line matches a given filter
func (p *parser) lineHit(line []byte) bool {
	// If no filter was provided then we do not care about this
	if p.filter == "" {
		return false
	}
	// If regex was enbled, search by regex
	// Otherwise search normally by text bytes
	if p.regex != nil {
		found := p.regex.Find(line)
		return found != nil
	}

	return bytes.Contains(line, []byte(p.filter))
}

// This computes the final output
func (p *parser) getOutput(text string) string {
	// Format input as JSON if needed
	if p.text == "json" {
		re := regexp.MustCompile(`{".*:.*}`)
		jsonMatches := re.FindAll([]byte(text), -1)
		if jsonMatches != nil {
			for _, m := range jsonMatches {
				text = strings.Replace(text, string(m), formatJSON(m), -1)
			}
		}
	}
	// If no filter was provided give the text as it is
	if p.filter == "" {
		return text
	}
	// For regex/normal search highlight filtered input text
	if p.regex != nil {
		// Find all matches
		matches := p.regex.FindAllString(text, -1)
		if matches == nil {
			return text
		}
		// Highlight every match in given input
		for _, m := range matches {
			text = strings.Replace(text, m, success(m), -1)
		}
		return text
	}

	return strings.Replace(text, p.filter, success(p.filter), -1)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"testing"
)

var (
	basepath       = "/var/cache/pbuilder/base.cow"
	basepathJessie = "/var/cache/pbuilder/base-jessie.cow"
	basetgz        = "/var/cache/pbuilder/base.tgz"
	basetgzJessie  = "/var/cache/pbuilder/base-jessie.tgz"
)

func TestSetBasepath(t *testing.T) {
	c := config{}
	c.Codename = "sid"
	c.setBasepath()
	if c.Basepath != basepath {
		t.Fatalf("%v, want: %s", c.Basepath, basepath)
	}
	c.Codename = "jessie"
	c.setBasepath()
	if c.Basepath != basepathJessie {
		t.Fatalf("%v, want: %s", c.Basepath, basepathJessie)
	}
}

func TestSetBasetgz(t *testing.T) {
	c := config{}
	c.Codename = "sid"
	c.setBasetgz()
	if c.Basetgz != basetgz {
		t.Fatalf("%v, want: %s", c.Basetgz, basetgz)
	}
	c.Codename = "jessie"
	c.setBasetgz()
	if c.Basetgz != basetgzJessie {
		t.Fatalf("%v, want: %s", c.Basetgz, basetgzJessie)
	}
}

func TestPreparePbuilderrc(t *testing.T) {
	c := &config{}
	c.TempDirpath = "temp"
	c.WorkDirpath = "temp"
	c.Codename = "sid"
	c.ResultsDirpath = "temp"

	os.Mkdir(c.WorkDirpath, dirPerm)
	if p := c.preparePbuilderrc(); p != "temp/.pbuilderrc" {
		t.Fatalf("%v, want: temp/.pbuilderrc", p)
	}
	c.cleanDirs()
}
<commit_msg>Added unit tests pbuilder.<commit_after>package main

import (
	"os"
	"testing"
)

var (
	basepath       = "/var/cache/pbuilder/base.cow"
	basepathJessie = "/var/cache/pbuilder/base-jessie.cow"
	basetgz        = "/var/cache/pbuilder/base.tgz"
	basetgzJessie  = "/var/cache/pbuilder/base-jessie.tgz"
)

func TestUpdateCowbuilder(t *testing.T) {
	c := config{}
	rnr = testRunner{}
	c.updateCowbuilder()
}

func TestUpdatePbuilder(t *testing.T) {
	c := config{}
	rnr = testRunner{}
	c.updatePbuilder()
}

func TestBuildPkg(t *testing.T) {
	rnr = testRunner{}
	buildPkg("/path/to/pbuilderrc", "/path/to/base.cow", "/path/to/some.dsc")
}

func TestMkBuildDeps(t *testing.T) {
	rnr = testRunner{}
	mkBuildDeps("/path/to/control")
}

func TestPurgeBuildDeps(t *testing.T) {
	rnr = testRunner{}
	purgeBuildDeps("somepkg")
}

func TestGitBuildPkg(t *testing.T) {
	c := config{}
	rnr = testRunner{}
	c.gitBuildPkg()
}

func TestSetBasepath(t *testing.T) {
	c := config{}
	c.Codename = "sid"
	c.setBasepath()
	if c.Basepath != basepath {
		t.Fatalf("%v, want: %s", c.Basepath, basepath)
	}
	c.Codename = "jessie"
	c.setBasepath()
	if c.Basepath != basepathJessie {
		t.Fatalf("%v, want: %s", c.Basepath, basepathJessie)
	}
}

func TestSetBasetgz(t *testing.T) {
	c := config{}
	c.Codename = "sid"
	c.setBasetgz()
	if c.Basetgz != basetgz {
		t.Fatalf("%v, want: %s", c.Basetgz, basetgz)
	}
	c.Codename = "jessie"
	c.setBasetgz()
	if c.Basetgz != basetgzJessie {
		t.Fatalf("%v, want: %s", c.Basetgz, basetgzJessie)
	}
}

func TestPreparePbuilderrc(t *testing.T) {
	c := &config{}
	c.TempDirpath = "temp"
	c.WorkDirpath = "temp"
	c.Codename = "sid"
	c.ResultsDirpath = "temp"

	os.Mkdir(c.WorkDirpath, dirPerm)
	if p := c.preparePbuilderrc(); p != "temp/.pbuilderrc" {
		t.Fatalf("%v, want: temp/.pbuilderrc", p)
	}
	c.cleanDirs()
}
<|endoftext|>


<commit_before>package colly

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"testing"

	"github.com/PuerkitoBio/goquery"
)

var testServerPort int = 31337
var testServerAddr string = fmt.Sprintf("127.0.0.1:%d", testServerPort)
var testServerRootURL string = fmt.Sprintf("http://%s/", testServerAddr)
var serverIndexResponse []byte = []byte("hello world\n")
var robotsFile string = `
User-agent: *
Allow: /allowed
Disallow: /disallowed
`

func init() {
	srv := &http.Server{}
	listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: testServerPort})
	if err != nil {
		panic(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write(serverIndexResponse)
	})

	http.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Conent-Type", "text/html")
		w.Write([]byte(`<!DOCTYPE html>

<title>Test Page</title>

Hello World

This is a test page

This is a test paragraph

		`))
	})

	http.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			w.Header().Set("Conent-Type", "text/html")
			w.Write([]byte(r.FormValue("name")))
		}
	})

	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte(robotsFile))
	})

	http.HandleFunc("/allowed", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("allowed"))
	})

	http.HandleFunc("/disallowed", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("disallowed"))
	})

	http.HandleFunc("/set_cookie", func(w http.ResponseWriter, r *http.Request) {
		c := &http.Cookie{Name: "test", Value: "testv", HttpOnly: false}
		http.SetCookie(w, c)
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})

	http.HandleFunc("/check_cookie", func(w http.ResponseWriter, r *http.Request) {
		cs := r.Cookies()
		if len(cs) != 1 || r.Cookies()[0].Value != "testv" {
			w.WriteHeader(500)
			w.Write([]byte("nok"))
			return
		}
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})

	go func() {
		if err := srv.Serve(listener); err != nil {
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()
}

func TestCollectorVisit(t *testing.T) {
	c := NewCollector()

	onRequestCalled := false
	onResponseCalled := false
	onScrapedCalled := false

	c.OnRequest(func(r *Request) {
		onRequestCalled = true
		r.Ctx.Put("x", "y")
	})

	c.OnResponse(func(r *Response) {
		onResponseCalled = true

		if r.Ctx.Get("x") != "y" {
			t.Error("Failed to retrieve context value for key 'x'")
		}

		if !bytes.Equal(r.Body, serverIndexResponse) {
			t.Error("Response body does not match with the original content")
		}
	})

	c.OnScraped(func(r *Response) {
		if onResponseCalled == false {
			t.Error("OnScraped called before OnResponse")
		}

		if onRequestCalled == false {
			t.Error("OnScraped called before OnRequest")
		}

		onScrapedCalled = true
	})

	c.Visit(testServerRootURL)

	if !onRequestCalled {
		t.Error("Failed to call OnRequest callback")
	}

	if !onResponseCalled {
		t.Error("Failed to call OnResponse callback")
	}

	if !onScrapedCalled {
		t.Error("Failed to call OnScraped callback")
	}
}

func TestCollectorOnHTML(t *testing.T) {
	c := NewCollector()

	titleCallbackCalled := false
	paragraphCallbackCount := 0

	c.OnHTML("title", func(e *HTMLElement) {
		titleCallbackCalled = true
		if e.Text != "Test Page" {
			t.Error("Title element text does not match, got", e.Text)
		}
	})

	c.OnHTML("p", func(e *HTMLElement) {
		paragraphCallbackCount += 1
		if e.Attr("class") != "description" {
			t.Error("Failed to get paragraph's class attribute")
		}
	})

	c.OnHTML("body", func(e *HTMLElement) {
		if e.ChildAttr("p", "class") != "description" {
			t.Error("Invalid class value")
		}
		classes := e.ChildAttrs("p", "class")
		if len(classes) != 2 {
			t.Error("Invalid class values")
		}
	})

	c.Visit(testServerRootURL + "/html")

	if !titleCallbackCalled {
		t.Error("Failed to call OnHTML callback for <title> tag")
	}

	if paragraphCallbackCount != 2 {
		t.Error("Failed to find all
 tags")
	}
}

func TestCollectorURLRevisit(t *testing.T) {
	c := NewCollector()

	visitCount := 0

	c.OnRequest(func(r *Request) {
		visitCount += 1
	})

	c.Visit(testServerRootURL)
	c.Visit(testServerRootURL)

	if visitCount != 1 {
		t.Error("URL revisited")
	}

	c.AllowURLRevisit = true

	c.Visit(testServerRootURL)
	c.Visit(testServerRootURL)

	if visitCount != 3 {
		t.Error("URL not revisited")
	}
}

func TestCollectorPost(t *testing.T) {
	postValue := "hello"
	c := NewCollector()

	c.OnResponse(func(r *Response) {
		if postValue != string(r.Body) {
			t.Error("Failed to send data with POST")
		}
	})

	c.Post(testServerRootURL+"login", map[string]string{
		"name": postValue,
	})
}

func TestCollectorCookies(t *testing.T) {
	c := NewCollector()

	if err := c.Visit(testServerRootURL + "set_cookie"); err != nil {
		t.Fatal(err)
	}

	if err := c.Visit(testServerRootURL + "check_cookie"); err != nil {
		t.Fatalf("Failed to use previously set cookies: %s", err)
	}
}

func BenchmarkVisit(b *testing.B) {
	c := NewCollector()
	c.OnHTML("p", func(_ *HTMLElement) {})

	for n := 0; n < b.N; n++ {
		c.Visit(fmt.Sprintf("%shtml?q=%d", testServerRootURL, n))
	}
}

func TestRobotsWhenAllowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = false

	c.OnResponse(func(resp *Response) {
		if resp.StatusCode != 200 {
			t.Fatalf("Wrong response code: %d", resp.StatusCode)
		}
	})

	err := c.Visit(testServerRootURL + "allowed")

	if err != nil {
		t.Fatal(err)
	}
}

func TestRobotsWhenDisallowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = false

	c.OnResponse(func(resp *Response) {
		t.Fatalf("Received response: %d", resp.StatusCode)
	})

	err := c.Visit(testServerRootURL + "disallowed")
	if err.Error() != "URL blocked by robots.txt" {
		t.Fatalf("wrong error message: %v", err)
	}
}

func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = true

	c.OnResponse(func(resp *Response) {
		if resp.StatusCode != 200 {
			t.Fatalf("Wrong response code: %d", resp.StatusCode)
		}
	})

	err := c.Visit(testServerRootURL + "disallowed")

	if err != nil {
		t.Fatal(err)
	}

}

func TestHTMLElement(t *testing.T) {
	ctx := &Context{}
	resp := &Response{
		Request: &Request{
			Ctx: ctx,
		},
		Ctx: ctx,
	}

	in := `<a href="http://go-colly.org">Colly</a>`
	sel := "a[href]"
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer([]byte(in)))
	if err != nil {
		t.Fatal(err)
	}
	elements := []*HTMLElement{}
	doc.Find(sel).Each(func(i int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			elements = append(elements, NewHTMLElementFromSelectionNode(resp, s, n))
		}
	})
	elementsLen := len(elements)
	if elementsLen != 1 {
		t.Errorf("element length mismatch. got %d, expected %d.\n", elementsLen, 1)
	}
	v := elements[0]
	if v.Name != "a" {
		t.Errorf("element tag mismatch. got %s, expected %s.\n", v.Name, "a")
	}
	if v.Text != "Colly" {
		t.Errorf("element content mismatch. got %s, expected %s.\n", v.Text, "Colly")
	}
	if v.Attr("href") != "http://go-colly.org" {
		t.Errorf("element href mismatch. got %s, expected %s.\n", v.Attr("href"), "http://go-colly.org")
	}
}
<commit_msg>[enh] small simplification in tests regarding bool checks<commit_after>package colly

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"testing"

	"github.com/PuerkitoBio/goquery"
)

var testServerPort int = 31337
var testServerAddr string = fmt.Sprintf("127.0.0.1:%d", testServerPort)
var testServerRootURL string = fmt.Sprintf("http://%s/", testServerAddr)
var serverIndexResponse []byte = []byte("hello world\n")
var robotsFile string = `
User-agent: *
Allow: /allowed
Disallow: /disallowed
`

func init() {
	srv := &http.Server{}
	listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: testServerPort})
	if err != nil {
		panic(err)
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write(serverIndexResponse)
	})

	http.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Conent-Type", "text/html")
		w.Write([]byte(`<!DOCTYPE html>

<title>Test Page</title>

Hello World

This is a test page

This is a test paragraph

		`))
	})

	http.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			w.Header().Set("Conent-Type", "text/html")
			w.Write([]byte(r.FormValue("name")))
		}
	})

	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte(robotsFile))
	})

	http.HandleFunc("/allowed", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("allowed"))
	})

	http.HandleFunc("/disallowed", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("disallowed"))
	})

	http.HandleFunc("/set_cookie", func(w http.ResponseWriter, r *http.Request) {
		c := &http.Cookie{Name: "test", Value: "testv", HttpOnly: false}
		http.SetCookie(w, c)
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})

	http.HandleFunc("/check_cookie", func(w http.ResponseWriter, r *http.Request) {
		cs := r.Cookies()
		if len(cs) != 1 || r.Cookies()[0].Value != "testv" {
			w.WriteHeader(500)
			w.Write([]byte("nok"))
			return
		}
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})

	go func() {
		if err := srv.Serve(listener); err != nil {
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()
}

func TestCollectorVisit(t *testing.T) {
	c := NewCollector()

	onRequestCalled := false
	onResponseCalled := false
	onScrapedCalled := false

	c.OnRequest(func(r *Request) {
		onRequestCalled = true
		r.Ctx.Put("x", "y")
	})

	c.OnResponse(func(r *Response) {
		onResponseCalled = true

		if r.Ctx.Get("x") != "y" {
			t.Error("Failed to retrieve context value for key 'x'")
		}

		if !bytes.Equal(r.Body, serverIndexResponse) {
			t.Error("Response body does not match with the original content")
		}
	})

	c.OnScraped(func(r *Response) {
		if !onResponseCalled {
			t.Error("OnScraped called before OnResponse")
		}

		if !onRequestCalled {
			t.Error("OnScraped called before OnRequest")
		}

		onScrapedCalled = true
	})

	c.Visit(testServerRootURL)

	if !onRequestCalled {
		t.Error("Failed to call OnRequest callback")
	}

	if !onResponseCalled {
		t.Error("Failed to call OnResponse callback")
	}

	if !onScrapedCalled {
		t.Error("Failed to call OnScraped callback")
	}
}

func TestCollectorOnHTML(t *testing.T) {
	c := NewCollector()

	titleCallbackCalled := false
	paragraphCallbackCount := 0

	c.OnHTML("title", func(e *HTMLElement) {
		titleCallbackCalled = true
		if e.Text != "Test Page" {
			t.Error("Title element text does not match, got", e.Text)
		}
	})

	c.OnHTML("p", func(e *HTMLElement) {
		paragraphCallbackCount += 1
		if e.Attr("class") != "description" {
			t.Error("Failed to get paragraph's class attribute")
		}
	})

	c.OnHTML("body", func(e *HTMLElement) {
		if e.ChildAttr("p", "class") != "description" {
			t.Error("Invalid class value")
		}
		classes := e.ChildAttrs("p", "class")
		if len(classes) != 2 {
			t.Error("Invalid class values")
		}
	})

	c.Visit(testServerRootURL + "/html")

	if !titleCallbackCalled {
		t.Error("Failed to call OnHTML callback for <title> tag")
	}

	if paragraphCallbackCount != 2 {
		t.Error("Failed to find all
 tags")
	}
}

func TestCollectorURLRevisit(t *testing.T) {
	c := NewCollector()

	visitCount := 0

	c.OnRequest(func(r *Request) {
		visitCount += 1
	})

	c.Visit(testServerRootURL)
	c.Visit(testServerRootURL)

	if visitCount != 1 {
		t.Error("URL revisited")
	}

	c.AllowURLRevisit = true

	c.Visit(testServerRootURL)
	c.Visit(testServerRootURL)

	if visitCount != 3 {
		t.Error("URL not revisited")
	}
}

func TestCollectorPost(t *testing.T) {
	postValue := "hello"
	c := NewCollector()

	c.OnResponse(func(r *Response) {
		if postValue != string(r.Body) {
			t.Error("Failed to send data with POST")
		}
	})

	c.Post(testServerRootURL+"login", map[string]string{
		"name": postValue,
	})
}

func TestCollectorCookies(t *testing.T) {
	c := NewCollector()

	if err := c.Visit(testServerRootURL + "set_cookie"); err != nil {
		t.Fatal(err)
	}

	if err := c.Visit(testServerRootURL + "check_cookie"); err != nil {
		t.Fatalf("Failed to use previously set cookies: %s", err)
	}
}

func BenchmarkVisit(b *testing.B) {
	c := NewCollector()
	c.OnHTML("p", func(_ *HTMLElement) {})

	for n := 0; n < b.N; n++ {
		c.Visit(fmt.Sprintf("%shtml?q=%d", testServerRootURL, n))
	}
}

func TestRobotsWhenAllowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = false

	c.OnResponse(func(resp *Response) {
		if resp.StatusCode != 200 {
			t.Fatalf("Wrong response code: %d", resp.StatusCode)
		}
	})

	err := c.Visit(testServerRootURL + "allowed")

	if err != nil {
		t.Fatal(err)
	}
}

func TestRobotsWhenDisallowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = false

	c.OnResponse(func(resp *Response) {
		t.Fatalf("Received response: %d", resp.StatusCode)
	})

	err := c.Visit(testServerRootURL + "disallowed")
	if err.Error() != "URL blocked by robots.txt" {
		t.Fatalf("wrong error message: %v", err)
	}
}

func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
	c := NewCollector()
	c.IgnoreRobotsTxt = true

	c.OnResponse(func(resp *Response) {
		if resp.StatusCode != 200 {
			t.Fatalf("Wrong response code: %d", resp.StatusCode)
		}
	})

	err := c.Visit(testServerRootURL + "disallowed")

	if err != nil {
		t.Fatal(err)
	}

}

func TestHTMLElement(t *testing.T) {
	ctx := &Context{}
	resp := &Response{
		Request: &Request{
			Ctx: ctx,
		},
		Ctx: ctx,
	}

	in := `<a href="http://go-colly.org">Colly</a>`
	sel := "a[href]"
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer([]byte(in)))
	if err != nil {
		t.Fatal(err)
	}
	elements := []*HTMLElement{}
	doc.Find(sel).Each(func(i int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			elements = append(elements, NewHTMLElementFromSelectionNode(resp, s, n))
		}
	})
	elementsLen := len(elements)
	if elementsLen != 1 {
		t.Errorf("element length mismatch. got %d, expected %d.\n", elementsLen, 1)
	}
	v := elements[0]
	if v.Name != "a" {
		t.Errorf("element tag mismatch. got %s, expected %s.\n", v.Name, "a")
	}
	if v.Text != "Colly" {
		t.Errorf("element content mismatch. got %s, expected %s.\n", v.Text, "Colly")
	}
	if v.Attr("href") != "http://go-colly.org" {
		t.Errorf("element href mismatch. got %s, expected %s.\n", v.Attr("href"), "http://go-colly.org")
	}
}
<|endoftext|>


<commit_before>package main

import (
	log "code.google.com/p/log4go"
	itime "github.com/Terry-Mao/goim/libs/time"
	"sync"
	"time"
)

type RoomOptions struct {
	ChannelSize int
	BatchNum    int
	SignalTime  time.Duration
}

type Room struct {
	id     int32
	rLock  sync.RWMutex
	signal chan int
	// TODO use double-linked list
	chs map[*Channel]struct{} // map room id with channels
	// buffer msg
	mn   int
	n    int
	vers []int32
	ops  []int32
	msgs [][]byte
}

// NewRoom new a room struct, store channel room info.
func NewRoom(id int32, t *itime.Timer, options RoomOptions) (r *Room) {
	r = new(Room)
	r.id = id
	r.signal = make(chan int, SignalNum)
	r.chs = make(map[*Channel]struct{}, options.ChannelSize)
	r.n = 0
	r.mn = options.BatchNum
	r.vers = make([]int32, options.BatchNum)
	r.ops = make([]int32, options.BatchNum)
	r.msgs = make([][]byte, options.BatchNum)
	go r.push(t, options.BatchNum, options.SignalTime)
	return
}

// Put put channel into the room.
func (r *Room) Put(ch *Channel) {
	r.rLock.Lock()
	r.chs[ch] = struct{}{}
	r.rLock.Unlock()
	return
}

// Del delete channel from the room.
func (r *Room) Del(ch *Channel) {
	r.rLock.Lock()
	delete(r.chs, ch)
	r.rLock.Unlock()
}

// push merge proto and push msgs in batch.
func (r *Room) push(timer *itime.Timer, batch int, sigTime time.Duration) {
	var (
		done  bool
		ch    *Channel
		last  time.Time
		td    *itime.TimerData
		least time.Duration
	)
	if Debug {
		log.Debug("start room: %d goroutine", r.id)
	}
	td = timer.Add(sigTime, func() {
		select {
		case r.signal <- ProtoReady:
		default:
		}
	})
	for {
		if r.n > 0 {
			if least = sigTime - time.Now().Sub(last); least > 0 {
				timer.Set(td, least)
			} else {
				// timeout
				done = true
			}
		} else {
			last = time.Now()
		}
		if !done {
			if <-r.signal != ProtoReady {
				break
			}
			// try merge msg hard
			if r.n < batch {
				continue
			}
		}
		r.rLock.RLock()
		for ch, _ = range r.chs {
			ch.Pushs(r.vers[:r.n], r.ops[:r.n], r.msgs[:r.n])
		}
		// r.n set here and Push, so RLock is exclusive with Lock
		r.n = 0
		r.rLock.RUnlock()
		done = false
	}
	timer.Del(td)
	if Debug {
		log.Debug("room: %d goroutine exit", r.id)
	}
}

// Push push msg to the room.
func (r *Room) Push(ver int16, operation int32, msg []byte) (err error) {
	r.rLock.Lock()
	if r.n < r.mn {
		r.vers[r.n] = int32(ver)
		r.ops[r.n] = operation
		r.msgs[r.n] = msg
		r.n++
	}
	r.rLock.Unlock()
	select {
	case r.signal <- ProtoReady:
	default:
	}
	return
}

// Online get online number.
func (r *Room) Online() (o int) {
	r.rLock.RLock()
	o = len(r.chs)
	r.rLock.RUnlock()
	return
}

// Close close the room.
func (r *Room) Close() {
	var ch *Channel
	// if chan full, wait
	r.signal <- ProtoFinish
	r.rLock.RLock()
	for ch, _ = range r.chs {
		ch.Close()
	}
	r.rLock.RUnlock()
}
<commit_msg>fix gc bug<commit_after>package main

import (
	log "code.google.com/p/log4go"
	itime "github.com/Terry-Mao/goim/libs/time"
	"sync"
	"time"
)

type RoomOptions struct {
	ChannelSize int
	BatchNum    int
	SignalTime  time.Duration
}

type Room struct {
	id     int32
	rLock  sync.RWMutex
	signal chan int
	// TODO use double-linked list
	chs map[*Channel]struct{} // map room id with channels
	// buffer msg
	mn   int
	n    int
	vers []int32
	ops  []int32
	msgs [][]byte
}

// NewRoom new a room struct, store channel room info.
func NewRoom(id int32, t *itime.Timer, options RoomOptions) (r *Room) {
	r = new(Room)
	r.id = id
	r.signal = make(chan int, SignalNum)
	r.chs = make(map[*Channel]struct{}, options.ChannelSize)
	r.n = 0
	r.mn = options.BatchNum
	r.vers = make([]int32, options.BatchNum)
	r.ops = make([]int32, options.BatchNum)
	r.msgs = make([][]byte, options.BatchNum)
	go r.push(t, options.BatchNum, options.SignalTime)
	return
}

// Put put channel into the room.
func (r *Room) Put(ch *Channel) {
	r.rLock.Lock()
	r.chs[ch] = struct{}{}
	r.rLock.Unlock()
	return
}

// Del delete channel from the room.
func (r *Room) Del(ch *Channel) {
	r.rLock.Lock()
	delete(r.chs, ch)
	r.rLock.Unlock()
}

// push merge proto and push msgs in batch.
func (r *Room) push(timer *itime.Timer, batch int, sigTime time.Duration) {
	var (
		done  bool
		last  time.Time
		least time.Duration
		ch    *Channel
		td    *itime.TimerData
	)
	if Debug {
		log.Debug("start room: %d goroutine", r.id)
	}
	td = timer.Add(sigTime, func() {
		select {
		case r.signal <- ProtoReady:
		default:
		}
	})
	for {
		if r.n > 0 {
			if least = sigTime - time.Now().Sub(last); least > 0 {
				timer.Set(td, least)
			} else {
				// timeout
				done = true
			}
		} else {
			last = time.Now()
		}
		if !done {
			if <-r.signal != ProtoReady {
				break
			}
			// try merge msg hard
			if r.n < batch {
				continue
			}
		}
		r.rLock.RLock()
		for ch, _ = range r.chs {
			ch.Pushs(r.vers[:r.n], r.ops[:r.n], r.msgs[:r.n])
		}
		// r.n set here and Push, so RLock is exclusive with Lock
		r.n = 0
		r.rLock.RUnlock()
		done = false
		ch = nil // avoid gc memory leak
	}
	timer.Del(td)
	if Debug {
		log.Debug("room: %d goroutine exit", r.id)
	}
}

// Push push msg to the room.
func (r *Room) Push(ver int16, operation int32, msg []byte) (err error) {
	r.rLock.Lock()
	if r.n < r.mn {
		r.vers[r.n] = int32(ver)
		r.ops[r.n] = operation
		r.msgs[r.n] = msg
		r.n++
	}
	r.rLock.Unlock()
	select {
	case r.signal <- ProtoReady:
	default:
	}
	return
}

// Online get online number.
func (r *Room) Online() (o int) {
	r.rLock.RLock()
	o = len(r.chs)
	r.rLock.RUnlock()
	return
}

// Close close the room.
func (r *Room) Close() {
	var ch *Channel
	// if chan full, wait
	r.signal <- ProtoFinish
	r.rLock.RLock()
	for ch, _ = range r.chs {
		ch.Close()
	}
	r.rLock.RUnlock()
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"errors"
	"net/http"

	policy "github.com/databus23/goslo.policy"
	"github.com/gorilla/mux"
)

//Token represents a user's token, as passed through the X-Auth-Token header of
//a request.
type Token struct {
	enforcer *policy.Enforcer
	context  policy.Context
	err      error
	//some attributes from the policy.Context that are useful for logging
	UserUUID string
	UserName string
}

//CheckToken checks the validity of the request's X-Auth-Token in Keystone, and
//returns a Token instance for checking authorization. Any errors that occur
//during this function are deferred until Require() is called.
func (p *v1Provider) CheckToken(r *http.Request) *Token {
	str := r.Header.Get("X-Auth-Token")
	if str == "" {
		return &Token{err: errors.New("X-Auth-Token header missing")}
	}

	t := &Token{enforcer: p.Config.API.PolicyEnforcer}
	t.context, t.err = p.Cluster.Config.Auth.ValidateToken(str)
	t.context.Request = mux.Vars(r)
	t.UserUUID = t.context.Auth["user_id"]
	t.UserName = t.context.Auth["user_name"]
	return t
}

//Require checks if the given token has the given permission according to the
//policy.json that is in effect. If not, an error response is written and false
//is returned.
func (t *Token) Require(w http.ResponseWriter, rule string) bool {
	if t.err != nil {
		http.Error(w, t.err.Error(), 401)
		return false
	}

	if !t.enforcer.Enforce(rule, t.context) {
		http.Error(w, "Forbidden", 403)
		return false
	}
	return true
}

//Check is like Require, but does not write error responses.
func (t *Token) Check(rule string) bool {
	return t.err == nil && t.enforcer.Enforce(rule, t.context)
}
<commit_msg>make cluster ID available to the oslo.policy checker<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"errors"
	"net/http"

	policy "github.com/databus23/goslo.policy"
	"github.com/gorilla/mux"
)

//Token represents a user's token, as passed through the X-Auth-Token header of
//a request.
type Token struct {
	enforcer *policy.Enforcer
	context  policy.Context
	err      error
	//some attributes from the policy.Context that are useful for logging
	UserUUID string
	UserName string
}

//CheckToken checks the validity of the request's X-Auth-Token in Keystone, and
//returns a Token instance for checking authorization. Any errors that occur
//during this function are deferred until Require() is called.
func (p *v1Provider) CheckToken(r *http.Request) *Token {
	str := r.Header.Get("X-Auth-Token")
	if str == "" {
		return &Token{err: errors.New("X-Auth-Token header missing")}
	}

	t := &Token{enforcer: p.Config.API.PolicyEnforcer}
	t.context, t.err = p.Cluster.Config.Auth.ValidateToken(str)
	t.context.Request = mux.Vars(r)

	//provide the cluster ID to the policy (this can be used e.g. to restrict
	//cloud-admin operations to a certain cluster)
	if t.context.Auth != nil {
		t.context.Auth["cluster_id"] = p.Cluster.ID
	}

	t.UserUUID = t.context.Auth["user_id"]
	t.UserName = t.context.Auth["user_name"]
	return t
}

//Require checks if the given token has the given permission according to the
//policy.json that is in effect. If not, an error response is written and false
//is returned.
func (t *Token) Require(w http.ResponseWriter, rule string) bool {
	if t.err != nil {
		http.Error(w, t.err.Error(), 401)
		return false
	}

	if !t.enforcer.Enforce(rule, t.context) {
		http.Error(w, "Forbidden", 403)
		return false
	}
	return true
}

//Check is like Require, but does not write error responses.
func (t *Token) Check(rule string) bool {
	return t.err == nil && t.enforcer.Enforce(rule, t.context)
}
<|endoftext|>


<commit_before>// Package auth provides functionality for authenticating
// users to the system.
//
// Default Configuration Values
// * Password Encryption Method: sha256
// * Salt length in bits: 64
package auth

import (
	"crypto/sha256"
	"math/rand"

	"fmt"
)

// Settings is a struct of all the cofiguration options
// used for configuring the auth package
type Settings struct {
	EncryptionMethod string
	SaltLength       int
	UseSalt          bool
}

// Authenticator object for authentication
type Authenticator interface {
	Encrypt(string) (string, string, error)
	Validate(string, string, string) (bool, error)
}

type authAdapter struct {
}

var authSettings Settings

func init() {
	authSettings = Settings{
		EncryptionMethod: "sha256",
		SaltLength:       64,
		UseSalt:          true,
	}
}

// NewAuthenticator returns a new authenticator that can be used for authenticating
func NewAuthenticator(settings *Settings) (Authenticator, error) {
	a := authAdapter{}

	if settings != nil {
		authSettings = *settings
	}

	return &a, nil
}

func (a *authAdapter) Encrypt(v string) (eValue, salt string, err error) {
	if authSettings.UseSalt {
		e, s := sha256WithSalt(v, "")
		eValue = string(e[:])
		salt = string(s[:])
		return
	}

	e := sha256WithoutSalt(v)
	eValue = string(e[:])

	return
}

func (a *authAdapter) Validate(input, salt, password string) (valid bool, err error) {
	var encryptedString string
	valid = false

	if authSettings.UseSalt {
		e, _ := sha256WithSalt(input, salt)
		encryptedString = string(e[:])
	} else {
		e := sha256WithoutSalt(input)
		encryptedString = string(e[:])
	}

	if encryptedString == password {
		valid = true
	}

	return
}

/*
// Read the stored usernames and password hashes from the file
func CheckCredentials(uname string, pword [32]byte) bool {
	valid := false
	file, _ := os.Open(pwloc)
	defer file.Close()

	pws := string(pword[:])
	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		str := scanner.Text()
		line := strings.Split(str, ":")
		pw := line[1]

		if line[0] == uname && pw == pws {
			valid = true
			break
		} else if line[0] == uname && pw == pws {
			break
		}
	}

	return valid
}

// Create a cookie with a life of one day
func SetSessionID(w http.ResponseWriter) {
	var value int

	value = rgen.Intn(max-min) + min
	for used(value) {
		value = rgen.Intn(max-min) + min
	}

	expiration := time.Now().Add(time.Duration(24) * time.Hour)
	cookie := http.Cookie{Name: cookieName, Value: strconv.Itoa(value), Expires: expiration}
	http.SetCookie(w, &cookie)

	writeCookieValue(value)
}

// Check if there is a valid cookie stored
func CheckSessionID(r *http.Request) bool {
	cookie, _ := r.Cookie(cookieName)

	if cookie != nil {
		return checkID(cookie.Value)
	} else {
		return false
	}
}

func RemoveCookie(w http.ResponseWriter, r *http.Request) {
	cookie, _ := r.Cookie(cookieName)

	if cookie != nil {
		cookie.MaxAge = -1
		http.SetCookie(w, cookie)
	}
}

// Check if the ID is in the database of cookies
func checkID(value string) bool {
	valid := false

	file, _ := os.Open(cloc)
	defer file.Close()

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		str := scanner.Text()

		if str == value {
			valid = true
			break
		}
	}

	return valid
}

func UsernameInUse(uname string) bool {
	used := false

	file, _ := os.Open(pwloc)
	defer file.Close()

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		str := scanner.Text()
		line := strings.Split(str, ":")

		if line[0] == uname {
			used = true
			break
		}
	}

	return used
}

func SavePassword(uname string, pword [32]byte) {
	file, _ := os.OpenFile(pwloc, os.O_APPEND|os.O_WRONLY, 0600)
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	w.WriteString(uname + ":" + string(pword[:]) + "\n")
}

func writeCookieValue(value int) {
	file, _ := os.OpenFile(cloc, os.O_APPEND|os.O_WRONLY, 0600)
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	w.WriteString(strconv.Itoa(value) + "\n")
	cookieValues = append(cookieValues, value)
}

func used(value int) bool {
	for _, v := range cookieValues {
		if v == value {
			return true
		}
	}

	return false
}
*/
func sha256WithoutSalt(value string) [sha256.Size]byte {
	encrypted := sha256.Sum256([]byte(value))

	return encrypted
}

func sha256WithSalt(value, saltValue string) ([sha256.Size]byte, []byte) {
	salt := make([]byte, authSettings.SaltLength)

	if saltValue != "" {
		salt = []byte(saltValue)
	} else {
		n, err := rand.Read(salt)

		if err != nil {
			fmt.Println("There was an error generating a salt: ", err)
			return [sha256.Size]byte{}, nil
		}

		if n != authSettings.SaltLength {
			fmt.Printf("Only %d characters were read.\n", n)
			return [sha256.Size]byte{}, nil
		}
	}

	saltedVal := append([]byte(value), salt...)
	encrypted := sha256.Sum256(saltedVal)

	return encrypted, salt
}
<commit_msg>removed old auth functions<commit_after>// Package auth provides functionality for authenticating
// users to the system.
//
// Default Configuration Values
// * Password Encryption Method: sha256
// * Salt length in bits: 64
package auth

import (
	"crypto/sha256"
	"math/rand"

	"fmt"
)

// Settings is a struct of all the cofiguration options
// used for configuring the auth package
type Settings struct {
	EncryptionMethod string
	SaltLength       int
	UseSalt          bool
}

// Authenticator object for authentication
type Authenticator interface {
	Encrypt(string) (string, string, error)
	Validate(string, string, string) (bool, error)
}

type authAdapter struct {
}

var authSettings Settings

func init() {
	authSettings = Settings{
		EncryptionMethod: "sha256",
		SaltLength:       64,
		UseSalt:          true,
	}
}

// NewAuthenticator returns a new authenticator that can be used for authenticating
func NewAuthenticator(settings *Settings) (Authenticator, error) {
	a := authAdapter{}

	if settings != nil {
		authSettings = *settings
	}

	return &a, nil
}

func (a *authAdapter) Encrypt(v string) (eValue, salt string, err error) {
	if authSettings.UseSalt {
		e, s := sha256WithSalt(v, "")
		eValue = string(e[:])
		salt = string(s[:])
		return
	}

	e := sha256WithoutSalt(v)
	eValue = string(e[:])

	return
}

func (a *authAdapter) Validate(input, salt, password string) (valid bool, err error) {
	var encryptedString string
	valid = false

	if authSettings.UseSalt {
		e, _ := sha256WithSalt(input, salt)
		encryptedString = string(e[:])
	} else {
		e := sha256WithoutSalt(input)
		encryptedString = string(e[:])
	}

	if encryptedString == password {
		valid = true
	}

	return
}

func sha256WithoutSalt(value string) [sha256.Size]byte {
	encrypted := sha256.Sum256([]byte(value))

	return encrypted
}

func sha256WithSalt(value, saltValue string) ([sha256.Size]byte, []byte) {
	salt := make([]byte, authSettings.SaltLength)

	if saltValue != "" {
		salt = []byte(saltValue)
	} else {
		n, err := rand.Read(salt)

		if err != nil {
			fmt.Println("There was an error generating a salt: ", err)
			return [sha256.Size]byte{}, nil
		}

		if n != authSettings.SaltLength {
			fmt.Printf("Only %d characters were read.\n", n)
			return [sha256.Size]byte{}, nil
		}
	}

	saltedVal := append([]byte(value), salt...)
	encrypted := sha256.Sum256(saltedVal)

	return encrypted, salt
}
<|endoftext|>


<commit_before>package agora

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/PuerkitoBio/agora/compiler"
	"github.com/PuerkitoBio/agora/runtime"
	"github.com/PuerkitoBio/agora/runtime/stdlib"
)

// This test runs all source files in ../testdata/src/*.agora and checks if
// the results are as expected.
//
// The header of each source code file can define a YAML front-matter block
// with the following fields:
// * output: the expected output (may contain \n for newlines)
// * result: the expected result value
// * long: if true, this test is skipped if the -short flag is set
// * args: the command-line arguments to pass to the test file
// * error: the expected error message (omit if no error is expected)

const (
	srcDir = "./testdata/src"
)

func TestSourceFiles(t *testing.T) {
	// Change working directory to where the source files are
	os.Chdir(srcDir)
	fis, err := ioutil.ReadDir(".")
	if err != nil {
		panic(err)
	}
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) == ".agora" {
			testFile(t, fi)
		}
	}
}

func testFile(t *testing.T, fi os.FileInfo) {
	f, e := os.Open(fi.Name())
	if e != nil {
		panic(e)
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	m := readFrontMatter(s)
	if len(m) == 0 {
		if testing.Verbose() {
			fmt.Printf("no front matter, skipping file %s...\n", fi.Name())
		}
		return
	}
	// Keep the rest of the file in a reader
	buf := bytes.NewBuffer(nil)
	for s.Scan() {
		buf.WriteString(s.Text())
		buf.WriteString("\n")
	}
	if s.Err() != nil {
		panic(s.Err())
	}
	// And actually run and test the file
	if _, ok := m["long"]; ok {
		if testing.Short() {
			if testing.Verbose() {
				fmt.Printf("skipping long test file %s...\n", fi.Name())
			}
			return
		}
	}
	if testing.Verbose() {
		fmt.Printf("testing file %s...\n", fi.Name())
	}
	runAndAssertFile(t, strings.TrimSuffix(fi.Name(), filepath.Ext(fi.Name())), bytes.NewReader(buf.Bytes()), m)
}

type testResolver struct {
	r  io.Reader
	mr runtime.ModuleResolver
}

func (t *testResolver) Resolve(id string) (io.Reader, error) {
	if t.r != nil {
		r := t.r
		t.r = nil
		return r, nil
	}
	return t.mr.Resolve(id)
}

func runAndAssertFile(t *testing.T, id string, r io.Reader, m map[string]string) {
	// Use the custom test resolver to return the reader
	buf := bytes.NewBuffer(nil)
	ctx := runtime.NewCtx(&testResolver{
		r,
		new(runtime.FileResolver),
	}, new(compiler.Compiler))
	ctx.Stdout = buf
	ctx.RegisterNativeModule(new(stdlib.FilepathMod))
	ctx.RegisterNativeModule(new(stdlib.FmtMod))
	ctx.RegisterNativeModule(new(stdlib.MathMod))
	ctx.RegisterNativeModule(new(stdlib.OsMod))
	ctx.RegisterNativeModule(new(stdlib.StringsMod))
	ctx.RegisterNativeModule(new(stdlib.TimeMod))

	mod, err := ctx.Load(id)
	var ret runtime.Val
	if err == nil {
		var args []runtime.Val
		if v, ok := m["args"]; ok {
			s := strings.Split(v, " ")
			args = make([]runtime.Val, len(s))
			for i, arg := range s {
				args[i] = runtime.String(arg)
			}
		}
		ret, err = mod.Run(args...)
	}

	assert := false
	if v, ok := m["error"]; ok {
		assert = true
		if err == nil {
			t.Errorf("[%s] - expected error '%s', got none", id, v)
		} else if err.Error() != v {
			t.Errorf("[%s] - expected error '%s', got '%s'", id, v, err)
		}
	} else if err != nil {
		t.Errorf("[%s] - expected no error, got '%s'", id, err)
	}
	if v, ok := m["result"]; ok {
		assert = true
		v = strings.Replace(v, "\\n", "\n", -1)
		v = strings.Replace(v, "\\t", "\t", -1)
		switch retv := ret.(type) {
		case runtime.Object, runtime.Func:
			str := fmt.Sprintf("%s", retv)
			if str != v {
				t.Errorf("[%s] - expected result '%s', got '%s'", id, v, str)
			}
		default:
			if retv.String() != v {
				t.Errorf("[%s] - expected result '%s', got '%s'", id, v, retv)
			}
		}
	}
	if v, ok := m["output"]; ok {
		assert = true
		v = strings.Replace(v, "\\n", "\n", -1)
		v = strings.Replace(v, "\\t", "\t", -1)
		if got := buf.String(); got != v {
			t.Errorf("[%s] - expected output '%s', got '%s'", id, v, got)
		}
	}
	if !assert {
		t.Errorf("[%s] - no assert", id)
	}
}

func readFrontMatter(s *bufio.Scanner) map[string]string {
	m := make(map[string]string)
	infm := false
	for s.Scan() {
		l := strings.Trim(s.Text(), " ")
		if l == "/*---" || l == "---*/" { // The front matter is delimited by 3 dashes and in a block comment
			if infm {
				// This signals the end of the front matter
				return m
			} else {
				// This is the start of the front matter
				infm = true
			}
		} else if infm {
			sections := strings.SplitN(l, ":", 2)
			if len(sections) != 2 {
				// Invalid front matter line
				return nil
			}
			m[sections[0]] = strings.Trim(sections[1], " ")
		} else if l != "" {
			// No front matter, quit
			return nil
		}
	}
	if err := s.Err(); err != nil {
		// The scanner stopped because of an error
		return nil
	}
	return nil
}
<commit_msg>improve object cmp for tests<commit_after>package agora

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/PuerkitoBio/agora/compiler"
	"github.com/PuerkitoBio/agora/runtime"
	"github.com/PuerkitoBio/agora/runtime/stdlib"
)

// This test runs all source files in ../testdata/src/*.agora and checks if
// the results are as expected.
//
// The header of each source code file can define a YAML front-matter block
// with the following fields:
// * output: the expected output (may contain \n for newlines)
// * result: the expected result value
// * long: if true, this test is skipped if the -short flag is set
// * args: the command-line arguments to pass to the test file
// * error: the expected error message (omit if no error is expected)

const (
	srcDir = "./testdata/src"
)

func TestSourceFiles(t *testing.T) {
	// Change working directory to where the source files are
	os.Chdir(srcDir)
	fis, err := ioutil.ReadDir(".")
	if err != nil {
		panic(err)
	}
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) == ".agora" {
			testFile(t, fi)
		}
	}
}

func testFile(t *testing.T, fi os.FileInfo) {
	f, e := os.Open(fi.Name())
	if e != nil {
		panic(e)
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	m := readFrontMatter(s)
	if len(m) == 0 {
		if testing.Verbose() {
			fmt.Printf("no front matter, skipping file %s...\n", fi.Name())
		}
		return
	}
	// Keep the rest of the file in a reader
	buf := bytes.NewBuffer(nil)
	for s.Scan() {
		buf.WriteString(s.Text())
		buf.WriteString("\n")
	}
	if s.Err() != nil {
		panic(s.Err())
	}
	// And actually run and test the file
	if _, ok := m["long"]; ok {
		if testing.Short() {
			if testing.Verbose() {
				fmt.Printf("skipping long test file %s...\n", fi.Name())
			}
			return
		}
	}
	if testing.Verbose() {
		fmt.Printf("testing file %s...\n", fi.Name())
	}
	runAndAssertFile(t, strings.TrimSuffix(fi.Name(), filepath.Ext(fi.Name())), bytes.NewReader(buf.Bytes()), m)
}

type testResolver struct {
	r  io.Reader
	mr runtime.ModuleResolver
}

func (t *testResolver) Resolve(id string) (io.Reader, error) {
	if t.r != nil {
		r := t.r
		t.r = nil
		return r, nil
	}
	return t.mr.Resolve(id)
}

func runAndAssertFile(t *testing.T, id string, r io.Reader, m map[string]string) {
	// Use the custom test resolver to return the reader
	buf := bytes.NewBuffer(nil)
	ctx := runtime.NewCtx(&testResolver{
		r,
		new(runtime.FileResolver),
	}, new(compiler.Compiler))
	ctx.Stdout = buf
	ctx.RegisterNativeModule(new(stdlib.FilepathMod))
	ctx.RegisterNativeModule(new(stdlib.FmtMod))
	ctx.RegisterNativeModule(new(stdlib.MathMod))
	ctx.RegisterNativeModule(new(stdlib.OsMod))
	ctx.RegisterNativeModule(new(stdlib.StringsMod))
	ctx.RegisterNativeModule(new(stdlib.TimeMod))

	mod, err := ctx.Load(id)
	var ret runtime.Val
	if err == nil {
		var args []runtime.Val
		if v, ok := m["args"]; ok {
			s := strings.Split(v, " ")
			args = make([]runtime.Val, len(s))
			for i, arg := range s {
				args[i] = runtime.String(arg)
			}
		}
		ret, err = mod.Run(args...)
	}

	assert := false
	if v, ok := m["error"]; ok {
		assert = true
		if err == nil {
			t.Errorf("[%s] - expected error '%s', got none", id, v)
		} else if err.Error() != v {
			t.Errorf("[%s] - expected error '%s', got '%s'", id, v, err)
		}
	} else if err != nil {
		t.Errorf("[%s] - expected no error, got '%s'", id, err)
	}
	if v, ok := m["result"]; ok {
		assert = true
		v = strings.Replace(v, "\\n", "\n", -1)
		v = strings.Replace(v, "\\t", "\t", -1)
		switch retv := ret.(type) {
		// comapre runtime.Object with special function
		case runtime.Object:
			if compareStringifiedObjects(retv.String(), v) {
				str := fmt.Sprintf("%s", retv)
				t.Errorf("[%s] - expected result '%s', got '%s'", id, v, str)
			}
		case runtime.Func:
			str := fmt.Sprintf("%s", retv)
			if str != v {
				t.Errorf("[%s] - expected result '%s', got '%s'", id, v, str)
			}
		default:
			if retv.String() != v {
				t.Errorf("[%s] - expected result '%s', got '%s'", id, v, retv)
			}
		}
	}
	if v, ok := m["output"]; ok {
		assert = true
		v = strings.Replace(v, "\\n", "\n", -1)
		v = strings.Replace(v, "\\t", "\t", -1)
		// compare output with special function
		if got := buf.String(); compareOutputsWithObjects(got, v) {
			t.Errorf("[%s] - expected output '%s', got '%s'", id, v, got)
		}
	}
	if !assert {
		t.Errorf("[%s] - no assert", id)
	}
}

/*
# Reason
--------------------------------------------------------------------------------
$ go test
--- FAIL: TestSourceFiles (38.34s)
        agora_test.go:161: [66-yield-complex] - expected output '1
                nil
                nil
                {f:test,e:7,d:4,c:3,b:2,a:1}    <--------------+
                ', got '1                                      |
                nil                                            |  just the same
                nil                                            |
                {c:3,b:2,a:1,f:test,e:7,d:4}    <--------------+
                '
--------------------------------------------------------------------------------
$ go test
--- FAIL: TestSourceFiles (42.53s)
        agora_test.go:148: [37-literal-obj] - expected result '{_:_-value!!,4:4-value!,s:s-value,i:i-value,4:int-4-value,5:int-5-value-via-i-var,string:int-s-value-via-s-var}', got '{4:4-value!,s:s-value,i:i-value,4:int-4-value,5:int-5-value-via-i-var,string:int-s-value-via-s-var,_:_-value!!}'
--------------------------------------------------------------------------------
"abc" and "abc"                                          //=> should be true
"abc, {a:1,b:2,c:3}" and "abc, {b:2,c:3,a:1}"            //=> should be true too
--------------------------------------------------------------------------------
Because runtime.Object is map[Val]Val, but map doesn't support ordering.
Playground implementation: http://play.golang.org/p/ENgKLm798i
*/
// res = result of script (runtime.Object.String())
// fmr  = front matter result
// return: false if equal, otherwise true
func compareStringifiedObjects(res string /*(runtime.Object).String()*/, fmr string) bool {
	type item struct {
		key, value string
	}
	parse_string_to_object := func(out string) ([]item, bool /*success*/) {
		// out: "{i:i-string,_:_-string,4:4-string}"
		out = strings.TrimPrefix(out, "{")
		// out: "i:i-string,_:_-string,4:4-string}"
		out = strings.TrimSuffix(out, "}")
		// out: "i:i-string,_:_-string,4:4-string"
		outs := strings.FieldsFunc(out, func(r rune) bool {
			return r == ','
		})
		// outs: [ "i:i-string", "_:_-string", "4:4-string" ]
		out_obj := make([]item, 0, len(outs))
		// fill out_obj
		for _, o := range outs {
			// o: "i:i-string"
			okv := strings.FieldsFunc(o, func(r rune) bool {
				return r == ':'
			})
			// okv: [ "i", "i-string" ]
			// check okv length
			if len(okv) != 2 {
				return nil, false
			}
			out_obj = append(out_obj, item{okv[0], okv[1]})
			// out_obj: [item{"i", "i-string"}, item{"4","4-string"}]
		}
		return out_obj, true
	}
	// front matter
	fm, ok := parse_string_to_object(fmr)
	if !ok {
		if testing.Verbose() {
			fmt.Println("unrecognized front matter object")
		}
		return true // not equal
	}
	// script result
	sr, ok := parse_string_to_object(res) // obj.String())
	if !ok {
		if testing.Verbose() {
			fmt.Println("unrecognized script object")
		}
		return true // not equal
	}
	// len(fm) == len(sr) ?
	if len(fm) != len(sr) {
		if testing.Verbose() {
			fmt.Printf(
				"improper length of the object, expected: %d, got: %d\n",
				len(fm),
				len(sr),
			)
		}
		return true // not equal
	}
	// compare objects
	for _, fmi := range fm {
		for i := 0; i < len(sr); i++ {
			if sr[i].key == fmi.key && sr[i].value == fmi.value {
				// delete item from sr slice
				sr = append(sr[:i], sr[i+1:]...)
				break
			}
		}
	}
	if len(sr) != 0 {
		return true // not equal
	}
	// all right
	return false // equal
}

// out = ouput of script
// fm  = front matter output
// return: false if equal, otherwise true
func compareOutputsWithObjects(out, fm string) bool {
	type slices struct {
		even bool
		body []string
	}
	// object regexp: {a:value,b:value-of-b!!!}
	rxp := regexp.MustCompile(`\{([^:}]+\:[^,}]+,)*([^:}]+\:[^,}]+)\}`)
	// func example: http://play.golang.org/p/fqzoSzaZqb
	cut := func(s string) (*slices, bool /*cuted*/) {
		ix := rxp.FindAllStringIndex(s, -1)
		if ix == nil {
			return nil, false
		}
		var ss []string
		var cur int
		for _, ixx := range ix {
			if cur != ixx[0] {
				ss = append(ss, s[cur:ixx[0]])
			}
			ss = append(ss, s[ixx[0]:ixx[1]])
			cur = ixx[1]
		}
		if cur != len(s) {
			ss = append(ss, s[cur:len(s)])
		}
		even := ix[0][0] != 0
		return &slices{
			even: even,
			body: ss,
		}, true
	}
	// cut out and fm
	outs, cuted := cut(out)
	if !cuted {
		// outs not cuted
		return out != fm // simple string cmp
	}
	fms, cuted := cut(fm)
	// check all
	if !cuted || outs.even != fms.even || len(outs.body) != len(fms.body) {
		return true // not equal
	}
	// loop
	for i := 0; i < len(outs.body); i++ {
		// if (ous.even and even) or (not outs.even and not even)
		if (outs.even && i%2 != 0) || (!outs.even && i%2 == 0) {
			if compareStringifiedObjects(outs.body[i], fms.body[i]) {
				return true // not equal
			}
		} else {
			if outs.body[i] != fms.body[i] {
				return true // not equal
			}
		}
	}
	// all right
	return false // equal
}

func readFrontMatter(s *bufio.Scanner) map[string]string {
	m := make(map[string]string)
	infm := false
	for s.Scan() {
		l := strings.Trim(s.Text(), " ")
		if l == "/*---" || l == "---*/" { // The front matter is delimited by 3 dashes and in a block comment
			if infm {
				// This signals the end of the front matter
				return m
			} else {
				// This is the start of the front matter
				infm = true
			}
		} else if infm {
			sections := strings.SplitN(l, ":", 2)
			if len(sections) != 2 {
				// Invalid front matter line
				return nil
			}
			m[sections[0]] = strings.Trim(sections[1], " ")
		} else if l != "" {
			// No front matter, quit
			return nil
		}
	}
	if err := s.Err(); err != nil {
		// The scanner stopped because of an error
		return nil
	}
	return nil
}
<|endoftext|>


<commit_before>package goraph

import (
	"fmt"
)

// TopoSort returns a slice of the vertices in g in topologically
// sorted order. (https://en.wikipedia.org/wiki/Topological_sorting)
//
// If deterministic is true, then the vertices are sorted by id before
// the algorithm begins, guaranteeing a repeatable result.
func TopoSort(g Graph, deterministic bool) (result []Vertex, err error) {
	if !g.IsDirected() {
		err = fmt.Errorf("graph is not a directed graph")
		return
	}
	verts := VertexSlice(g.Vertices())
	if deterministic {
		verts.Sort()
	}

	fmt.Println(g.Vertices())

	rlen := len(verts)
	result = make(VertexSlice, 0, rlen)
	marked := make(map[Vertex]bool, rlen) // visited vertices

	var visit func(Vertex)
	visit = func(vtx Vertex) {
		if permanent, ok := marked[vtx]; ok {
			if !permanent {
				err = fmt.Errorf("graph is not a directed graph")
			}
			return
		}
		// Mark vtx temporarily
		marked[vtx] = false
		for _, v := range g.Neighbours(vtx) {
			visit(v)
		}
		// mark vtx permanently
		marked[vtx] = true
		result = append(result, vtx)
	}

	for _, v := range verts {
		visit(v)
	}

	// The algorithm asks us to prepend to the result, but since we're using a
	// slice here, just reverse it after appending items.
	for i := 0; i < len(result)/2; i++ {
		result[i], result[rlen-i-1] = result[rlen-i-1], result[i]
	}

	return
}

// ShortestPath returns the shortest path between source and target using
// Dijkstra's algorithm.
func ShortestPath(g Graph, source, target Vertex) []Vertex {
	pq := make(priorityQueue, 0)

	// Vertex ancestry mapping
	prev := make(map[Vertex]Vertex)

	// Map of visited nodes to their distance from the source
	visited := make(map[Vertex]int)
	prev[source] = source

	s := &qItem{vertex: source, priority: 0}
	pq.Push(s)

	for pq.Len() > 0 {
		v := pq.Pop().(*qItem)
		if v.vertex == target {
			return walkAncestry(prev, v.vertex)
		}
		for _, w := range g.Neighbours(v.vertex) {
			newPrio := v.priority + 1
			if oldPrio, ok := visited[w]; ok {
				if newPrio > oldPrio {
					continue
				}
			}
			visited[w] = newPrio
			prev[w] = v.vertex
			q := &qItem{vertex: w, priority: v.priority + 1}
			pq.Push(q)
		}
	}

	return nil
}

func walkAncestry(ancestors map[Vertex]Vertex, v Vertex) []Vertex {
	result := []Vertex{v}
	if ancestors[v] == v {
		return result
	}
	result = append(walkAncestry(ancestors, ancestors[v]), result...)
	return result
}
<commit_msg>Remove debug println<commit_after>package goraph

import (
	"fmt"
)

// TopoSort returns a slice of the vertices in g in topologically
// sorted order. (https://en.wikipedia.org/wiki/Topological_sorting)
//
// If deterministic is true, then the vertices are sorted by id before
// the algorithm begins, guaranteeing a repeatable result.
func TopoSort(g Graph, deterministic bool) (result []Vertex, err error) {
	if !g.IsDirected() {
		err = fmt.Errorf("graph is not a directed graph")
		return
	}
	verts := VertexSlice(g.Vertices())
	if deterministic {
		verts.Sort()
	}

	rlen := len(verts)
	result = make(VertexSlice, 0, rlen)
	marked := make(map[Vertex]bool, rlen) // visited vertices

	var visit func(Vertex)
	visit = func(vtx Vertex) {
		if permanent, ok := marked[vtx]; ok {
			if !permanent {
				err = fmt.Errorf("graph is not a directed graph")
			}
			return
		}
		// Mark vtx temporarily
		marked[vtx] = false
		for _, v := range g.Neighbours(vtx) {
			visit(v)
		}
		// mark vtx permanently
		marked[vtx] = true
		result = append(result, vtx)
	}

	for _, v := range verts {
		visit(v)
	}

	// The algorithm asks us to prepend to the result, but since we're using a
	// slice here, just reverse it after appending items.
	for i := 0; i < len(result)/2; i++ {
		result[i], result[rlen-i-1] = result[rlen-i-1], result[i]
	}

	return
}

// ShortestPath returns the shortest path between source and target using
// Dijkstra's algorithm.
func ShortestPath(g Graph, source, target Vertex) []Vertex {
	pq := make(priorityQueue, 0)

	// Vertex ancestry mapping
	prev := make(map[Vertex]Vertex)

	// Map of visited nodes to their distance from the source
	visited := make(map[Vertex]int)
	prev[source] = source

	s := &qItem{vertex: source, priority: 0}
	pq.Push(s)

	for pq.Len() > 0 {
		v := pq.Pop().(*qItem)
		if v.vertex == target {
			return walkAncestry(prev, v.vertex)
		}
		for _, w := range g.Neighbours(v.vertex) {
			newPrio := v.priority + 1
			if oldPrio, ok := visited[w]; ok {
				if newPrio > oldPrio {
					continue
				}
			}
			visited[w] = newPrio
			prev[w] = v.vertex
			q := &qItem{vertex: w, priority: v.priority + 1}
			pq.Push(q)
		}
	}

	return nil
}

func walkAncestry(ancestors map[Vertex]Vertex, v Vertex) []Vertex {
	result := []Vertex{v}
	if ancestors[v] == v {
		return result
	}
	result = append(walkAncestry(ancestors, ancestors[v]), result...)
	return result
}
<|endoftext|>


<commit_before>// Go Problem 3 - Fizzbuzz
// Hugh Brady - G00338260 - 28/09/17

package main

import (
	"fmt"
)

func main() {
	var i int = 0

	// for loop running through numbers 1 - 100
	for i < 100{
		i++

		// check if i is divisible by 3 or 5
		if i%3 == 0 && i%5 == 0{
			fmt.Printf("FizzBuzz\n")
		} else {
			fmt.Printf("%d\n", i)
		}// end if else

	}

}<commit_msg>Problem 3 finished and commented<commit_after>// Go Problem 3 - Fizzbuzz
// Hugh Brady - G00338260 - 28/09/17

package main

import (
	"fmt"
)

func main() {
	var i int = 0

	// for loop running through numbers 1 - 100
	for i < 100{
		i++

		// check if i is divisible by 3 or 5
		if i%3 == 0 && i%5 == 0{
			fmt.Printf("FizzBuzz\n")
		} else if i%3 == 0 {
			fmt.Printf("Fizz\n")
		} else if i%5 == 0{
			fmt.Printf("Buzz\n")
		} else {
			fmt.Printf("%d\n", i)
		}// end if else
	}// end for
}// main<|endoftext|>