<commit_before>// Package ql exposes implementations and functions that henables ngorm to work
// with ql database.
//
// ql is an embedded sql database. This database doesn't conform 100% qith the
// SQL standard. The link to the project is https://github.com/cznic/ql
package ql

import (
	"fmt"
	"math/big"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/gernest/ngorm/model"
)

//QL implements the dialects.Dialect interface that uses ql database as the SQl
//backend.
//
// For some reason the ql database doesn't support multiple databases, as
// databases are file based. So, the name of the file is the name of the
// database.. Which doesn't affect the querries, since the database name is
// irrelevant assuming the SQLCommon interface is the handle over the open
// database.
type QL struct {
	name string
	db   model.SQLCommon
}

// Memory returns the dialect for in memory ql database. This is not persistent
// everything will be lost when the process exits.
func Memory() *QL {
	return &QL{name: "ql-mem"}
}

//File returns the dialcet for file backed ql database. This is the recommended
//way use the Memory only for testing else you might lose all of your data.
func File() *QL {
	return &QL{name: "ql"}
}

// GetName get dialect's name
func (q *QL) GetName() string {
	return q.name
}

// SetDB set db for dialect
func (q *QL) SetDB(db model.SQLCommon) {
	q.db = db
}

// BindVar return the placeholder for actual values in SQL statements, in many dbs it is "?", Postgres using $1
func (q QL) BindVar(i int) string {
	return fmt.Sprintf("$%d", i)
}

// Quote quotes field name to avoid SQL parsing exceptions by using a reserved word as a field name
func (q *QL) Quote(key string) string {
	return fmt.Sprintf(`"%s"`, key)
}

// DataTypeOf return data's sql type
func (q *QL) DataTypeOf(field *model.StructField) (string, error) {
	var dataValue, sqlType, _, additionalType = model.ParseFieldStructForDialect(field)

	if sqlType == "" {
		switch dataValue.Kind() {
		case reflect.Bool:
			sqlType = "boolean"
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64,
			reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64,
			reflect.Float32,
			reflect.Float64,
			reflect.String:
			sqlType = dataValue.Kind().String()
		case reflect.Struct:
			switch dataValue.Interface().(type) {
			case time.Time:
				sqlType = "time"
			case big.Int:
				sqlType = "bigint"
			case big.Rat:
				sqlType = "bigrat"
			}
		default:
			if _, ok := dataValue.Interface().([]byte); ok {
				sqlType = "blob"
			}
		}
	}

	if sqlType == "" {
		return "", fmt.Errorf("invalid sql type %s (%s) for ql", dataValue.Type().Name(), dataValue.Kind().String())
	}

	if strings.TrimSpace(additionalType) == "" {
		return sqlType, nil
	}
	return fmt.Sprintf("%v %v", sqlType, additionalType), nil
}

// HasIndex check has index or not
func (q *QL) HasIndex(tableName string, indexName string) bool {
	querry := "select count() from __Index where Name=$1  && TableName=$2"
	var count int
	err := q.db.QueryRow(querry, indexName, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// HasForeignKey check has foreign key or not
func (q *QL) HasForeignKey(tableName string, foreignKeyName string) bool {
	return false
}

// RemoveIndex remove index
func (q *QL) RemoveIndex(tableName string, indexName string) error {
	tx, err := q.db.Begin()
	if err != nil {
		return err
	}
	_, err = tx.Exec(fmt.Sprintf("DROP INDEX %v", indexName))
	if err != nil {
		return err
	}
	return tx.Commit()
}

// HasTable check has table or not
func (q *QL) HasTable(tableName string) bool {
	querry := "select count() from __Table where Name=$1"
	var count int
	err := q.db.QueryRow(querry, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// HasColumn check has column or not
func (q *QL) HasColumn(tableName string, columnName string) bool {
	querry := "select count() from __Column where Name=$1  && TableName=$2"
	var count int
	err := q.db.QueryRow(querry, columnName, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// LimitAndOffsetSQL return generated SQL with Limit and Offset, as mssql has special case
func (q *QL) LimitAndOffsetSQL(limit, offset interface{}) (sql string) {
	if limit != nil {
		if parsedLimit, err := strconv.ParseInt(fmt.Sprint(limit), 0, 0); err == nil && parsedLimit > 0 {
			sql += fmt.Sprintf(" LIMIT %d", parsedLimit)
		}
	}
	if offset != nil {
		if parsedOffset, err := strconv.ParseInt(fmt.Sprint(offset), 0, 0); err == nil && parsedOffset > 0 {
			sql += fmt.Sprintf(" OFFSET %d", parsedOffset)
		}
	}
	return
}

// SelectFromDummyTable return select values, for most dbs, `SELECT values` just works, mysql needs `SELECT value FROM DUAL`
func (q *QL) SelectFromDummyTable() string {
	return ""
}

// LastInsertIDReturningSuffix ost dbs support LastInsertId, but postgres needs to use `RETURNING`
func (q *QL) LastInsertIDReturningSuffix(tableName, columnName string) string {
	return ""
}

// BuildForeignKeyName returns a foreign key name for the given table, field and reference
func (q *QL) BuildForeignKeyName(tableName, field, dest string) string {
	return ""
}

// CurrentDatabase return current database name
func (q *QL) CurrentDatabase() string {
	return ""
}
<commit_msg>[dialects] Implement *QL.BuildForeignKeyName<commit_after>// Package ql exposes implementations and functions that henables ngorm to work
// with ql database.
//
// ql is an embedded sql database. This database doesn't conform 100% qith the
// SQL standard. The link to the project is https://github.com/cznic/ql
package ql

import (
	"fmt"
	"math/big"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/regexes"
)

//QL implements the dialects.Dialect interface that uses ql database as the SQl
//backend.
//
// For some reason the ql database doesn't support multiple databases, as
// databases are file based. So, the name of the file is the name of the
// database.. Which doesn't affect the querries, since the database name is
// irrelevant assuming the SQLCommon interface is the handle over the open
// database.
type QL struct {
	name string
	db   model.SQLCommon
}

// Memory returns the dialect for in memory ql database. This is not persistent
// everything will be lost when the process exits.
func Memory() *QL {
	return &QL{name: "ql-mem"}
}

//File returns the dialcet for file backed ql database. This is the recommended
//way use the Memory only for testing else you might lose all of your data.
func File() *QL {
	return &QL{name: "ql"}
}

// GetName get dialect's name
func (q *QL) GetName() string {
	return q.name
}

// SetDB set db for dialect
func (q *QL) SetDB(db model.SQLCommon) {
	q.db = db
}

// BindVar return the placeholder for actual values in SQL statements, in many dbs it is "?", Postgres using $1
func (q QL) BindVar(i int) string {
	return fmt.Sprintf("$%d", i)
}

// Quote quotes field name to avoid SQL parsing exceptions by using a reserved word as a field name
func (q *QL) Quote(key string) string {
	return fmt.Sprintf(`"%s"`, key)
}

// DataTypeOf return data's sql type
func (q *QL) DataTypeOf(field *model.StructField) (string, error) {
	var dataValue, sqlType, _, additionalType = model.ParseFieldStructForDialect(field)

	if sqlType == "" {
		switch dataValue.Kind() {
		case reflect.Bool:
			sqlType = "boolean"
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64,
			reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64,
			reflect.Float32,
			reflect.Float64,
			reflect.String:
			sqlType = dataValue.Kind().String()
		case reflect.Struct:
			switch dataValue.Interface().(type) {
			case time.Time:
				sqlType = "time"
			case big.Int:
				sqlType = "bigint"
			case big.Rat:
				sqlType = "bigrat"
			}
		default:
			if _, ok := dataValue.Interface().([]byte); ok {
				sqlType = "blob"
			}
		}
	}

	if sqlType == "" {
		return "", fmt.Errorf("invalid sql type %s (%s) for ql", dataValue.Type().Name(), dataValue.Kind().String())
	}

	if strings.TrimSpace(additionalType) == "" {
		return sqlType, nil
	}
	return fmt.Sprintf("%v %v", sqlType, additionalType), nil
}

// HasIndex check has index or not
func (q *QL) HasIndex(tableName string, indexName string) bool {
	querry := "select count() from __Index where Name=$1  && TableName=$2"
	var count int
	err := q.db.QueryRow(querry, indexName, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// HasForeignKey check has foreign key or not
func (q *QL) HasForeignKey(tableName string, foreignKeyName string) bool {
	return false
}

// RemoveIndex remove index
func (q *QL) RemoveIndex(tableName string, indexName string) error {
	tx, err := q.db.Begin()
	if err != nil {
		return err
	}
	_, err = tx.Exec(fmt.Sprintf("DROP INDEX %v", indexName))
	if err != nil {
		return err
	}
	return tx.Commit()
}

// HasTable check has table or not
func (q *QL) HasTable(tableName string) bool {
	querry := "select count() from __Table where Name=$1"
	var count int
	err := q.db.QueryRow(querry, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// HasColumn check has column or not
func (q *QL) HasColumn(tableName string, columnName string) bool {
	querry := "select count() from __Column where Name=$1  && TableName=$2"
	var count int
	err := q.db.QueryRow(querry, columnName, tableName).Scan(&count)
	if err != nil {
		//TODO; Propery log or return this error?
	}
	return count > 0
}

// LimitAndOffsetSQL return generated SQL with Limit and Offset, as mssql has special case
func (q *QL) LimitAndOffsetSQL(limit, offset interface{}) (sql string) {
	if limit != nil {
		if parsedLimit, err := strconv.ParseInt(fmt.Sprint(limit), 0, 0); err == nil && parsedLimit > 0 {
			sql += fmt.Sprintf(" LIMIT %d", parsedLimit)
		}
	}
	if offset != nil {
		if parsedOffset, err := strconv.ParseInt(fmt.Sprint(offset), 0, 0); err == nil && parsedOffset > 0 {
			sql += fmt.Sprintf(" OFFSET %d", parsedOffset)
		}
	}
	return
}

// SelectFromDummyTable return select values, for most dbs, `SELECT values` just works, mysql needs `SELECT value FROM DUAL`
func (q *QL) SelectFromDummyTable() string {
	return ""
}

// LastInsertIDReturningSuffix ost dbs support LastInsertId, but postgres needs to use `RETURNING`
func (q *QL) LastInsertIDReturningSuffix(tableName, columnName string) string {
	return ""
}

// BuildForeignKeyName returns a foreign key name for the given table, field and reference
func (q *QL) BuildForeignKeyName(tableName, field, dest string) string {
	keyName := fmt.Sprintf("%s_%s_%s_foreign", tableName, field, dest)
	keyName = regexes.KeyName.ReplaceAllString(keyName, "_")
	return keyName
}

// CurrentDatabase return current database name
func (q *QL) CurrentDatabase() string {
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	_ "image/jpeg"
	_ "image/png"
	"log"
	"os"
	"syscall"

	"github.com/nfnt/resize"
	"golang.org/x/crypto/ssh/terminal"

	tc "github.com/thijzert/go-termcolours"
)

var (
	text_aspect  = flag.Float64("text_aspect", 0.944444, "Aspect ratio for your terminal font")
	use_24bit    = flag.Bool("use_24bit", false, "Use 24-bit colours")
	force_width  = flag.Int("width", 0, "Force output width")
	force_height = flag.Int("height", 0, "Force output height")
)

const BLOCK = "\xe2\x96\x80"

func init() {
	flag.Parse()
}

func main() {
	var err error
	termx, termy := *force_width, *force_height
	if termx == 0 && termy == 0 {
		termx, termy, err = terminal.GetSize(syscall.Stdout)
		if err != nil {
			log.Print(err)
			termx, termy = 80, 25
		}
	} else {
		if termx == 0 {
			termx = termy * 1000
		} else if termy == 0 {
			termy = termx * 1000
		}
	}

	// We can stack two pixels in one character
	termy *= 2

	for _, image_file := range flag.Args() {
		reader, err := os.Open(image_file)
		if err != nil {
			log.Fatal(err)
		}
		defer reader.Close()

		m, _, err := image.Decode(reader)
		if err != nil {
			log.Fatal(err)
		}

		bounds := m.Bounds()
		fmt.Printf("Image is %s by %s pixels wide\n", tc.Green(fmt.Sprintf("%d", bounds.Max.X)), tc.Green(fmt.Sprintf("%d", bounds.Max.Y)))

		nx, ny := boundbox(bounds.Max.X, bounds.Max.Y, termx, termy)
		mm := resize.Resize(uint(nx), uint(ny), convertRGBA(m), resize.Lanczos3)

		bounds = mm.Bounds()
		fmt.Printf("Image is now %s by %s pixels wide\n", tc.Green(fmt.Sprintf("%d", bounds.Max.X)), tc.Green(fmt.Sprintf("%d", bounds.Max.Y)))

		if *use_24bit {
			Write24(mm)
		} else {
			Write8(mm)
		}
	}
}

func convertRGBA(in image.Image) image.Image {
	if m, ok := in.(*image.RGBA); ok {
		return m
	}

	bounds := in.Bounds()

	m := image.NewRGBA(bounds)
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			m.Set(x, y, in.At(x, y))
		}
	}
	return m
}

func boundbox(imgx, imgy, bx, by int) (x, y int) {
	if imgx < 1 || imgy < 1 || bx < 1 || by < 1 {
		return 1, 1
	}

	term_aspect := float64(by) / float64(bx)
	aspect := *text_aspect * (float64(imgy) / float64(imgx))

	if aspect >= term_aspect {
		y = by
		x = int((float64(by) / aspect) + 0.5)
		if x > bx {
			x = bx
		}
	} else {
		x = bx
		// We can stack two pixels in one character
		y = 2 * int((float64(bx)*aspect*0.5)+0.5)
		if y > by {
			y = by
		}
	}
	return
}

func Write24(i image.Image) {
	bounds := i.Bounds()
	var c0, c1 color.Color
	for y := bounds.Min.Y; y < bounds.Max.Y; y += 2 {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			c0 = i.At(x, y)
			c1 = color.Black
			if (y + 1) < bounds.Max.Y {
				c1 = i.At(x, y+1)
			}

			fmt.Print(tc.Background24(c1, tc.Foreground24(c0, BLOCK)))
		}
		fmt.Print("\n")
	}
}

func cdiff(before color.Color, after tc.C256) (r, g, b int32) {
	r0, g0, b0, _ := before.RGBA()
	r1, g1, b1, _ := after.RGBA()

	r = int32(r1) - int32(r0)
	g = int32(g1) - int32(g0)
	b = int32(b1) - int32(b0)
	return
}

func pos(a, b int32) uint32 {
	a += b
	if a < 0 {
		return 0
	}
	return uint32(a)
}

func iadd(i image.Image, x, y int, dR, dG, dB int32, multiplier float64) {
	col := i.At(x, y)

	r, g, b, _ := col.RGBA()
	r = pos(int32(r), int32(float64(dR)*multiplier))
	g = pos(int32(g), int32(float64(dG)*multiplier))
	b = pos(int32(b), int32(float64(dB)*multiplier))

	cnew := color.RGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), 0}
	i.(*image.RGBA).Set(x, y, cnew)
}

func iget(i image.Image, bounds image.Rectangle, x, y int) tc.C256 {
	col := i.At(x, y)
	aft := tc.Convert256(col)

	dr, dg, db := cdiff(col, aft)

	if (x + 1) < bounds.Max.X {
		iadd(i, x+1, y, dr, db, dg, 7.0/16.0)
		if (y + 1) < bounds.Max.Y {
			iadd(i, x+1, y+1, dr, db, dg, 1.0/16.0)
		}
	}
	if (y + 1) < bounds.Max.Y {
		iadd(i, x, y+1, dr, db, dg, 5.0/16.0)
		if (x - 1) >= bounds.Min.X {
			iadd(i, x-1, y+1, dr, db, dg, 3.0/16.0)
		}
	}

	return aft
}

func Write8(img image.Image) {
	bounds := img.Bounds()
	prevline := make([]tc.C256, bounds.Max.X-bounds.Min.X)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		i := y - bounds.Min.Y

		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			j := x - bounds.Min.X

			c0 := iget(img, bounds, x, y)
			if i%2 == 0 {
				prevline[j] = c0
			} else {
				fmt.Print(tc.Background8(c0, tc.Foreground8(prevline[j], BLOCK)))
			}
		}
		if i%2 == 1 {
			fmt.Print("\n")
		}
	}

	if (bounds.Max.Y-bounds.Min.Y)%2 == 1 {
		for _, c0 := range prevline {
			fmt.Print(tc.Foreground8(c0, BLOCK))
		}
		fmt.Print("\n")
	}
}
<commit_msg>Fix math error - render images in correct aspect ratio<commit_after>package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	_ "image/jpeg"
	_ "image/png"
	"log"
	"os"
	"syscall"

	"github.com/nfnt/resize"
	"golang.org/x/crypto/ssh/terminal"

	tc "github.com/thijzert/go-termcolours"
)

var (
	text_aspect  = flag.Float64("text_aspect", 0.944444, "Aspect ratio for your terminal font")
	use_24bit    = flag.Bool("use_24bit", false, "Use 24-bit colours")
	force_width  = flag.Int("width", 0, "Force output width")
	force_height = flag.Int("height", 0, "Force output height")
)

const BLOCK = "\xe2\x96\x80"

func init() {
	flag.Parse()
}

func main() {
	var err error
	termx, termy := *force_width, *force_height
	if termx == 0 && termy == 0 {
		termx, termy, err = terminal.GetSize(syscall.Stdout)
		if err != nil {
			log.Print(err)
			termx, termy = 80, 25
		}
	} else {
		if termx == 0 {
			termx = termy * 1000
		} else if termy == 0 {
			termy = termx * 1000
		}
	}

	// We can stack two pixels in one character
	termy *= 2

	for _, image_file := range flag.Args() {
		reader, err := os.Open(image_file)
		if err != nil {
			log.Fatal(err)
		}
		defer reader.Close()

		m, _, err := image.Decode(reader)
		if err != nil {
			log.Fatal(err)
		}

		bounds := m.Bounds()
		fmt.Printf("Image is %s by %s pixels wide\n", tc.Green(fmt.Sprintf("%d", bounds.Max.X)), tc.Green(fmt.Sprintf("%d", bounds.Max.Y)))

		nx, ny := boundbox(bounds.Max.X, bounds.Max.Y, termx, termy)
		mm := resize.Resize(uint(nx), uint(ny), convertRGBA(m), resize.Lanczos3)

		bounds = mm.Bounds()
		fmt.Printf("Image is now %s by %s pixels wide\n", tc.Green(fmt.Sprintf("%d", bounds.Max.X)), tc.Green(fmt.Sprintf("%d", bounds.Max.Y)))

		if *use_24bit {
			Write24(mm)
		} else {
			Write8(mm)
		}
	}
}

func convertRGBA(in image.Image) image.Image {
	if m, ok := in.(*image.RGBA); ok {
		return m
	}

	bounds := in.Bounds()

	m := image.NewRGBA(bounds)
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			m.Set(x, y, in.At(x, y))
		}
	}
	return m
}

func boundbox(imgx, imgy, bx, by int) (x, y int) {
	if imgx < 1 || imgy < 1 || bx < 1 || by < 1 {
		return 1, 1
	}

	term_aspect := float64(by) / float64(bx)
	aspect := (float64(imgy) / float64(imgx)) / *text_aspect

	if aspect >= term_aspect {
		y = by
		x = int((float64(by) / aspect) + 0.5)
		if x > bx {
			x = bx
		}
	} else {
		x = bx
		// We can stack two pixels in one character
		y = 2 * int((float64(bx)*aspect*0.5)+0.5)
		if y > by {
			y = by
		}
	}
	return
}

func Write24(i image.Image) {
	bounds := i.Bounds()
	var c0, c1 color.Color
	for y := bounds.Min.Y; y < bounds.Max.Y; y += 2 {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			c0 = i.At(x, y)
			c1 = color.Black
			if (y + 1) < bounds.Max.Y {
				c1 = i.At(x, y+1)
			}

			fmt.Print(tc.Background24(c1, tc.Foreground24(c0, BLOCK)))
		}
		fmt.Print("\n")
	}
}

func cdiff(before color.Color, after tc.C256) (r, g, b int32) {
	r0, g0, b0, _ := before.RGBA()
	r1, g1, b1, _ := after.RGBA()

	r = int32(r1) - int32(r0)
	g = int32(g1) - int32(g0)
	b = int32(b1) - int32(b0)
	return
}

func pos(a, b int32) uint32 {
	a += b
	if a < 0 {
		return 0
	}
	return uint32(a)
}

func iadd(i image.Image, x, y int, dR, dG, dB int32, multiplier float64) {
	col := i.At(x, y)

	r, g, b, _ := col.RGBA()
	r = pos(int32(r), int32(float64(dR)*multiplier))
	g = pos(int32(g), int32(float64(dG)*multiplier))
	b = pos(int32(b), int32(float64(dB)*multiplier))

	cnew := color.RGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), 0}
	i.(*image.RGBA).Set(x, y, cnew)
}

func iget(i image.Image, bounds image.Rectangle, x, y int) tc.C256 {
	col := i.At(x, y)
	aft := tc.Convert256(col)

	dr, dg, db := cdiff(col, aft)

	if (x + 1) < bounds.Max.X {
		iadd(i, x+1, y, dr, db, dg, 7.0/16.0)
		if (y + 1) < bounds.Max.Y {
			iadd(i, x+1, y+1, dr, db, dg, 1.0/16.0)
		}
	}
	if (y + 1) < bounds.Max.Y {
		iadd(i, x, y+1, dr, db, dg, 5.0/16.0)
		if (x - 1) >= bounds.Min.X {
			iadd(i, x-1, y+1, dr, db, dg, 3.0/16.0)
		}
	}

	return aft
}

func Write8(img image.Image) {
	bounds := img.Bounds()
	prevline := make([]tc.C256, bounds.Max.X-bounds.Min.X)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		i := y - bounds.Min.Y

		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			j := x - bounds.Min.X

			c0 := iget(img, bounds, x, y)
			if i%2 == 0 {
				prevline[j] = c0
			} else {
				fmt.Print(tc.Background8(c0, tc.Foreground8(prevline[j], BLOCK)))
			}
		}
		if i%2 == 1 {
			fmt.Print("\n")
		}
	}

	if (bounds.Max.Y-bounds.Min.Y)%2 == 1 {
		for _, c0 := range prevline {
			fmt.Print(tc.Foreground8(c0, BLOCK))
		}
		fmt.Print("\n")
	}
}
<|endoftext|>


<commit_before>package chat

import (
	"context"
	"sync"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type IdentifyNotifier struct {
	libkb.Contextified
	sync.RWMutex
	identCache map[string]keybase1.CanonicalTLFNameAndIDWithBreaks
}

func NewIdentifyNotifier(g *libkb.GlobalContext) *IdentifyNotifier {
	return &IdentifyNotifier{
		Contextified: libkb.NewContextified(g),
		identCache:   make(map[string]keybase1.CanonicalTLFNameAndIDWithBreaks),
	}
}

func (i *IdentifyNotifier) Send(update keybase1.CanonicalTLFNameAndIDWithBreaks) {
	i.RLock()
	tlfName := update.CanonicalName.String()
	if stored, ok := i.identCache[tlfName]; ok {
		// We have the exact update stored, don't send it again
		if stored.Eq(update) {
			defer i.RUnlock()
			i.G().Log.Debug("IdentifyNotifier: hit cache, not sending notify: %s", tlfName)
			return
		}
	}
	i.RUnlock()

	i.Lock()
	defer i.Unlock()

	i.G().Log.Debug("IdentifyNotifier: cache miss, sending notify: %s dat: %v", tlfName, update)
	i.G().NotifyRouter.HandleChatIdentifyUpdate(context.Background(), update)
	i.identCache[tlfName] = update
}
<commit_msg>simplify lock code<commit_after>package chat

import (
	"context"
	"sync"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type IdentifyNotifier struct {
	libkb.Contextified
	sync.RWMutex
	identCache map[string]keybase1.CanonicalTLFNameAndIDWithBreaks
}

func NewIdentifyNotifier(g *libkb.GlobalContext) *IdentifyNotifier {
	return &IdentifyNotifier{
		Contextified: libkb.NewContextified(g),
		identCache:   make(map[string]keybase1.CanonicalTLFNameAndIDWithBreaks),
	}
}

func (i *IdentifyNotifier) Send(update keybase1.CanonicalTLFNameAndIDWithBreaks) {
	i.RLock()
	tlfName := update.CanonicalName.String()
	stored, ok := i.identCache[tlfName]
	i.RUnlock()
	if ok {
		// We have the exact update stored, don't send it again
		if stored.Eq(update) {
			i.G().Log.Debug("IdentifyNotifier: hit cache, not sending notify: %s", tlfName)
			return
		}
	}

	i.Lock()
	i.identCache[tlfName] = update
	i.Unlock()

	i.G().Log.Debug("IdentifyNotifier: cache miss, sending notify: %s dat: %v", tlfName, update)
	i.G().NotifyRouter.HandleChatIdentifyUpdate(context.Background(), update)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os/user"
	"strconv"
	"strings"
)

var uidMap = map[string]user.User{}

//This does the heavy lifting of making auditd messages into json.
//It is NOT thread (goroutine) safe, as it uses two shared global maps
//It can be made threadsafe by locking these if performance becomes an issue.
func makeJsonString(evBuf map[int]map[string]string, dtype uint16, dstring string, c chan<- string) {
	data := strings.Fields(dstring)
	_, seq := parseAuditHeader(data[0])
	if _, ok := evBuf[seq]; ok == false {
		evBuf[seq] = make(map[string]string)
	}
	splitz := []string{}

	//Add in some additional data
	evBuf[seq]["netlink_type"] = fmt.Sprintf("%d", dtype)
	evBuf[seq]["auditd_seq"] = fmt.Sprintf("%d", seq)

	//Divide into key value pairs for conversion to json
	for i := 0; i < len(data); i++ {
		a := data
		splitz = strings.SplitN(a[i], "=", 2)
		switch len(splitz) {
		//This means we found a key/value pair
		case 2:
			//leaving this here to test overwritten values
			//if evBuf[seq][splitz[0]] != "" {
			//	fmt.Println("overwrite! ", splitz[0])
			//}
			evBuf[seq][splitz[0]] = splitz[1]
			switch splitz[0] {
			//Get arg count and generate complete "command" element.
			case "argc":
				defer parseArgs(evBuf[seq])
			//Add contextual info per host if we can look up the uid
			case "uid":
				parseUid(evBuf[seq], "uid")
			case "auid":
				parseUid(evBuf[seq], "auid")
			}
		//Empty case here because this is what the header looks like and we don't want to error
		case 1:
		//This shouldn't happen
		default:
			log.Fatal("unexpected split: ", splitz)
		}
	}
	switch {
	//If the message is anything from 13xx(audit) but not 1320, lets combine them so don't output yet
	case (dtype >= 1300 && dtype <= 1319) || (dtype >= 1300 && dtype <= 1319):
		//fmt.Println(evBuf[seq])
	default:
		c <- mapToJsonString(evBuf[seq])
		delete(evBuf, seq)
		//End of the event - send to channel to process
		//This is a brutal hack and we need to detect the EOE properly
	}

}

//Parses the auditd headers. Example: 'audit(1446832550.594:2767175):'
//Where the first number in parens is the timestamp
//and the second is the sequence number of the event
func parseAuditHeader(data string) (time string, seq int) {
	if len(data) > 6 && data[:6] == "audit(" {
		timeAndSeq := strings.Split(strings.Trim(data, "audit():"), ":")
		seq, _ := strconv.Atoi(timeAndSeq[1])
		return timeAndSeq[0], seq
	} else {
		return
	}
}

//Iterates over argc(int) and combines individual a[num] into a single "command" key
func parseArgs(m map[string]string) {
	argc, _ := strconv.Atoi(m["argc"])
	if _, ok := m["argc"]; ok {
		command := []byte{}
		for i := 0; i < argc; i++ {
			element := fmt.Sprintf("a%d", i)
			//If surrounded by quotes, it is a literal. Just remove the double quotes.
			//If not surrounded by quotes, it is hex encoded. Decode the text.
			if len(m[element]) > 0 {
				if m[element][0] == '"' {
					m[element] = strings.Trim(m[element], "\"")
					command = append(command, ' ')
					command = append(command, m[element]...)

				} else {
					n, _ := hex.DecodeString(m[element])
					command = append(command, ' ')
					command = append(command, n...)
					m[element] = string(n)
				}
			} else {
				log.Output(1, string("Failed to parse args in "+fmt.Sprintf("%s", m)))
			}

		}
		//Store assembled command text into "command" key in passed map
		//Drop the first character, which is always a space
		m["command"] = string(command)[1:]
	}
}

//Simple wrapper to make the map into json
func mapToJsonString(m map[string]string) string {
	s, err := json.Marshal(m)
	if err != nil {
		log.Fatal(err)
	}
	out := fmt.Sprintf("%s", s)
	return out
}

//This takes the map to modify and a key name and adds the username to a new key with "_username"
func parseUid(m map[string]string, uidKey string) {
	//Make sure we have a uid element to work with.
	if uid, ok := m[uidKey]; ok {
		//Give a default value in case we don't find something.
		m[uidKey+"_username"] = "UNKNOWN_USER"
		if luser, ok := uidMap[uid]; ok {
			m[uidKey+"_username"] = luser.Username
		} else {
			luser, err := user.LookupId(uid)
			if err == nil {
				m[uidKey+"_username"] = luser.Username
				uidMap[uid] = *luser
				//Probably redundant. FIX
			} else {
				uidMap[uid] = user.User{Username: "UNKNOWN_USER"}
			}
		}
	}
}
<commit_msg>use function that works with golang < 1.5<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os/user"
	"strconv"
	"strings"
)

var uidMap = map[string]user.User{}

//This does the heavy lifting of making auditd messages into json.
//It is NOT thread (goroutine) safe, as it uses two shared global maps
//It can be made threadsafe by locking these if performance becomes an issue.
func makeJsonString(evBuf map[int]map[string]string, dtype uint16, dstring string, c chan<- string) {
	data := strings.Fields(dstring)
	_, seq := parseAuditHeader(data[0])
	if _, ok := evBuf[seq]; ok == false {
		evBuf[seq] = make(map[string]string)
	}
	splitz := []string{}

	//Add in some additional data
	evBuf[seq]["netlink_type"] = fmt.Sprintf("%d", dtype)
	evBuf[seq]["auditd_seq"] = fmt.Sprintf("%d", seq)

	//Divide into key value pairs for conversion to json
	for i := 0; i < len(data); i++ {
		a := data
		splitz = strings.SplitN(a[i], "=", 2)
		switch len(splitz) {
		//This means we found a key/value pair
		case 2:
			//leaving this here to test overwritten values
			//if evBuf[seq][splitz[0]] != "" {
			//	fmt.Println("overwrite! ", splitz[0])
			//}
			evBuf[seq][splitz[0]] = splitz[1]
			switch splitz[0] {
			//Get arg count and generate complete "command" element.
			case "argc":
				defer parseArgs(evBuf[seq])
			//Add contextual info per host if we can look up the uid
			case "uid":
				parseUid(evBuf[seq], "uid")
			case "auid":
				parseUid(evBuf[seq], "auid")
			}
		//Empty case here because this is what the header looks like and we don't want to error
		case 1:
		//This shouldn't happen
		default:
			log.Fatal("unexpected split: ", splitz)
		}
	}
	switch {
	//If the message is anything from 13xx(audit) but not 1320, lets combine them so don't output yet
	case (dtype >= 1300 && dtype <= 1319) || (dtype >= 1300 && dtype <= 1319):
		//fmt.Println(evBuf[seq])
	default:
		c <- mapToJsonString(evBuf[seq])
		delete(evBuf, seq)
		//End of the event - send to channel to process
		//This is a brutal hack and we need to detect the EOE properly
	}

}

//Parses the auditd headers. Example: 'audit(1446832550.594:2767175):'
//Where the first number in parens is the timestamp
//and the second is the sequence number of the event
func parseAuditHeader(data string) (time string, seq int) {
	if len(data) > 6 && data[:6] == "audit(" {
		timeAndSeq := strings.Split(strings.Trim(data, "audit():"), ":")
		seq, _ := strconv.Atoi(timeAndSeq[1])
		return timeAndSeq[0], seq
	} else {
		return
	}
}

//Iterates over argc(int) and combines individual a[num] into a single "command" key
func parseArgs(m map[string]string) {
	argc, _ := strconv.Atoi(m["argc"])
	if _, ok := m["argc"]; ok {
		command := []byte{}
		for i := 0; i < argc; i++ {
			element := fmt.Sprintf("a%d", i)
			//If surrounded by quotes, it is a literal. Just remove the double quotes.
			//If not surrounded by quotes, it is hex encoded. Decode the text.
			if len(m[element]) > 0 {
				if m[element][0] == '"' {
					m[element] = strings.Trim(m[element], "\"")
					command = append(command, ' ')
					command = append(command, m[element]...)

				} else {
					n, _ := hex.DecodeString(m[element])
					command = append(command, ' ')
					command = append(command, n...)
					m[element] = string(n)
				}
			} else {
				log.Printf("Failed to parse args in " + fmt.Sprintf("%s", m))
			}

		}
		//Store assembled command text into "command" key in passed map
		//Drop the first character, which is always a space
		m["command"] = string(command)[1:]
	}
}

//Simple wrapper to make the map into json
func mapToJsonString(m map[string]string) string {
	s, err := json.Marshal(m)
	if err != nil {
		log.Fatal(err)
	}
	out := fmt.Sprintf("%s", s)
	return out
}

//This takes the map to modify and a key name and adds the username to a new key with "_username"
func parseUid(m map[string]string, uidKey string) {
	//Make sure we have a uid element to work with.
	if uid, ok := m[uidKey]; ok {
		//Give a default value in case we don't find something.
		m[uidKey+"_username"] = "UNKNOWN_USER"
		if luser, ok := uidMap[uid]; ok {
			m[uidKey+"_username"] = luser.Username
		} else {
			luser, err := user.LookupId(uid)
			if err == nil {
				m[uidKey+"_username"] = luser.Username
				uidMap[uid] = *luser
				//Probably redundant. FIX
			} else {
				uidMap[uid] = user.User{Username: "UNKNOWN_USER"}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"reflect"
)

type JsonSchema struct {
	Schema string `json:"$schema"`
	Type string `json:"type"`
	Id string `json:"id"`
	Title string `json:"title"`
	Description string `json:"description"`
	Default interface{} `json:"default"`
	Enum []interface{} `json:"enum"`
	Definitions map[string]JsonSchema `json:"definitions"`
	Ref string `json:"$ref"`

	// string
	MinLength json.Number `json:"minLength"`
	MaxLength json.Number `json:"maxLength"`
	Pattern string `json:"pattern"`
	Format string `json:"format"`

	// numeric
	MultipleOf json.Number `json:"multipleOf"`
	Minimum json.Number `json:"minimum"`
	Maximum json.Number `json:"maximum"`
	ExclusiveMaximum bool `json:"exclusiveMaximum"`

	// object
	Properties map[string]JsonSchema `json:"properties"`
	AdditionalProperties interface{} `json:"additionalProperties"`
	Required []string `json:"required"`
	MinProperties json.Number `json:"minProperties"`
	MaxProperties json.Number `json:"maxProperties"`
	Dependencies map[string]JsonSchema `json:"dependencies"`

	// array
	Items interface{} `json:"items"`
	AdditionalItems bool `json:"additionalItems"`
	MinItems json.Number `json:"minItems"`
	MaxItems json.Number `json:"maxItems"`
	UniqueItems bool `json:"uniqueItems"`

	// combining
	AllOf []JsonSchema `json:"allOf"`
	AnyOf []JsonSchema `json:"anyOf"`
	OneOf []JsonSchema `json:"oneOf"`
	Not []JsonSchema `json:"not"`
}

func (schema *JsonSchema) GetItemList() ([]JsonSchema) {
	if schema.Items == nil {
		return nil
	}
	j, _ := json.Marshal(schema.Items)
	var res []JsonSchema
	switch reflect.ValueOf(schema.Items).Kind() {
	case reflect.Array, reflect.Slice:
		json.Unmarshal(j, &res)
	case reflect.Map:
		var s JsonSchema
		json.Unmarshal(j, &s)
		res = []JsonSchema{ s }
	}
	return res
}

func (schema *JsonSchema) IsRequired(key string) (bool) {
	if schema.Required == nil {
		return false
	}
	for _, r := range schema.Required {
		if key == r {
			return true
		}
	}
	return false
}

<commit_msg>implemented GetRefSchema<commit_after>package main

import (
	"encoding/json"
	"reflect"
	"strings"
	"github.com/xeipuuv/gojsonpointer"
)

type JsonSchema struct {
	Schema string `json:"$schema"`
	Type string `json:"type"`
	Id string `json:"id"`
	Title string `json:"title"`
	Description string `json:"description"`
	Default interface{} `json:"default"`
	Enum []interface{} `json:"enum"`
	Definitions map[string]JsonSchema `json:"definitions"`
	Ref string `json:"$ref"`

	// string
	MinLength json.Number `json:"minLength"`
	MaxLength json.Number `json:"maxLength"`
	Pattern string `json:"pattern"`
	Format string `json:"format"`

	// numeric
	MultipleOf json.Number `json:"multipleOf"`
	Minimum json.Number `json:"minimum"`
	Maximum json.Number `json:"maximum"`
	ExclusiveMaximum bool `json:"exclusiveMaximum"`

	// object
	Properties map[string]JsonSchema `json:"properties"`
	AdditionalProperties interface{} `json:"additionalProperties"`
	Required []string `json:"required"`
	MinProperties json.Number `json:"minProperties"`
	MaxProperties json.Number `json:"maxProperties"`
	Dependencies map[string]JsonSchema `json:"dependencies"`

	// array
	Items interface{} `json:"items"`
	AdditionalItems bool `json:"additionalItems"`
	MinItems json.Number `json:"minItems"`
	MaxItems json.Number `json:"maxItems"`
	UniqueItems bool `json:"uniqueItems"`

	// combining
	AllOf []JsonSchema `json:"allOf"`
	AnyOf []JsonSchema `json:"anyOf"`
	OneOf []JsonSchema `json:"oneOf"`
	Not []JsonSchema `json:"not"`
}

func (schema *JsonSchema) GetItemList() ([]JsonSchema) {
	if schema.Items == nil {
		return nil
	}
	j, _ := json.Marshal(schema.Items)
	var res []JsonSchema
	switch reflect.ValueOf(schema.Items).Kind() {
	case reflect.Array, reflect.Slice:
		json.Unmarshal(j, &res)
	case reflect.Map:
		var s JsonSchema
		json.Unmarshal(j, &s)
		res = []JsonSchema{ s }
	}
	return res
}

func (schema *JsonSchema) IsRequired(key string) (bool) {
	if schema.Required == nil {
		return false
	}
	for _, r := range schema.Required {
		if key == r {
			return true
		}
	}
	return false
}

func (schema JsonSchema) GetRefSchema(pointer *gojsonpointer.JsonPointer) (JsonSchema) {
	path := strings.Split(pointer.String(), "/")
	s := schema
	i := 0
	for i < len(path) {
		p := path[i]
		if p != "" && p == "definitions" {
			i++
			s = s.Definitions[path[i]]
		}
		i++
	}
	return s
}<|endoftext|>


<commit_before>package crawlbot

import (
	"errors"
	"github.com/moovweb/gokogiri/html"
	"mime"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type State int

// URL states
const (
	StateNotFound State = iota
	StatePending  State = iota
	StateRunning  State = iota
	StateRejected State = iota
	StateDone     State = iota
)

type Response struct {
	Resp    *http.Response
	URL     string
	Error   error
	Crawler *Crawler
	Doc     *html.HtmlDocument
	Bytes   []byte
}

type Crawler struct {
	URLs        []string                  // A list of URLs to start crawling. This is your list of seed URLs
	NumWorkers  int                       // Number of concurrent workers
	Handler     HandlerFunc               // Each page will be passed to this handler. Put your business logic here.
	CheckURL    CheckURLFunc              // Before a URL is crawled it is passed to this function to see if it should be followed or not. By default we follow the link if it's in the same domain as our seed URLs.
	CheckHeader CheckHeaderFunc           // Before reading in the body we can check the headers to see if we want to continue. By default we abort if it's not HTML.
	LinkFinder  LinkFinderFunc            // Call this function to find new links in the document to crawl. By default it will find all <a href> links
	Client      ClientFunc                // The crawler will call this function when it needs a new http.Client to give to a worker. By default it uses the built-in net/http Client.
	workers     []worker                  // List of all workers
	urlstate    map[string]State          // List of URLs and their current state.
	urlindex    map[State]map[string]bool // Index of URLs by their state
	urlmux      sync.RWMutex              // A mutex for protecting urlstate and urlindex
	state       bool                      // True means running. False means stopped.
}

// For each page crawled this function will be called.
// This is where your business logic should reside.
type HandlerFunc func(resp *Response)

// Check to see if the target URL should be crawled.
type CheckURLFunc func(crawler *Crawler, url string) bool

// Check to see if the target should be crawled after inspecting headers
type CheckHeaderFunc func(crawler *Crawler, url string, status int, header http.Header) bool

// Find new URLs to crawl
type LinkFinderFunc func(pres *Response) []string

// Check to see if the target URL should be crawled.
type ClientFunc func() *http.Client

// The default client is the built-in net/http Client with a 15 seconnd timeout
// A sensible alternative might be a simple round-tripper (eg. github.com/pkulak/simpletransport/simpletransport)
// If you wish to rate-throttle your crawler you would do so by implemting a custom http.Client
func DefaultClientFunc() *http.Client {
	return &http.Client{
		Timeout: 15 * time.Second,
	}
}

// The default URL Checker constrains the crawler to the domains of the seed URLs
func DefaultCheckURLFunc(crawler *Crawler, checkurl string) bool {
	parsedURL, err := url.Parse(checkurl)
	if err != nil {
		return false
	}
	for _, seedURL := range crawler.URLs {
		parsedSeed, err := url.Parse(seedURL)
		if err != nil {
			return false
		}
		if parsedSeed.Host == parsedURL.Host {
			return true
		}
	}
	return false
}

// The default header checker will only proceed if it's 200 OK and an HTML Content-Type
func DefaultCheckHeaderFunc(crawler *Crawler, url string, status int, header http.Header) bool {
	if status != 200 {
		return false
	}

	contentType := header.Get("Content-Type")
	if contentType == "" {
		return false
	}

	mediaType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		return false
	}

	if mediaType == "text/html" || mediaType == "application/xhtml+xml" {
		return true
	} else {
		return false
	}
}

// Create a new simple crawler
// If more customization options are needed then a Craweler{} should be created directly.
func NewCrawler(url string, handler HandlerFunc, numworkers int) *Crawler {
	return &Crawler{URLs: []string{url}, Handler: handler, CheckURL: DefaultCheckURLFunc, NumWorkers: numworkers, Client: DefaultClientFunc}
}

func (c *Crawler) Start() error {
	if c.state {
		return errors.New("Cannot start crawler that is already running")
	} else {
		c.state = true
	}

	if c.NumWorkers <= 0 {
		panic("Cannot create a new crawler with zero workers")
	}

	// Initialize urlstate and the starting URLs
	c.initializeURLs()

	// Initialize worker communication channels
	results := make(chan result)

	// Initialize workers
	c.workers = make([]worker, c.NumWorkers)
	for i := range c.workers {
		c.workers[i].crawler = c
		c.workers[i].results = results
		c.workers[i].client = c.Client()
	}

	// Start running in a for loop with selects
	for {
		select {
		case res := <-results:
			c.urlmux.Lock()

			c.urlstate[res.url] = StateDone
			delete(c.urlindex[StateRunning], res.url)
			c.urlindex[StateDone][res.url] = true

			res.owner.teardown()

			if res.err == nil {
				// Add the new items to our map
				for _, newurl := range res.newurls {
					if _, ok := c.urlstate[newurl]; ok {
						continue // Ignore URLs we already have
					}
					if c.CheckURL(c, newurl) {
						c.urlstate[newurl] = StatePending
						c.urlindex[StatePending][newurl] = true
					} else {
						c.urlstate[newurl] = StateRejected
						c.urlindex[StateRejected][newurl] = true
					}
				}
			}

			// Assign more work to the worker
			// If there's no work to do then skip
			if len(c.urlindex[StatePending]) == 0 {
				c.urlmux.Unlock()
				continue // continue select
			}

			c.assignWork(res.owner)
			c.urlmux.Unlock()
		default:
			c.urlmux.Lock()
			if len(c.urlindex[StatePending]) == 0 && len(c.urlindex[StateRunning]) == 0 {
				// We're done
				c.state = false
				c.urlmux.Unlock()
				return nil
			} else if len(c.urlindex[StatePending]) != 0 {
				for i := range c.workers {
					if !c.workers[i].state {
						c.assignWork(&c.workers[i])
						break
					}
				}
				c.urlmux.Unlock()
			} else {
				c.urlmux.Unlock()
				time.Sleep(100 * time.Millisecond)
			}
		}
	}
}

// Is the crawler currently running or is it stopped?
func (c *Crawler) IsRunning() bool {
	return c.state
}

// Add a URL to the crawler.
// If the item already exists this is a no-op
// @@TODO: change this behavior so an item is re-queued if it already exists -- tricky if the item is StateRunning
func (c *Crawler) AddURL(url string) {
	c.urlmux.Lock()
	if _, ok := c.urlstate[url]; ok {
		return
	}
	c.urlstate[url] = StatePending
	c.urlindex[StatePending][url] = true
	c.urlmux.Unlock()
}

// Get the current state for a URL
func (c *Crawler) GetURL(url string) State {
	c.urlmux.RLock()
	defer c.urlmux.RUnlock()

	state, ok := c.urlstate[url]
	if !ok {
		return StateNotFound
	}

	return state
}

// Assign work to a worker. Calling this function is unsafe unless wrapped inside a mutex lock
func (c *Crawler) assignWork(w *worker) {
	for url := range c.urlindex[StatePending] {
		c.urlstate[url] = StateRunning

		// Update the index
		delete(c.urlindex[StatePending], url)
		c.urlindex[StateRunning][url] = true

		// Assign work and return true
		w.setup(url)
		w.process()
		break
	}
}

// Build the index.
func (c *Crawler) initializeURLs() {
	c.urlmux.Lock()

	if c.urlstate == nil {
		c.urlstate = make(map[string]State)
	}
	for _, url := range c.URLs {
		if _, ok := c.urlstate[url]; !ok {
			c.urlstate[url] = StatePending
		}
	}

	// Build the index
	c.urlindex = make(map[State]map[string]bool)
	for _, state := range []State{StatePending, StateRejected, StateRunning, StateDone} {
		c.urlindex[state] = make(map[string]bool)
	}

	for url, state := range c.urlstate {
		c.urlindex[state][url] = true
	}

	c.urlmux.Unlock()
}
<commit_msg>streamlining and documentation<commit_after>package crawlbot

import (
	"errors"
	"github.com/moovweb/gokogiri/html"
	"mime"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type State int

// URL states
const (
	StateNotFound State = iota
	StatePending  State = iota
	StateRunning  State = iota
	StateRejected State = iota
	StateDone     State = iota
)

type Response struct {
	Resp    *http.Response
	URL     string
	Error   error
	Crawler *Crawler
	Doc     *html.HtmlDocument
	Bytes   []byte
}

type Crawler struct {
	// A list of URLs to start crawling. This is your list of seed URLs.
	URLs []string

	// Number of concurrent workers
	NumWorkers int

	// For each page crawled this function will be called.
	// This is where your business logic should reside.
	// There is no default. If Handler is not set the crawler will panic.
	Handler func(resp *Response)

	// Before a URL is crawled it is passed to this function to see if it should be followed or not.
	// By default we follow the link if it's in one of the same domains as our seed URLs.
	CheckURL func(crawler *Crawler, url string) bool

	// Before reading in the body we can check the headers to see if we want to continue.
	// By default we abort if it's not HTTP 200 OK or not an html Content-Type.
	// Override this function if you wish to handle non-html files such as binary images
	CheckHeader func(crawler *Crawler, url string, status int, header http.Header) bool

	// This function is called to find new urls in the document to crawl. By default it will
	// find all <a href> links in an html document. Override this function if you wish to follow
	// non <a href> links such as <img src>, or if you wish to find links in non-html documents.
	LinkFinder func(pres *Response) []string

	// The crawler will call this function when it needs a new http.Client to give to a worker.
	// The default client is the built-in net/http Client with a 15 seconnd timeout
	// A sensible alternative might be a simple round-tripper (eg. github.com/pkulak/simpletransport/simpletransport)
	// If you wish to rate-throttle your crawler you would do so by implemting a custom http.Client
	Client func() *http.Client

	workers  []worker                  // List of all workers
	urlstate map[string]State          // List of URLs and their current state.
	urlindex map[State]map[string]bool // Index of URLs by their state
	urlmux   sync.RWMutex              // A mutex for protecting urlstate and urlindex
	state    bool                      // True means running. False means stopped.
}

// The default client is the built-in net/http Client with a 15 second timeout
func DefaultClient() *http.Client {
	return &http.Client{
		Timeout: 15 * time.Second,
	}
}

// The default URL Checker constrains the crawler to the domains of the seed URLs
func DefaultCheckURL(crawler *Crawler, checkurl string) bool {
	parsedURL, err := url.Parse(checkurl)
	if err != nil {
		return false
	}
	for _, seedURL := range crawler.URLs {
		parsedSeed, err := url.Parse(seedURL)
		if err != nil {
			return false
		}
		if parsedSeed.Host == parsedURL.Host {
			return true
		}
	}
	return false
}

// The default header checker will only proceed if it's 200 OK and an HTML Content-Type
func DefaultCheckHeader(crawler *Crawler, url string, status int, header http.Header) bool {
	if status != 200 {
		return false
	}

	contentType := header.Get("Content-Type")
	if contentType == "" {
		return false
	}

	mediaType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		return false
	}

	if mediaType == "text/html" || mediaType == "application/xhtml+xml" {
		return true
	} else {
		return false
	}
}

// Create a new simple crawler
// If more customization options are needed then a Craweler{} should be created directly.
func NewCrawler(url string, handler func(resp *Response), numworkers int) *Crawler {
	return &Crawler{URLs: []string{url}, Handler: handler, NumWorkers: numworkers}
}

func (c *Crawler) Start() error {
	// Check to see if the crawler is already running
	if c.state {
		return errors.New("Cannot start crawler that is already running")
	} else {
		c.state = true
	}

	// Sanity check
	if c.NumWorkers <= 0 {
		panic("Cannot create a new crawler with zero workers")
	}
	if c.Handler == nil {
		panic("Cannot start a crawler that doesn't have a Hanlder function.")
	}
	if len(c.URLs) == 0 {
		panic("Cannot start a crawler with no URLs.")
	}

	// Initialize the default functions
	if c.CheckHeader == nil {
		c.CheckHeader = DefaultCheckHeader
	}
	if c.CheckURL == nil {
		c.CheckURL = DefaultCheckURL
	}
	if c.Client == nil {
		c.Client = DefaultClient
	}

	// Initialize urlstate and the starting URLs
	c.initializeURLs()

	// Initialize worker communication channels
	results := make(chan result)

	// Initialize workers
	c.workers = make([]worker, c.NumWorkers)
	for i := range c.workers {
		c.workers[i].crawler = c
		c.workers[i].results = results
		c.workers[i].client = c.Client()
	}

	// Start running in a for loop with selects
	for {
		select {
		case res := <-results:
			c.urlmux.Lock()

			c.urlstate[res.url] = StateDone
			delete(c.urlindex[StateRunning], res.url)
			c.urlindex[StateDone][res.url] = true

			res.owner.teardown()

			if res.err == nil {
				// Add the new items to our map
				for _, newurl := range res.newurls {
					if _, ok := c.urlstate[newurl]; ok {
						continue // Ignore URLs we already have
					}
					if c.CheckURL(c, newurl) {
						c.urlstate[newurl] = StatePending
						c.urlindex[StatePending][newurl] = true
					} else {
						c.urlstate[newurl] = StateRejected
						c.urlindex[StateRejected][newurl] = true
					}
				}
			}

			// Assign more work to the worker
			// If there's no work to do then skip
			if len(c.urlindex[StatePending]) == 0 {
				c.urlmux.Unlock()
				continue // continue select
			}

			c.assignWork(res.owner)
			c.urlmux.Unlock()
		default:
			c.urlmux.Lock()
			if len(c.urlindex[StatePending]) == 0 && len(c.urlindex[StateRunning]) == 0 {
				// We're done
				c.state = false
				c.urlmux.Unlock()
				return nil
			} else if len(c.urlindex[StatePending]) != 0 {
				for i := range c.workers {
					if !c.workers[i].state {
						c.assignWork(&c.workers[i])
						break
					}
				}
				c.urlmux.Unlock()
			} else {
				c.urlmux.Unlock()
				time.Sleep(100 * time.Millisecond)
			}
		}
	}
}

// Is the crawler currently running or is it stopped?
func (c *Crawler) IsRunning() bool {
	return c.state
}

// Add a URL to the crawler.
// If the item already exists this is a no-op
// @@TODO: change this behavior so an item is re-queued if it already exists -- tricky if the item is StateRunning
func (c *Crawler) AddURL(url string) {
	c.urlmux.Lock()
	if _, ok := c.urlstate[url]; ok {
		return
	}
	c.urlstate[url] = StatePending
	c.urlindex[StatePending][url] = true
	c.urlmux.Unlock()
}

// Get the current state for a URL
func (c *Crawler) GetURL(url string) State {
	c.urlmux.RLock()
	defer c.urlmux.RUnlock()

	state, ok := c.urlstate[url]
	if !ok {
		return StateNotFound
	}

	return state
}

// Assign work to a worker. Calling this function is unsafe unless wrapped inside a mutex lock
func (c *Crawler) assignWork(w *worker) {
	for url := range c.urlindex[StatePending] {
		c.urlstate[url] = StateRunning

		// Update the index
		delete(c.urlindex[StatePending], url)
		c.urlindex[StateRunning][url] = true

		// Assign work and return true
		w.setup(url)
		w.process()
		break
	}
}

// Build the index.
func (c *Crawler) initializeURLs() {
	c.urlmux.Lock()

	if c.urlstate == nil {
		c.urlstate = make(map[string]State)
	}
	for _, url := range c.URLs {
		if _, ok := c.urlstate[url]; !ok {
			c.urlstate[url] = StatePending
		}
	}

	// Build the index
	c.urlindex = make(map[State]map[string]bool)
	for _, state := range []State{StatePending, StateRejected, StateRunning, StateDone} {
		c.urlindex[state] = make(map[string]bool)
	}

	for url, state := range c.urlstate {
		c.urlindex[state][url] = true
	}

	c.urlmux.Unlock()
}
<|endoftext|>


<commit_before>package libnetwork

import (
	"encoding/json"
	"fmt"
	"net"

	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
)

// EndpointInfo provides an interface to retrieve network resources bound to the endpoint.
type EndpointInfo interface {
	// Iface returns InterfaceInfo, go interface that can be used
	// to get more information on the interface which was assigned to
	// the endpoint by the driver. This can be used after the
	// endpoint has been created.
	Iface() InterfaceInfo

	// Gateway returns the IPv4 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	Gateway() net.IP

	// GatewayIPv6 returns the IPv6 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	GatewayIPv6() net.IP

	// StaticRoutes returns the list of static routes configured by the network
	// driver when the container joins a network
	StaticRoutes() []*types.StaticRoute

	// Sandbox returns the attached sandbox if there, nil otherwise.
	Sandbox() Sandbox
}

// InterfaceInfo provides an interface to retrieve interface addresses bound to the endpoint.
type InterfaceInfo interface {
	// MacAddress returns the MAC address assigned to the endpoint.
	MacAddress() net.HardwareAddr

	// Address returns the IPv4 address assigned to the endpoint.
	Address() *net.IPNet

	// AddressIPv6 returns the IPv6 address assigned to the endpoint.
	AddressIPv6() *net.IPNet

	// LinkLocalAddresses returns the list of link-local (IPv4/IPv6) addresses assigned to the endpoint.
	LinkLocalAddresses() []*net.IPNet
}

type endpointInterface struct {
	mac       net.HardwareAddr
	addr      *net.IPNet
	addrv6    *net.IPNet
	llAddrs   []*net.IPNet
	srcName   string
	dstPrefix string
	routes    []*net.IPNet
	v4PoolID  string
	v6PoolID  string
}

func (epi *endpointInterface) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	if epi.mac != nil {
		epMap["mac"] = epi.mac.String()
	}
	if epi.addr != nil {
		epMap["addr"] = epi.addr.String()
	}
	if epi.addrv6 != nil {
		epMap["addrv6"] = epi.addrv6.String()
	}
	if len(epi.llAddrs) != 0 {
		list := make([]string, 0, len(epi.llAddrs))
		for _, ll := range epi.llAddrs {
			list = append(list, ll.String())
		}
		epMap["llAddrs"] = list
	}
	epMap["srcName"] = epi.srcName
	epMap["dstPrefix"] = epi.dstPrefix
	var routes []string
	for _, route := range epi.routes {
		routes = append(routes, route.String())
	}
	epMap["routes"] = routes
	epMap["v4PoolID"] = epi.v4PoolID
	epMap["v6PoolID"] = epi.v6PoolID
	return json.Marshal(epMap)
}

func (epi *endpointInterface) UnmarshalJSON(b []byte) error {
	var (
		err   error
		epMap map[string]interface{}
	)
	if err = json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	if v, ok := epMap["mac"]; ok {
		if epi.mac, err = net.ParseMAC(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string))
		}
	}
	if v, ok := epMap["addr"]; ok {
		if epi.addr, err = types.ParseCIDR(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err)
		}
	}
	if v, ok := epMap["addrv6"]; ok {
		if epi.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface ipv6 address after json unmarshal: %v", err)
		}
	}
	if v, ok := epMap["llAddrs"]; ok {
		list := v.([]string)
		epi.llAddrs = make([]*net.IPNet, 0, len(list))
		for _, llS := range list {
			ll, err := types.ParseCIDR(llS)
			if err != nil {
				return types.InternalErrorf("failed to decode endpoint interface link-local address (%s) after json unmarshal: %v", llS, err)
			}
			epi.llAddrs = append(epi.llAddrs, ll)
		}
	}
	epi.srcName = epMap["srcName"].(string)
	epi.dstPrefix = epMap["dstPrefix"].(string)

	rb, _ := json.Marshal(epMap["routes"])
	var routes []string
	json.Unmarshal(rb, &routes)
	epi.routes = make([]*net.IPNet, 0)
	for _, route := range routes {
		ip, ipr, err := net.ParseCIDR(route)
		if err == nil {
			ipr.IP = ip
			epi.routes = append(epi.routes, ipr)
		}
	}
	epi.v4PoolID = epMap["v4PoolID"].(string)
	epi.v6PoolID = epMap["v6PoolID"].(string)

	return nil
}

func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error {
	dstEpi.mac = types.GetMacCopy(epi.mac)
	dstEpi.addr = types.GetIPNetCopy(epi.addr)
	dstEpi.addrv6 = types.GetIPNetCopy(epi.addrv6)
	dstEpi.srcName = epi.srcName
	dstEpi.dstPrefix = epi.dstPrefix
	dstEpi.v4PoolID = epi.v4PoolID
	dstEpi.v6PoolID = epi.v6PoolID
	if len(epi.llAddrs) != 0 {
		dstEpi.llAddrs = make([]*net.IPNet, 0, len(epi.llAddrs))
		for _, ll := range epi.llAddrs {
			dstEpi.llAddrs = append(dstEpi.llAddrs, ll)
		}
	}

	for _, route := range epi.routes {
		dstEpi.routes = append(dstEpi.routes, types.GetIPNetCopy(route))
	}

	return nil
}

type endpointJoinInfo struct {
	gw                    net.IP
	gw6                   net.IP
	StaticRoutes          []*types.StaticRoute
	driverTableEntries    []*tableEntry
	disableGatewayService bool
}

type tableEntry struct {
	tableName string
	key       string
	value     []byte
}

func (ep *endpoint) Info() EndpointInfo {
	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil
	}

	ep, err = n.getEndpointFromStore(ep.ID())
	if err != nil {
		return nil
	}

	sb, ok := ep.getSandbox()
	if !ok {
		// endpoint hasn't joined any sandbox.
		// Just return the endpoint
		return ep
	}

	if epi := sb.getEndpoint(ep.ID()); epi != nil {
		return epi
	}

	return nil
}

func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
	ep, err := ep.retrieveFromStore()
	if err != nil {
		return nil, err
	}

	if sb, ok := ep.getSandbox(); ok {
		if gwep := sb.getEndpointInGWNetwork(); gwep != nil && gwep.ID() != ep.ID() {
			return gwep.DriverInfo()
		}
	}

	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
	}

	driver, err := n.driver(true)
	if err != nil {
		return nil, fmt.Errorf("failed to get driver info: %v", err)
	}

	return driver.EndpointOperInfo(n.ID(), ep.ID())
}

func (ep *endpoint) Iface() InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (ep *endpoint) Interface() driverapi.InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (epi *endpointInterface) SetMacAddress(mac net.HardwareAddr) error {
	if epi.mac != nil {
		return types.ForbiddenErrorf("endpoint interface MAC address present (%s). Cannot be modified with %s.", epi.mac, mac)
	}
	if mac == nil {
		return types.BadRequestErrorf("tried to set nil MAC address to endpoint interface")
	}
	epi.mac = types.GetMacCopy(mac)
	return nil
}

func (epi *endpointInterface) SetIPAddress(address *net.IPNet) error {
	if address.IP == nil {
		return types.BadRequestErrorf("tried to set nil IP address to endpoint interface")
	}
	if address.IP.To4() == nil {
		return setAddress(&epi.addrv6, address)
	}
	return setAddress(&epi.addr, address)
}

func setAddress(ifaceAddr **net.IPNet, address *net.IPNet) error {
	if *ifaceAddr != nil {
		return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
	}
	*ifaceAddr = types.GetIPNetCopy(address)
	return nil
}

func (epi *endpointInterface) MacAddress() net.HardwareAddr {
	return types.GetMacCopy(epi.mac)
}

func (epi *endpointInterface) Address() *net.IPNet {
	return types.GetIPNetCopy(epi.addr)
}

func (epi *endpointInterface) AddressIPv6() *net.IPNet {
	return types.GetIPNetCopy(epi.addrv6)
}

func (epi *endpointInterface) LinkLocalAddresses() []*net.IPNet {
	return epi.llAddrs
}

func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
	epi.srcName = srcName
	epi.dstPrefix = dstPrefix
	return nil
}

func (ep *endpoint) InterfaceName() driverapi.InterfaceNameInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	r := types.StaticRoute{Destination: destination, RouteType: routeType, NextHop: nextHop}

	if routeType == types.NEXTHOP {
		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).
		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)
	} else {
		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.
		ep.iface.routes = append(ep.iface.routes, r.Destination)
	}
	return nil
}

func (ep *endpoint) AddTableEntry(tableName, key string, value []byte) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.driverTableEntries = append(ep.joinInfo.driverTableEntries, &tableEntry{
		tableName: tableName,
		key:       key,
		value:     value,
	})

	return nil
}

func (ep *endpoint) Sandbox() Sandbox {
	cnt, ok := ep.getSandbox()
	if !ok {
		return nil
	}
	return cnt
}

func (ep *endpoint) StaticRoutes() []*types.StaticRoute {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return nil
	}

	return ep.joinInfo.StaticRoutes
}

func (ep *endpoint) Gateway() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw)
}

func (ep *endpoint) GatewayIPv6() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw6)
}

func (ep *endpoint) SetGateway(gw net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw = types.GetIPCopy(gw)
	return nil
}

func (ep *endpoint) SetGatewayIPv6(gw6 net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw6 = types.GetIPCopy(gw6)
	return nil
}

func (ep *endpoint) retrieveFromStore() (*endpoint, error) {
	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil, fmt.Errorf("could not find network in store to get latest endpoint %s: %v", ep.Name(), err)
	}
	return n.getEndpointFromStore(ep.ID())
}

func (ep *endpoint) DisableGatewayService() {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.disableGatewayService = true
}

func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	if epj.gw != nil {
		epMap["gw"] = epj.gw.String()
	}
	if epj.gw6 != nil {
		epMap["gw6"] = epj.gw6.String()
	}
	epMap["disableGatewayService"] = epj.disableGatewayService
	epMap["StaticRoutes"] = epj.StaticRoutes
	return json.Marshal(epMap)
}

func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
	var (
		err   error
		epMap map[string]interface{}
	)
	if err = json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	if v, ok := epMap["gw"]; ok {
		epj.gw6 = net.ParseIP(v.(string))
	}
	if v, ok := epMap["gw6"]; ok {
		epj.gw6 = net.ParseIP(v.(string))
	}
	epj.disableGatewayService = epMap["disableGatewayService"].(bool)

	var tStaticRoute []types.StaticRoute
	if v, ok := epMap["StaticRoutes"]; ok {
		tb, _ := json.Marshal(v)
		var tStaticRoute []types.StaticRoute
		json.Unmarshal(tb, &tStaticRoute)
	}
	var StaticRoutes []*types.StaticRoute
	for _, r := range tStaticRoute {
		StaticRoutes = append(StaticRoutes, &r)
	}
	epj.StaticRoutes = StaticRoutes

	return nil
}

func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
	dstEpj.disableGatewayService = epj.disableGatewayService
	dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
	copy(dstEpj.StaticRoutes, epj.StaticRoutes)
	dstEpj.gw = types.GetIPCopy(epj.gw)
	dstEpj.gw = types.GetIPCopy(epj.gw6)
	return nil
}
<commit_msg>Ensure drivertable entries in joininfo is uptodate<commit_after>package libnetwork

import (
	"encoding/json"
	"fmt"
	"net"

	"github.com/docker/libnetwork/driverapi"
	"github.com/docker/libnetwork/types"
)

// EndpointInfo provides an interface to retrieve network resources bound to the endpoint.
type EndpointInfo interface {
	// Iface returns InterfaceInfo, go interface that can be used
	// to get more information on the interface which was assigned to
	// the endpoint by the driver. This can be used after the
	// endpoint has been created.
	Iface() InterfaceInfo

	// Gateway returns the IPv4 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	Gateway() net.IP

	// GatewayIPv6 returns the IPv6 gateway assigned by the driver.
	// This will only return a valid value if a container has joined the endpoint.
	GatewayIPv6() net.IP

	// StaticRoutes returns the list of static routes configured by the network
	// driver when the container joins a network
	StaticRoutes() []*types.StaticRoute

	// Sandbox returns the attached sandbox if there, nil otherwise.
	Sandbox() Sandbox
}

// InterfaceInfo provides an interface to retrieve interface addresses bound to the endpoint.
type InterfaceInfo interface {
	// MacAddress returns the MAC address assigned to the endpoint.
	MacAddress() net.HardwareAddr

	// Address returns the IPv4 address assigned to the endpoint.
	Address() *net.IPNet

	// AddressIPv6 returns the IPv6 address assigned to the endpoint.
	AddressIPv6() *net.IPNet

	// LinkLocalAddresses returns the list of link-local (IPv4/IPv6) addresses assigned to the endpoint.
	LinkLocalAddresses() []*net.IPNet
}

type endpointInterface struct {
	mac       net.HardwareAddr
	addr      *net.IPNet
	addrv6    *net.IPNet
	llAddrs   []*net.IPNet
	srcName   string
	dstPrefix string
	routes    []*net.IPNet
	v4PoolID  string
	v6PoolID  string
}

func (epi *endpointInterface) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	if epi.mac != nil {
		epMap["mac"] = epi.mac.String()
	}
	if epi.addr != nil {
		epMap["addr"] = epi.addr.String()
	}
	if epi.addrv6 != nil {
		epMap["addrv6"] = epi.addrv6.String()
	}
	if len(epi.llAddrs) != 0 {
		list := make([]string, 0, len(epi.llAddrs))
		for _, ll := range epi.llAddrs {
			list = append(list, ll.String())
		}
		epMap["llAddrs"] = list
	}
	epMap["srcName"] = epi.srcName
	epMap["dstPrefix"] = epi.dstPrefix
	var routes []string
	for _, route := range epi.routes {
		routes = append(routes, route.String())
	}
	epMap["routes"] = routes
	epMap["v4PoolID"] = epi.v4PoolID
	epMap["v6PoolID"] = epi.v6PoolID
	return json.Marshal(epMap)
}

func (epi *endpointInterface) UnmarshalJSON(b []byte) error {
	var (
		err   error
		epMap map[string]interface{}
	)
	if err = json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	if v, ok := epMap["mac"]; ok {
		if epi.mac, err = net.ParseMAC(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface mac address after json unmarshal: %s", v.(string))
		}
	}
	if v, ok := epMap["addr"]; ok {
		if epi.addr, err = types.ParseCIDR(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface ipv4 address after json unmarshal: %v", err)
		}
	}
	if v, ok := epMap["addrv6"]; ok {
		if epi.addrv6, err = types.ParseCIDR(v.(string)); err != nil {
			return types.InternalErrorf("failed to decode endpoint interface ipv6 address after json unmarshal: %v", err)
		}
	}
	if v, ok := epMap["llAddrs"]; ok {
		list := v.([]string)
		epi.llAddrs = make([]*net.IPNet, 0, len(list))
		for _, llS := range list {
			ll, err := types.ParseCIDR(llS)
			if err != nil {
				return types.InternalErrorf("failed to decode endpoint interface link-local address (%s) after json unmarshal: %v", llS, err)
			}
			epi.llAddrs = append(epi.llAddrs, ll)
		}
	}
	epi.srcName = epMap["srcName"].(string)
	epi.dstPrefix = epMap["dstPrefix"].(string)

	rb, _ := json.Marshal(epMap["routes"])
	var routes []string
	json.Unmarshal(rb, &routes)
	epi.routes = make([]*net.IPNet, 0)
	for _, route := range routes {
		ip, ipr, err := net.ParseCIDR(route)
		if err == nil {
			ipr.IP = ip
			epi.routes = append(epi.routes, ipr)
		}
	}
	epi.v4PoolID = epMap["v4PoolID"].(string)
	epi.v6PoolID = epMap["v6PoolID"].(string)

	return nil
}

func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error {
	dstEpi.mac = types.GetMacCopy(epi.mac)
	dstEpi.addr = types.GetIPNetCopy(epi.addr)
	dstEpi.addrv6 = types.GetIPNetCopy(epi.addrv6)
	dstEpi.srcName = epi.srcName
	dstEpi.dstPrefix = epi.dstPrefix
	dstEpi.v4PoolID = epi.v4PoolID
	dstEpi.v6PoolID = epi.v6PoolID
	if len(epi.llAddrs) != 0 {
		dstEpi.llAddrs = make([]*net.IPNet, 0, len(epi.llAddrs))
		for _, ll := range epi.llAddrs {
			dstEpi.llAddrs = append(dstEpi.llAddrs, ll)
		}
	}

	for _, route := range epi.routes {
		dstEpi.routes = append(dstEpi.routes, types.GetIPNetCopy(route))
	}

	return nil
}

type endpointJoinInfo struct {
	gw                    net.IP
	gw6                   net.IP
	StaticRoutes          []*types.StaticRoute
	driverTableEntries    []*tableEntry
	disableGatewayService bool
}

type tableEntry struct {
	tableName string
	key       string
	value     []byte
}

func (ep *endpoint) Info() EndpointInfo {
	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil
	}

	ep, err = n.getEndpointFromStore(ep.ID())
	if err != nil {
		return nil
	}

	sb, ok := ep.getSandbox()
	if !ok {
		// endpoint hasn't joined any sandbox.
		// Just return the endpoint
		return ep
	}

	if epi := sb.getEndpoint(ep.ID()); epi != nil {
		return epi
	}

	return nil
}

func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
	ep, err := ep.retrieveFromStore()
	if err != nil {
		return nil, err
	}

	if sb, ok := ep.getSandbox(); ok {
		if gwep := sb.getEndpointInGWNetwork(); gwep != nil && gwep.ID() != ep.ID() {
			return gwep.DriverInfo()
		}
	}

	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
	}

	driver, err := n.driver(true)
	if err != nil {
		return nil, fmt.Errorf("failed to get driver info: %v", err)
	}

	return driver.EndpointOperInfo(n.ID(), ep.ID())
}

func (ep *endpoint) Iface() InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (ep *endpoint) Interface() driverapi.InterfaceInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (epi *endpointInterface) SetMacAddress(mac net.HardwareAddr) error {
	if epi.mac != nil {
		return types.ForbiddenErrorf("endpoint interface MAC address present (%s). Cannot be modified with %s.", epi.mac, mac)
	}
	if mac == nil {
		return types.BadRequestErrorf("tried to set nil MAC address to endpoint interface")
	}
	epi.mac = types.GetMacCopy(mac)
	return nil
}

func (epi *endpointInterface) SetIPAddress(address *net.IPNet) error {
	if address.IP == nil {
		return types.BadRequestErrorf("tried to set nil IP address to endpoint interface")
	}
	if address.IP.To4() == nil {
		return setAddress(&epi.addrv6, address)
	}
	return setAddress(&epi.addr, address)
}

func setAddress(ifaceAddr **net.IPNet, address *net.IPNet) error {
	if *ifaceAddr != nil {
		return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
	}
	*ifaceAddr = types.GetIPNetCopy(address)
	return nil
}

func (epi *endpointInterface) MacAddress() net.HardwareAddr {
	return types.GetMacCopy(epi.mac)
}

func (epi *endpointInterface) Address() *net.IPNet {
	return types.GetIPNetCopy(epi.addr)
}

func (epi *endpointInterface) AddressIPv6() *net.IPNet {
	return types.GetIPNetCopy(epi.addrv6)
}

func (epi *endpointInterface) LinkLocalAddresses() []*net.IPNet {
	return epi.llAddrs
}

func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
	epi.srcName = srcName
	epi.dstPrefix = dstPrefix
	return nil
}

func (ep *endpoint) InterfaceName() driverapi.InterfaceNameInfo {
	ep.Lock()
	defer ep.Unlock()

	if ep.iface != nil {
		return ep.iface
	}

	return nil
}

func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	r := types.StaticRoute{Destination: destination, RouteType: routeType, NextHop: nextHop}

	if routeType == types.NEXTHOP {
		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).
		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)
	} else {
		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.
		ep.iface.routes = append(ep.iface.routes, r.Destination)
	}
	return nil
}

func (ep *endpoint) AddTableEntry(tableName, key string, value []byte) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.driverTableEntries = append(ep.joinInfo.driverTableEntries, &tableEntry{
		tableName: tableName,
		key:       key,
		value:     value,
	})

	return nil
}

func (ep *endpoint) Sandbox() Sandbox {
	cnt, ok := ep.getSandbox()
	if !ok {
		return nil
	}
	return cnt
}

func (ep *endpoint) StaticRoutes() []*types.StaticRoute {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return nil
	}

	return ep.joinInfo.StaticRoutes
}

func (ep *endpoint) Gateway() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw)
}

func (ep *endpoint) GatewayIPv6() net.IP {
	ep.Lock()
	defer ep.Unlock()

	if ep.joinInfo == nil {
		return net.IP{}
	}

	return types.GetIPCopy(ep.joinInfo.gw6)
}

func (ep *endpoint) SetGateway(gw net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw = types.GetIPCopy(gw)
	return nil
}

func (ep *endpoint) SetGatewayIPv6(gw6 net.IP) error {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.gw6 = types.GetIPCopy(gw6)
	return nil
}

func (ep *endpoint) retrieveFromStore() (*endpoint, error) {
	n, err := ep.getNetworkFromStore()
	if err != nil {
		return nil, fmt.Errorf("could not find network in store to get latest endpoint %s: %v", ep.Name(), err)
	}
	return n.getEndpointFromStore(ep.ID())
}

func (ep *endpoint) DisableGatewayService() {
	ep.Lock()
	defer ep.Unlock()

	ep.joinInfo.disableGatewayService = true
}

func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
	epMap := make(map[string]interface{})
	if epj.gw != nil {
		epMap["gw"] = epj.gw.String()
	}
	if epj.gw6 != nil {
		epMap["gw6"] = epj.gw6.String()
	}
	epMap["disableGatewayService"] = epj.disableGatewayService
	epMap["StaticRoutes"] = epj.StaticRoutes
	return json.Marshal(epMap)
}

func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
	var (
		err   error
		epMap map[string]interface{}
	)
	if err = json.Unmarshal(b, &epMap); err != nil {
		return err
	}
	if v, ok := epMap["gw"]; ok {
		epj.gw6 = net.ParseIP(v.(string))
	}
	if v, ok := epMap["gw6"]; ok {
		epj.gw6 = net.ParseIP(v.(string))
	}
	epj.disableGatewayService = epMap["disableGatewayService"].(bool)

	var tStaticRoute []types.StaticRoute
	if v, ok := epMap["StaticRoutes"]; ok {
		tb, _ := json.Marshal(v)
		var tStaticRoute []types.StaticRoute
		json.Unmarshal(tb, &tStaticRoute)
	}
	var StaticRoutes []*types.StaticRoute
	for _, r := range tStaticRoute {
		StaticRoutes = append(StaticRoutes, &r)
	}
	epj.StaticRoutes = StaticRoutes

	return nil
}

func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
	dstEpj.disableGatewayService = epj.disableGatewayService
	dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
	copy(dstEpj.StaticRoutes, epj.StaticRoutes)
	dstEpj.driverTableEntries = make([]*tableEntry, len(epj.driverTableEntries))
	copy(dstEpj.driverTableEntries, epj.driverTableEntries)
	dstEpj.gw = types.GetIPCopy(epj.gw)
	dstEpj.gw = types.GetIPCopy(epj.gw6)
	return nil
}
<|endoftext|>


<commit_before>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package backend

import (
	"atlantis/router/logger"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Server struct {
	Address   string
	Status    ServerStatus
	Metrics   ServerMetrics
	Transport *http.Transport
	copier    *Copier
}

func NewServer(address string) *Server {
	return &Server{
		Address: address,
		Status:  NewServerStatus(),
		Metrics: NewServerMetrics(),
		Transport: &http.Transport{
			MaxIdleConnsPerHost: 32,
		},
		copier: NewCopier(),
	}
}

type ResponseError struct {
	Response *http.Response
	Error    error
}

func (s *Server) RoundTrip(req *http.Request, ch chan ResponseError) {
	defer func() {
		if r := recover(); r != nil {
			err := fmt.Sprintf("%s", r)
			ch <- ResponseError{nil, errors.New(err)}
		}
	}()

	s.Metrics.RequestStart()
	defer s.Metrics.RequestDone()

	req.URL.Scheme = "http"
	req.URL.Host = s.Address

	resp, err := s.Transport.RoundTrip(req)
	if err == nil {
		ch <- ResponseError{resp, nil}
	} else {
		ch <- ResponseError{resp, err}
	}
}

func (s *Server) logPrefix(r *http.Request, tstart time.Time) string {
	now := time.Now()

	var rtt0, rtt1 int64

	// Calculate the total round trip time if front end inserted the atlantis-arrival-time
	// header before routing the request. The header is assumed to be created by calling
	// time.Now().UnixNano() or equivalent.
	arr, err := strconv.ParseInt(r.Header.Get("atlantis-arrival-time"), 10, 64)
	if err != nil {
		rtt0 = now.UnixNano() - arr
	}

	rtt1 = now.UnixNano() - tstart.UnixNano()

	// Log prefix includes server address, request source and URI, and round trip times.
	return fmt.Sprintf("[server %s][request %s|%s][rtt %d|%d]", s.Address, r.RemoteAddr,
		r.URL, rtt0, rtt1)
}

func (s *Server) Handle(w http.ResponseWriter, r *http.Request, tout time.Duration) {
	// X-Forwarded-For; we are a proxy.
	ip := strings.Split(r.RemoteAddr, ":")[0]
	r.Header.Add("X-Forwarded-For", ip)

	resErrCh := make(chan ResponseError)
	tstart := time.Now()
	go s.RoundTrip(r, resErrCh)

	select {
	case resErr := <-resErrCh:
		if resErr.Error == nil {
			logger.Printf("%s %d", s.logPrefix(r, tstart), resErr.Response.StatusCode)
			defer resErr.Response.Body.Close()
			for hdr, vals := range resErr.Response.Header {
				for _, val := range vals {
					w.Header().Add(hdr, val)
				}
			}
			w.WriteHeader(resErr.Response.StatusCode)
			_, err := s.copier.Copy(w, resErr.Response.Body)
			if err != nil {
				logger.Errorf("%s %s", s.logPrefix(r, tstart), err)
			}
		} else {
			logger.Errorf("%s %s", s.logPrefix(r, tstart), resErr.Error)
			http.Error(w, "Bad Gateway", http.StatusBadGateway)
		}
	case <-time.After(tout):
		s.Transport.CancelRequest(r)
		logger.Printf("%s timeout", s.logPrefix(r, tstart))
		http.Error(w, "Gateway Timeout", http.StatusGatewayTimeout)
	}
}

func (s *Server) CheckStatus(tout time.Duration) {
	r, _ := http.NewRequest("GET", "http://"+s.Address+"/healthz", nil)

	resErrCh := make(chan ResponseError)
	tstart := time.Now()
	go s.RoundTrip(r, resErrCh)

	select {
	case resErr := <-resErrCh:
		if resErr.Error == nil {
			logger.Printf("%s %d", s.logPrefix(r, tstart), resErr.Response.StatusCode)
			defer resErr.Response.Body.Close()
			s.Status.ParseAndSet(resErr.Response)
		} else {
			logger.Errorf("%s %s", s.logPrefix(r, tstart), resErr.Error)
			s.Status.Set(StatusCritical)
		}
	case <-time.After(tout):
		logger.Errorf("%s timeout", s.logPrefix(r, tstart))
		s.Status.Set(StatusCritical)
	}
}

func (s *Server) Cost(accept string) uint32 {
	return s.Status.Cost(accept) + s.Metrics.Cost()
}

func (s *Server) Shutdown() {
	s.copier.Shutdown()
}
<commit_msg>Bugfix, timeouts don't decrement count.<commit_after>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package backend

import (
	"atlantis/router/logger"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Server struct {
	Address   string
	Status    ServerStatus
	Metrics   ServerMetrics
	Transport *http.Transport
	copier    *Copier
}

func NewServer(address string) *Server {
	return &Server{
		Address: address,
		Status:  NewServerStatus(),
		Metrics: NewServerMetrics(),
		Transport: &http.Transport{
			MaxIdleConnsPerHost: 32,
		},
		copier: NewCopier(),
	}
}

type ResponseError struct {
	Response *http.Response
	Error    error
}

func (s *Server) RoundTrip(req *http.Request, ch chan ResponseError) {
	defer func() {
		if r := recover(); r != nil {
			err := fmt.Sprintf("%s", r)
			ch <- ResponseError{nil, errors.New(err)}
		}
	}()

	req.URL.Scheme = "http"
	req.URL.Host = s.Address

	resp, err := s.Transport.RoundTrip(req)
	if err == nil {
		ch <- ResponseError{resp, nil}
	} else {
		ch <- ResponseError{resp, err}
	}
}

func (s *Server) logPrefix(r *http.Request, tstart time.Time) string {
	now := time.Now()

	var rtt0, rtt1 int64

	// Calculate the total round trip time if front end inserted the atlantis-arrival-time
	// header before routing the request. The header is assumed to be created by calling
	// time.Now().UnixNano() or equivalent.
	arr, err := strconv.ParseInt(r.Header.Get("atlantis-arrival-time"), 10, 64)
	if err != nil {
		rtt0 = now.UnixNano() - arr
	}

	rtt1 = now.UnixNano() - tstart.UnixNano()

	// Log prefix includes server address, request source and URI, and round trip times.
	return fmt.Sprintf("[server %s][request %s|%s][rtt %d|%d]", s.Address, r.RemoteAddr,
		r.URL, rtt0, rtt1)
}

func (s *Server) Handle(w http.ResponseWriter, r *http.Request, tout time.Duration) {
	s.Metrics.RequestStart()
	defer s.Metrics.RequestDone()

	// X-Forwarded-For; we are a proxy.
	ip := strings.Split(r.RemoteAddr, ":")[0]
	r.Header.Add("X-Forwarded-For", ip)

	resErrCh := make(chan ResponseError)
	tstart := time.Now()
	go s.RoundTrip(r, resErrCh)

	select {
	case resErr := <-resErrCh:
		if resErr.Error == nil {
			logger.Printf("%s %d", s.logPrefix(r, tstart), resErr.Response.StatusCode)
			defer resErr.Response.Body.Close()
			for hdr, vals := range resErr.Response.Header {
				for _, val := range vals {
					w.Header().Add(hdr, val)
				}
			}
			w.WriteHeader(resErr.Response.StatusCode)
			_, err := s.copier.Copy(w, resErr.Response.Body)
			if err != nil {
				logger.Errorf("%s %s", s.logPrefix(r, tstart), err)
			}
		} else {
			logger.Errorf("%s %s", s.logPrefix(r, tstart), resErr.Error)
			http.Error(w, "Bad Gateway", http.StatusBadGateway)
		}
	case <-time.After(tout):
		s.Transport.CancelRequest(r)
		logger.Printf("%s timeout", s.logPrefix(r, tstart))
		http.Error(w, "Gateway Timeout", http.StatusGatewayTimeout)
	}
}

func (s *Server) CheckStatus(tout time.Duration) {
	r, _ := http.NewRequest("GET", "http://"+s.Address+"/healthz", nil)

	resErrCh := make(chan ResponseError)
	tstart := time.Now()
	go s.RoundTrip(r, resErrCh)

	select {
	case resErr := <-resErrCh:
		if resErr.Error == nil {
			logger.Printf("%s %d", s.logPrefix(r, tstart), resErr.Response.StatusCode)
			defer resErr.Response.Body.Close()
			s.Status.ParseAndSet(resErr.Response)
		} else {
			logger.Errorf("%s %s", s.logPrefix(r, tstart), resErr.Error)
			s.Status.Set(StatusCritical)
		}
	case <-time.After(tout):
		logger.Errorf("%s timeout", s.logPrefix(r, tstart))
		s.Status.Set(StatusCritical)
	}
}

func (s *Server) Cost(accept string) uint32 {
	return s.Status.Cost(accept) + s.Metrics.Cost()
}

func (s *Server) Shutdown() {
	s.copier.Shutdown()
}
<|endoftext|>


<commit_before>package heka_clever_plugins

import (
	"encoding/json"
	"fmt"
	"github.com/inconshreveable/go-keen"
	"github.com/mozilla-services/heka/pipeline"
)

type KeenOutput struct {
	client *keen.Client
}

type KeenOutputConfig struct {
	ApiKey       string `toml:"api_key"`
	ProjectToken string `toml:"project_token"`
}

func (ko *KeenOutput) ConfigStruct() interface{} {
	return &KeenOutputConfig{}
}

func (ko *KeenOutput) Init(rawConf interface{}) error {
	config := rawConf.(*KeenOutputConfig)
	ko.client = &keen.Client{ApiKey: config.ApiKey, ProjectToken: config.ProjectToken}
	return nil
}

func (ko *KeenOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
	var (
		// e    error
		pack *pipeline.PipelinePack
	)

	for pack = range or.InChan() {
		value, ok := pack.Message.GetFieldValue("Data")
		event := make(map[string]interface{})
		err = json.Unmarshal([]byte(value.(string)), &event)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%#v %#v\n", ok, event)
		err = ko.client.AddEvent("job-finished", event)
		if err != nil {
			panic(err)
		}
		pack.Recycle()
	}
	return
}

func init() {
	pipeline.RegisterPlugin("KeenOutput", func() interface{} {
		return new(KeenOutput)
	})
}
<commit_msg>Adds error handling/logging for keen-output plugin<commit_after>package heka_clever_plugins

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/inconshreveable/go-keen"
	"github.com/mozilla-services/heka/pipeline"
)

type KeenOutput struct {
	client *keen.Client
}

type KeenOutputConfig struct {
	ApiKey       string `toml:"api_key"`
	ProjectToken string `toml:"project_token"`
}

func (ko *KeenOutput) ConfigStruct() interface{} {
	return &KeenOutputConfig{}
}

func (ko *KeenOutput) Init(rawConf interface{}) error {
	config := rawConf.(*KeenOutputConfig)
	ko.client = &keen.Client{ApiKey: config.ApiKey, ProjectToken: config.ProjectToken}
	return nil
}

func (ko *KeenOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
	var (
		err  error
		pack *pipeline.PipelinePack
	)

	for pack = range or.InChan() {
		value, ok := pack.Message.GetFieldValue("Data")
		if !ok {
			return errors.New("Could not get field value 'Data'")
		}

		event := make(map[string]interface{})
		err = json.Unmarshal([]byte(value.(string)), &event)
		if err != nil {
			or.LogError(err)
		}
		err = ko.client.AddEvent("job-finished", event)
		if err != nil {
			or.LogError(err)
		}
		pack.Recycle()
	}
	return nil
}

func init() {
	pipeline.RegisterPlugin("KeenOutput", func() interface{} {
		return new(KeenOutput)
	})
}
<|endoftext|>