<commit_before>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package ini provides INI file read and write functionality in Go.
package ini

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"runtime"
)

const (
	// Name for default section. You can use this constant or the string literal.
	// In most of cases, an empty string is all you need to access the section.
	DEFAULT_SECTION = "DEFAULT"

	// Maximum allowed depth when recursively substituing variable names.
	_DEPTH_VALUES = 99
	_VERSION      = "1.36.0"
)

// Version returns current package version literal.
func Version() string {
	return _VERSION
}

var (
	// Delimiter to determine or compose a new line.
	// This variable will be changed to "\r\n" automatically on Windows
	// at package init time.
	LineBreak = "\n"

	// Variable regexp pattern: %(variable)s
	varPattern = regexp.MustCompile(`%\(([^\)]+)\)s`)

	// Indicate whether to align "=" sign with spaces to produce pretty output
	// or reduce all possible spaces for compact format.
	PrettyFormat = true

	// Place spaces around "=" sign even when PrettyFormat is false
	PrettyEqual = false

	// Explicitly write DEFAULT section header
	DefaultHeader = false

	// Indicate whether to put a line between sections
	PrettySection = true
)

func init() {
	if runtime.GOOS == "windows" {
		LineBreak = "\r\n"
	}
}

func inSlice(str string, s []string) bool {
	for _, v := range s {
		if str == v {
			return true
		}
	}
	return false
}

// dataSource is an interface that returns object which can be read and closed.
type dataSource interface {
	ReadCloser() (io.ReadCloser, error)
}

// sourceFile represents an object that contains content on the local file system.
type sourceFile struct {
	name string
}

func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) {
	return os.Open(s.name)
}

// sourceData represents an object that contains content in memory.
type sourceData struct {
	data []byte
}

func (s *sourceData) ReadCloser() (io.ReadCloser, error) {
	return ioutil.NopCloser(bytes.NewReader(s.data)), nil
}

// sourceReadCloser represents an input stream with Close method.
type sourceReadCloser struct {
	reader io.ReadCloser
}

func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) {
	return s.reader, nil
}

func parseDataSource(source interface{}) (dataSource, error) {
	switch s := source.(type) {
	case string:
		return sourceFile{s}, nil
	case []byte:
		return &sourceData{s}, nil
	case io.ReadCloser:
		return &sourceReadCloser{s}, nil
	default:
		return nil, fmt.Errorf("error parsing data source: unknown type '%s'", s)
	}
}

type LoadOptions struct {
	// Loose indicates whether the parser should ignore nonexistent files or return error.
	Loose bool
	// Insensitive indicates whether the parser forces all section and key names to lowercase.
	Insensitive bool
	// IgnoreContinuation indicates whether to ignore continuation lines while parsing.
	IgnoreContinuation bool
	// IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value.
	IgnoreInlineComment bool
	// AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing.
	// This type of keys are mostly used in my.cnf.
	AllowBooleanKeys bool
	// AllowShadows indicates whether to keep track of keys with same name under same section.
	AllowShadows bool
	// AllowNestedValues indicates whether to allow AWS-like nested values.
	// Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values
	AllowNestedValues bool
	// AllowPythonMultilineValues indicates whether to allow Python-like multi-line values.
	// Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
	// Relevant quote:  Values can also span multiple lines, as long as they are indented deeper
	// than the first line of the value.
	AllowPythonMultilineValues bool
	// SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value.
	// Docs: https://docs.python.org/2/library/configparser.html
	// Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names.
	// In the latter case, they need to be preceded by a whitespace character to be recognized as a comment.
	SpaceBeforeInlineComment bool
	// UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format
	// when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value"
	UnescapeValueDoubleQuotes bool
	// UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format
	// when value is NOT surrounded by any quotes.
	// Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all.
	UnescapeValueCommentSymbols bool
	// Some INI formats allow group blocks that store a block of raw content that doesn't otherwise
	// conform to key/value pairs. Specify the names of those blocks here.
	UnparseableSections []string
}

func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) {
	sources := make([]dataSource, len(others)+1)
	sources[0], err = parseDataSource(source)
	if err != nil {
		return nil, err
	}
	for i := range others {
		sources[i+1], err = parseDataSource(others[i])
		if err != nil {
			return nil, err
		}
	}
	f := newFile(sources, opts)
	if err = f.Reload(); err != nil {
		return nil, err
	}
	return f, nil
}

// Load loads and parses from INI data sources.
// Arguments can be mixed of file name with string type, or raw data in []byte.
// It will return error if list contains nonexistent files.
func Load(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{}, source, others...)
}

// LooseLoad has exactly same functionality as Load function
// except it ignores nonexistent files instead of returning error.
func LooseLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{Loose: true}, source, others...)
}

// InsensitiveLoad has exactly same functionality as Load function
// except it forces all section and key names to be lowercased.
func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{Insensitive: true}, source, others...)
}

// InsensitiveLoad has exactly same functionality as Load function
// except it allows have shadow keys.
func ShadowLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{AllowShadows: true}, source, others...)
}
<commit_msg>Bump version<commit_after>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package ini provides INI file read and write functionality in Go.
package ini

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"runtime"
)

const (
	// Name for default section. You can use this constant or the string literal.
	// In most of cases, an empty string is all you need to access the section.
	DEFAULT_SECTION = "DEFAULT"

	// Maximum allowed depth when recursively substituing variable names.
	_DEPTH_VALUES = 99
	_VERSION      = "1.37.0"
)

// Version returns current package version literal.
func Version() string {
	return _VERSION
}

var (
	// Delimiter to determine or compose a new line.
	// This variable will be changed to "\r\n" automatically on Windows
	// at package init time.
	LineBreak = "\n"

	// Variable regexp pattern: %(variable)s
	varPattern = regexp.MustCompile(`%\(([^\)]+)\)s`)

	// Indicate whether to align "=" sign with spaces to produce pretty output
	// or reduce all possible spaces for compact format.
	PrettyFormat = true

	// Place spaces around "=" sign even when PrettyFormat is false
	PrettyEqual = false

	// Explicitly write DEFAULT section header
	DefaultHeader = false

	// Indicate whether to put a line between sections
	PrettySection = true
)

func init() {
	if runtime.GOOS == "windows" {
		LineBreak = "\r\n"
	}
}

func inSlice(str string, s []string) bool {
	for _, v := range s {
		if str == v {
			return true
		}
	}
	return false
}

// dataSource is an interface that returns object which can be read and closed.
type dataSource interface {
	ReadCloser() (io.ReadCloser, error)
}

// sourceFile represents an object that contains content on the local file system.
type sourceFile struct {
	name string
}

func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) {
	return os.Open(s.name)
}

// sourceData represents an object that contains content in memory.
type sourceData struct {
	data []byte
}

func (s *sourceData) ReadCloser() (io.ReadCloser, error) {
	return ioutil.NopCloser(bytes.NewReader(s.data)), nil
}

// sourceReadCloser represents an input stream with Close method.
type sourceReadCloser struct {
	reader io.ReadCloser
}

func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) {
	return s.reader, nil
}

func parseDataSource(source interface{}) (dataSource, error) {
	switch s := source.(type) {
	case string:
		return sourceFile{s}, nil
	case []byte:
		return &sourceData{s}, nil
	case io.ReadCloser:
		return &sourceReadCloser{s}, nil
	default:
		return nil, fmt.Errorf("error parsing data source: unknown type '%s'", s)
	}
}

type LoadOptions struct {
	// Loose indicates whether the parser should ignore nonexistent files or return error.
	Loose bool
	// Insensitive indicates whether the parser forces all section and key names to lowercase.
	Insensitive bool
	// IgnoreContinuation indicates whether to ignore continuation lines while parsing.
	IgnoreContinuation bool
	// IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value.
	IgnoreInlineComment bool
	// AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing.
	// This type of keys are mostly used in my.cnf.
	AllowBooleanKeys bool
	// AllowShadows indicates whether to keep track of keys with same name under same section.
	AllowShadows bool
	// AllowNestedValues indicates whether to allow AWS-like nested values.
	// Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values
	AllowNestedValues bool
	// AllowPythonMultilineValues indicates whether to allow Python-like multi-line values.
	// Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
	// Relevant quote:  Values can also span multiple lines, as long as they are indented deeper
	// than the first line of the value.
	AllowPythonMultilineValues bool
	// SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value.
	// Docs: https://docs.python.org/2/library/configparser.html
	// Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names.
	// In the latter case, they need to be preceded by a whitespace character to be recognized as a comment.
	SpaceBeforeInlineComment bool
	// UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format
	// when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value"
	UnescapeValueDoubleQuotes bool
	// UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format
	// when value is NOT surrounded by any quotes.
	// Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all.
	UnescapeValueCommentSymbols bool
	// Some INI formats allow group blocks that store a block of raw content that doesn't otherwise
	// conform to key/value pairs. Specify the names of those blocks here.
	UnparseableSections []string
}

func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) {
	sources := make([]dataSource, len(others)+1)
	sources[0], err = parseDataSource(source)
	if err != nil {
		return nil, err
	}
	for i := range others {
		sources[i+1], err = parseDataSource(others[i])
		if err != nil {
			return nil, err
		}
	}
	f := newFile(sources, opts)
	if err = f.Reload(); err != nil {
		return nil, err
	}
	return f, nil
}

// Load loads and parses from INI data sources.
// Arguments can be mixed of file name with string type, or raw data in []byte.
// It will return error if list contains nonexistent files.
func Load(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{}, source, others...)
}

// LooseLoad has exactly same functionality as Load function
// except it ignores nonexistent files instead of returning error.
func LooseLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{Loose: true}, source, others...)
}

// InsensitiveLoad has exactly same functionality as Load function
// except it forces all section and key names to be lowercased.
func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{Insensitive: true}, source, others...)
}

// InsensitiveLoad has exactly same functionality as Load function
// except it allows have shadow keys.
func ShadowLoad(source interface{}, others ...interface{}) (*File, error) {
	return LoadSources(LoadOptions{AllowShadows: true}, source, others...)
}
<|endoftext|>


<commit_before><commit_msg>Remove redundant slice<commit_after><|endoftext|>


<commit_before>package main

import (
	"image/color"

	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/ora"
)

type terrain struct {
	Base, Top minecraft.Block
	TopLevel  uint8
}

var (
	terrainColours = color.Palette{
		color.RGBA{255, 255, 0, 255},   // Yellow - Sand
		color.RGBA{0, 255, 0, 255},     // Green - Grass
		color.RGBA{87, 59, 12, 255},    // Brown - Dirt
		color.RGBA{255, 128, 0, 255},   // Orange - Farm
		color.RGBA{127, 127, 127, 255}, // Grey - Stone
		color.RGBA{255, 255, 255, 255}, // White - Snow
	}
	terrainBlocks = []terrain{
		{minecaft.Block{ID: 24, Data: 2}, minecaft.Block{ID: 12}, 5}, // Sandstone - Sand
		{minecraft.Block{ID: 3}, minecaft.Block{2}, 1},               // Dirt - Grass
		{minecaft.Block{ID: 3}, minecaft.Block{ID: 3}, 0},            // Dirt - Dirt
		{minecaft.Block{ID: 3}, minecaft.Block{ID: 60, Data: 7}, 1},  // Dirt - Farmland
		{minecaft.Block{ID: 1}, minecaft.Block{ID: 1}, 0},            // Stone - Stone
		{minecaft.Block{ID: 1}, minecaft.Block{ID: 50}, 3},           // Stone - Snow
	}
)

func buildMap(o *ora.ORA, c chan paint) {
	defer close(c)
}
<commit_msg>Building initial terrain<commit_after>package main

import (
	"fmt"
	"image"
	"image/color"
	"image/draw"

	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/minecraft/nbt"
	"github.com/MJKWoolnough/ora"
)

type terrain struct {
	Base, Top minecraft.Block
	TopLevel  uint8
}

var (
	terrainColours = color.Palette{
		color.RGBA{},
		color.RGBA{255, 255, 0, 255},   // Yellow - Sand
		color.RGBA{0, 255, 0, 255},     // Green - Grass
		color.RGBA{87, 59, 12, 255},    // Brown - Dirt
		color.RGBA{255, 128, 0, 255},   // Orange - Farm
		color.RGBA{128, 128, 128, 255}, // Grey - Stone
		color.RGBA{255, 255, 255, 255}, // White - Snow
	}
	terrainBlocks = []terrain{
		{},
		{minecraft.Block{ID: 24, Data: 2}, minecraft.Block{ID: 12}, 5}, // Sandstone - Sand
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 2}, 1},            // Dirt - Grass
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 3}, 0},            // Dirt - Dirt
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 60, Data: 7}, 1},  // Dirt - Farmland
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 1}, 0},            // Stone - Stone
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 50}, 3},           // Stone - Snow
	}
)

func modeTerrain(p *image.Paletted) uint8 {
	b := p.Bounds()
	var modeMap [7]uint16
	most := uint16(0)
	mode := uint8(0)
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			pos := p.ColorIndexAt(i, j)
			modeMap[pos]++
			if m := modeMap[pos]; m > most {
				most = m
				mode = pos
			}
		}
	}
	return mode
}

func meanHeight(g *image.Gray) uint8 {
	b := g.Bounds()
	var total int64
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			total += int64(g.GrayAt(i, j).Y)
		}
	}
	return uint8(total / int64((b.Dx() * b.Dy())))
}

type chunkCache struct {
	mem   *minecraft.MemPath
	level *minecraft.Level
	clear nbt.Tag
	cache map[uint16]nbt.Tag
}

func newCache() *chunkCache {
	mem := minecraft.NewMemPath()
	l, _ := minecraft.NewLevel(mem)

	bedrock := minecraft.Block{ID: 7}

	l.SetBlock(0, 0, 0, minecraft.Block{})
	l.Save()
	l.Close()
	clearChunk, _ := mem.GetChunk(0, 0)

	for i := int32(-16); i < 16; i++ {
		for j := int32(0); j < 255; j++ {
			for k := int32(-16); j < 16; j++ {
				l.SetBlock(i, j, k, bedrock)
			}
		}
	}
	l.Save()
	l.Close()
	mem.SetChunk(clearChunk)
	return &chunkCache{
		mem,
		l,
		clearChunk,
		make(map[uint16]nbt.Tag),
	}
}

func (c *chunkCache) getFromCache(x, z int32, terrain uint8, height int32) nbt.Tag {
	cacheID := uint16(terrain)<<8 | uint16(height)
	chunk, ok := c.cache[cacheID]
	if !ok {
		b := terrainBlocks[terrain].Base
		for j := int32(0); j < height; j++ {
			for i := int32(0); i < 16; i++ {
				for k := int32(0); k < 16; k++ {
					c.level.SetBlock(i, j, k, b)
				}
			}
		}
		c.level.Save()
		c.level.Close()
		chunk, _ = c.mem.GetChunk(0, 0)
		c.mem.SetChunk(c.clear)
		c.cache[cacheID] = chunk
	}
	ld := chunk.Data().(nbt.Compound).Get("Level").Data().(nbt.Compound)
	ld.Set(nbt.NewTag("xPos", nbt.Int(x)))
	ld.Set(nbt.NewTag("zPos", nbt.Int(z)))
	return chunk
}

func buildMap(o *ora.ORA, c chan paint, m chan string) {
	defer close(c)
	terrain := o.Layer("terrain")
	height := o.Layer("height")
	sTerrain := image.NewPaletted(o.Bounds(), terrainColours)
	terrainI, err := terrain.Image()
	if err != nil {
		c <- paint{Err: err}
		return
	}
	draw.Draw(sTerrain, image.Rect(terrain.X, terrain.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), terrainI, image.Point{}, draw.Src)
	terrainI = nil
	sHeight := image.NewGray(o.Bounds())
	heightI, err := height.Image()
	if err != nil {
		c <- paint{Err: err}
		return
	}
	draw.Draw(sHeight, image.Rect(height.X, height.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), heightI, image.Point{}, draw.Src)
	heightI = nil
	p, err := minecraft.NewFilePath("./test/")
	if err != nil {
		c <- paint{Err: err}
		return
	}

	m <- "Building Terrain"
	buildTerrain(p, sTerrain, sHeight, c)
	m <- "Building Height Map"
}

func buildTerrain(mpath minecraft.Path, terrain *image.Paletted, height *image.Gray, c chan paint) {
	cc := newCache()
	b := terrain.Bounds()
	for j := 0; j < b.Max.Y; j += 16 {
		chunkZ := int32(j >> 4)
		for i := 0; i < b.Max.X; i += 16 {
			chunkX := int32(i >> 4)
			p := terrain.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Paletted)
			g := height.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Gray)
			t := modeTerrain(p)
			h := int32(meanHeight(g))
			mpath.SetChunk(cc.getFromCache(chunkX, chunkZ, t, h))
			fmt.Print(t)
			c <- paint{
				terrainColours[t],
				chunkX, chunkZ,
				nil,
			}
		}
		fmt.Println()
	}
}

func buildHeight(level *minecraft.Level, terrain *image.Paletted, height *image.Gray, c chan paint) {

}
<|endoftext|>


<commit_before><commit_msg>Defaulting to use the JSON SeverityFormatter for logrus logging<commit_after><|endoftext|>


<commit_before><commit_msg>Add key value to error message<commit_after><|endoftext|>


<commit_before><commit_msg>Min function added<commit_after><|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/cydev/stun"
)

const (
	version = "0.1"
)

func wrapLogrus(f func(c *cli.Context) error) func(c *cli.Context) error {
	return func(c *cli.Context) error {
		err := f(c)
		if err != nil {
			logrus.Errorln("discover error:", err)
		}
		return err
	}
}

func discover(c *cli.Context) error {
	conn, err := net.Dial("udp", stun.Normalize(c.String("server")))
	if err != nil {
		return err
	}
	m := stun.AcquireMessage()
	m.Type = stun.MessageType{
		Method: stun.MethodBinding,
		Class:  stun.ClassRequest,
	}
	m.TransactionID = stun.NewTransactionID()
	m.AddSoftware("cydev/stun alpha")
	m = stun.AcquireFields(stun.Message{
		TransactionID: stun.NewTransactionID(),
		Type: stun.MessageType{
			Method: stun.MethodBinding,
			Class: stun.ClassRequest,
		},
	})
	m.WriteHeader()
	timeout := 100 * time.Millisecond
	for i := 0; i < 9; i++ {
		_, err := m.WriteTo(conn)
		if err != nil {
			return err
		}
		if err = conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
			return err
		}
		if timeout < 1600*time.Millisecond {
			timeout *= 2
		}
		var (
			ip   net.IP
			port int
		)
		if err == nil {
			mRec := stun.AcquireMessage()
			if _, err = mRec.ReadFrom(conn); err != nil {
				return err
			}
			if mRec.TransactionID != m.TransactionID {
				return errors.New("TransactionID missmatch")
			}
			ip, port, err = mRec.GetXORMappedAddress()
			if err != nil {
				return err
			}
			fmt.Println(ip, port)
			stun.ReleaseMessage(mRec)
			break
		} else {
			if !err.(net.Error).Timeout() {
				return err
			}
		}
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "stun"
	app.Usage = "command line client for STUN"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "server",
			Value:       "ci.cydev.ru",
			Usage:       "STUN server address",
		},
	}
	app.Action = wrapLogrus(discover)
	app.Version = version
	app.Run(os.Args)
}
<commit_msg>client: fix bug in message init<commit_after>package main

import (
	"errors"
	"fmt"
	"net"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/cydev/stun"
)

const (
	version = "0.1"
)

func wrapLogrus(f func(c *cli.Context) error) func(c *cli.Context) error {
	return func(c *cli.Context) error {
		err := f(c)
		if err != nil {
			logrus.Errorln("discover error:", err)
		}
		return err
	}
}

func discover(c *cli.Context) error {
	conn, err := net.Dial("udp", stun.Normalize(c.String("server")))
	if err != nil {
		return err
	}
	m := stun.AcquireFields(stun.Message{
		TransactionID: stun.NewTransactionID(),
		Type: stun.MessageType{
			Method: stun.MethodBinding,
			Class: stun.ClassRequest,
		},
	})
	m.AddSoftware("cydev/stun alpha")
	m.WriteHeader()
	timeout := 100 * time.Millisecond
	for i := 0; i < 9; i++ {
		_, err := m.WriteTo(conn)
		if err != nil {
			return err
		}
		if err = conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
			return err
		}
		if timeout < 1600*time.Millisecond {
			timeout *= 2
		}
		var (
			ip   net.IP
			port int
		)
		if err == nil {
			mRec := stun.AcquireMessage()
			if _, err = mRec.ReadFrom(conn); err != nil {
				return err
			}
			if mRec.TransactionID != m.TransactionID {
				return errors.New("TransactionID missmatch")
			}
			ip, port, err = mRec.GetXORMappedAddress()
			if err != nil {
				return err
			}
			fmt.Println(ip, port)
			stun.ReleaseMessage(mRec)
			break
		} else {
			if !err.(net.Error).Timeout() {
				return err
			}
		}
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "stun"
	app.Usage = "command line client for STUN"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "server",
			Value:       "ci.cydev.ru",
			Usage:       "STUN server address",
		},
	}
	app.Action = wrapLogrus(discover)
	app.Version = version
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package xbps

import (
	"bytes"
	"exec"
	"fmt"
	. "github.com/str1ngs/go-ansi/color"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var (
	xbpsBin  = "xbps-bin.static -c %s -Ayr %s %s %s"
	xbpsSrc  = "xbps-src -C -m %s %s %s"
	mkIdx    = "xbps-src make-repoidx %s"
	binCache = "/home/strings/bincache"
	srcPath  = "/home/strings/github/vanilla/srcpkgs/"
	lfmt     = "%-10.10s %s"
	pkgfmt   = "%s-%s.x86_64.xbps"
	bufout   = new(bytes.Buffer)
)

func HandleError(err os.Error) {
	log.Print(err)
	log.Print(strings.Repeat("*", 80-len(log.Prefix())))
	io.Copy(os.Stderr, bufout)
	log.Print(strings.Repeat("*", 80-len(log.Prefix())))
}

func Seed(md *MasterDir) (err os.Error) {
	err = Install("base-chroot", md)
	if err != nil {
		return err
	}
	return
}

func Install(pack string, md *MasterDir) (err os.Error) {
	if IsInstalled(pack, md) {
		log.Printf(lfmt, "skip", pack)
		return
	}
	log.Printf(lfmt, "install", pack)
	err = NewCommand(xbpsBin, binCache, md.TargetPath, "install", pack).Run()
	if err != nil {
		return
	}
	return
}

func RmPackFile(pkg string) (err os.Error) {
	tmpl, err := FindTemplate(pkg, srcPath)
	if err != nil {
		return err
	}
	pkgname := fmt.Sprintf(pkgfmt, pkg, tmpl.Version)
	glob := fmt.Sprintf("%s/pkg-binpkgs/*/%s-*", hostPath, pkgname)
	files, err := filepath.Glob(glob)
	if err != nil {
		return
	}
	switch len(files) {
	case 0:
		log.Printf(lfmt, "no package", pkg)
		return
	case 1:
		log.Printf(lfmt, "remove", unExpand(files[0]))
		err = os.Remove(files[0])
		return err
	default:
		for _, f := range files {
			log.Printf(lfmt, "packages", unExpand(f))
		}
		return fmt.Errorf("expected one package to remove found %v", len(files))
	}
	panic("Should not reach here")
}

func Build(tmpl string, md *MasterDir) (err os.Error) {
	depends, err := GetAllDepends(tmpl)
	if err != nil {
		return err
	}
	log.Printf(lfmt, "depends", tmpl)
	for _, d := range depends {
		err = Install(d, md)
		if err != nil {
			return err
		}
	}
	log.Printf(lfmt, "build", tmpl)
	cmd := NewCommand(xbpsSrc, md.TargetPath, "install", tmpl)
	err = cmd.Run()
	if err != nil {
		return
	}
	return
}

func Package(tmpl string, md *MasterDir) (err os.Error) {
	if err = RmPackFile(tmpl); err != nil {
		return
	}
	log.Printf(lfmt, "package", tmpl)
	err = NewCommand(xbpsSrc, md.TargetPath, "build-pkg", tmpl).Run()
	if err != nil {
		return
	}

	log.Printf(lfmt, "index", tmpl)
	err = NewCommand(mkIdx, tmpl).Run()
	if err != nil {
		return
	}
	err = Clean(md)
	if err != nil {
		return
	}
	log.Printf(lfmt, "done", tmpl)
	return
}

func Clean(md *MasterDir) (err os.Error) {
	log.Printf(lfmt, "clean", md.TargetPath)
	if err = os.RemoveAll(md.TargetPath); err != nil {
		return
	}
	return
}

func IsInstalled(pack string, md *MasterDir) bool {
	cmd := NewCommand(xbpsBin, binCache, md.TargetPath, "show", TrimOp(pack))
	err := cmd.Run()
	if err != nil {
		return false
	}
	return true
}

// Creates a new exec.Cmd from a Printf style format
func NewCommand(format string, i ...interface{}) (cmd *exec.Cmd) {
	args := strings.Split(fmt.Sprintf(format, i...), " ")
	cmd = exec.Command(args[0], args[1:]...)
	bufout.Reset()
	bufout.WriteString(fmt.Sprintf("%s\n", Yellow(strings.Join(cmd.Args, " "))))
	cmd.Stdout = bufout
	cmd.Stderr = bufout
	return
}

// Get all dependancies for a template
func GetAllDepends(tmpl string) (depends []string, err os.Error) {
	rd, err := GetDepends("run", tmpl)
	if err != nil {
		return
	}
	bd, err := GetDepends("build", tmpl)
	if err != nil {
		return
	}
	depends = append(rd, bd...)
	return
}

// Get run/buil/all depends for a template
func GetDepends(kind, tmpl string) (depends []string, err os.Error) {
	b, err := Sh([]string{kind, tmpl})
	if err != nil {
		return
	}
	if len(b) == 0 {
		return
	}
	if b[0] == ' ' {
		b = b[1:]
	}
	depends = strings.Split(string(b), " ")
	TrimOps(depends)
	return
}

// Checks each build dependency and makes sure it is unique
func ChkDupDepends(name string) (required []string, err os.Error) {
	fmt.Printf("**** Checking Depends For (%s) ****\n", name)
	var (
		mreq    = make(map[string]bool)
		visited = make(map[string]string)
		depends = []string{}
	)
	if isSubTmpl(name) {
		depends, err = GetDepends("run", name)
	} else {
		depends, err = GetDepends("build", name)
	}
	if err != nil {
		return
	}
	// walk each depend
	for _, d := range depends {
		var c []string
		c, err = GetDepends("run", d)
		if err != nil {
			return
		}
		// walk each sub depend and mark it as visited
		for _, sc := range c {
			visited[sc] = d
			// if this sub depend is in required remove it
			if mreq[sc] {
				fmt.Printf("%-20.20s provided by %s\n", sc, visited[sc])
				mreq[sc] = false, false
			}
		}
		// if we visited this depend before it is not required
		_, exist := visited[d]
		if !exist {
			mreq[d] = true
		} else {
			fmt.Printf("%-20.20s provided by %s\n", d, visited[d])
		}
	}
	for r, _ := range mreq {
		required = append(required, r)
	}
	fmt.Printf("\n**** (%s) Template Build Depends ****\n", name)
	printSlice(depends)
	fmt.Printf("\n**** (%s) Actual Build Depend ****\n", name)
	printSlice(required)
	fmt.Println()
	return
}

func TrimOp(pack string) string {
	switch len(strings.Split(pack, ">")) {
	case 2:
		return strings.Split(pack, ">")[0]
	case 1:
		return strings.Split(pack, "<")[0]
	}
	panic(fmt.Sprintf("%s: Could not trim operator", pack))
}

func TrimOps(depends []string) {
	for i, d := range depends {
		depends[i] = TrimOp(d)
	}
}

func printSlice(slice []string) {
	for _, s := range slice {
		fmt.Println(s)
	}
}

func isSubTmpl(tmpl string) bool {
	file := fmt.Sprintf("%s/%s/%s.template", srcPath, tmpl, tmpl)
	return fileExists(file)
}

func Sh(args []string) (output []byte, err os.Error) {
	helper := "/usr/local/libexec/getdeps-helper"
	cmd := exec.Command(helper, args...)
	output, err = cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s %s", helper, err, output)
	}
	return
}
<commit_msg>remove ansi-color only using it in one place.<commit_after>package xbps

import (
	"bytes"
	"exec"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var (
	xbpsBin  = "xbps-bin.static -c %s -Ayr %s %s %s"
	xbpsSrc  = "xbps-src -C -m %s %s %s"
	mkIdx    = "xbps-src make-repoidx %s"
	binCache = "/home/strings/bincache"
	srcPath  = "/home/strings/github/vanilla/srcpkgs/"
	lfmt     = "%-10.10s %s"
	pkgfmt   = "%s-%s.x86_64.xbps"
	bufout   = new(bytes.Buffer)
)

func HandleError(err os.Error) {
	log.Print(err)
	log.Print(strings.Repeat("*", 80-len(log.Prefix())))
	io.Copy(os.Stderr, bufout)
	log.Print(strings.Repeat("*", 80-len(log.Prefix())))
}

func Seed(md *MasterDir) (err os.Error) {
	err = Install("base-chroot", md)
	if err != nil {
		return err
	}
	return
}

func Install(pack string, md *MasterDir) (err os.Error) {
	if IsInstalled(pack, md) {
		log.Printf(lfmt, "skip", pack)
		return
	}
	log.Printf(lfmt, "install", pack)
	err = NewCommand(xbpsBin, binCache, md.TargetPath, "install", pack).Run()
	if err != nil {
		return
	}
	return
}

func RmPackFile(pkg string) (err os.Error) {
	tmpl, err := FindTemplate(pkg, srcPath)
	if err != nil {
		return err
	}
	pkgname := fmt.Sprintf(pkgfmt, pkg, tmpl.Version)
	glob := fmt.Sprintf("%s/pkg-binpkgs/*/%s-*", hostPath, pkgname)
	files, err := filepath.Glob(glob)
	if err != nil {
		return
	}
	switch len(files) {
	case 0:
		log.Printf(lfmt, "no package", pkg)
		return
	case 1:
		log.Printf(lfmt, "remove", unExpand(files[0]))
		err = os.Remove(files[0])
		return err
	default:
		for _, f := range files {
			log.Printf(lfmt, "packages", unExpand(f))
		}
		return fmt.Errorf("expected one package to remove found %v", len(files))
	}
	panic("Should not reach here")
}

func Build(tmpl string, md *MasterDir) (err os.Error) {
	depends, err := GetAllDepends(tmpl)
	if err != nil {
		return err
	}
	log.Printf(lfmt, "depends", tmpl)
	for _, d := range depends {
		err = Install(d, md)
		if err != nil {
			return err
		}
	}
	log.Printf(lfmt, "build", tmpl)
	cmd := NewCommand(xbpsSrc, md.TargetPath, "install", tmpl)
	err = cmd.Run()
	if err != nil {
		return
	}
	return
}

func Package(tmpl string, md *MasterDir) (err os.Error) {
	if err = RmPackFile(tmpl); err != nil {
		return
	}
	log.Printf(lfmt, "package", tmpl)
	err = NewCommand(xbpsSrc, md.TargetPath, "build-pkg", tmpl).Run()
	if err != nil {
		return
	}

	log.Printf(lfmt, "index", tmpl)
	err = NewCommand(mkIdx, tmpl).Run()
	if err != nil {
		return
	}
	err = Clean(md)
	if err != nil {
		return
	}
	log.Printf(lfmt, "done", tmpl)
	return
}

func Clean(md *MasterDir) (err os.Error) {
	log.Printf(lfmt, "clean", md.TargetPath)
	if err = os.RemoveAll(md.TargetPath); err != nil {
		return
	}
	return
}

func IsInstalled(pack string, md *MasterDir) bool {
	cmd := NewCommand(xbpsBin, binCache, md.TargetPath, "show", TrimOp(pack))
	err := cmd.Run()
	if err != nil {
		return false
	}
	return true
}

// Creates a new exec.Cmd from a Printf style format
func NewCommand(format string, i ...interface{}) (cmd *exec.Cmd) {
	args := strings.Split(fmt.Sprintf(format, i...), " ")
	cmd = exec.Command(args[0], args[1:]...)
	bufout.Reset()
	bufout.WriteString(fmt.Sprintf("%s\n", strings.Join(cmd.Args, " ")))
	cmd.Stdout = bufout
	cmd.Stderr = bufout
	return
}

// Get all dependancies for a template
func GetAllDepends(tmpl string) (depends []string, err os.Error) {
	rd, err := GetDepends("run", tmpl)
	if err != nil {
		return
	}
	bd, err := GetDepends("build", tmpl)
	if err != nil {
		return
	}
	depends = append(rd, bd...)
	return
}

// Get run/buil/all depends for a template
func GetDepends(kind, tmpl string) (depends []string, err os.Error) {
	b, err := Sh([]string{kind, tmpl})
	if err != nil {
		return
	}
	if len(b) == 0 {
		return
	}
	if b[0] == ' ' {
		b = b[1:]
	}
	depends = strings.Split(string(b), " ")
	TrimOps(depends)
	return
}

// Checks each build dependency and makes sure it is unique
func ChkDupDepends(name string) (required []string, err os.Error) {
	fmt.Printf("**** Checking Depends For (%s) ****\n", name)
	var (
		mreq    = make(map[string]bool)
		visited = make(map[string]string)
		depends = []string{}
	)
	if isSubTmpl(name) {
		depends, err = GetDepends("run", name)
	} else {
		depends, err = GetDepends("build", name)
	}
	if err != nil {
		return
	}
	// walk each depend
	for _, d := range depends {
		var c []string
		c, err = GetDepends("run", d)
		if err != nil {
			return
		}
		// walk each sub depend and mark it as visited
		for _, sc := range c {
			visited[sc] = d
			// if this sub depend is in required remove it
			if mreq[sc] {
				fmt.Printf("%-20.20s provided by %s\n", sc, visited[sc])
				mreq[sc] = false, false
			}
		}
		// if we visited this depend before it is not required
		_, exist := visited[d]
		if !exist {
			mreq[d] = true
		} else {
			fmt.Printf("%-20.20s provided by %s\n", d, visited[d])
		}
	}
	for r, _ := range mreq {
		required = append(required, r)
	}
	fmt.Printf("\n**** (%s) Template Build Depends ****\n", name)
	printSlice(depends)
	fmt.Printf("\n**** (%s) Actual Build Depend ****\n", name)
	printSlice(required)
	fmt.Println()
	return
}

func TrimOp(pack string) string {
	switch len(strings.Split(pack, ">")) {
	case 2:
		return strings.Split(pack, ">")[0]
	case 1:
		return strings.Split(pack, "<")[0]
	}
	panic(fmt.Sprintf("%s: Could not trim operator", pack))
}

func TrimOps(depends []string) {
	for i, d := range depends {
		depends[i] = TrimOp(d)
	}
}

func printSlice(slice []string) {
	for _, s := range slice {
		fmt.Println(s)
	}
}

func isSubTmpl(tmpl string) bool {
	file := fmt.Sprintf("%s/%s/%s.template", srcPath, tmpl, tmpl)
	return fileExists(file)
}

func Sh(args []string) (output []byte, err os.Error) {
	helper := "/usr/local/libexec/getdeps-helper"
	cmd := exec.Command(helper, args...)
	output, err = cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s %s", helper, err, output)
	}
	return
}
<|endoftext|>


<commit_before>package views

import (
	"github.com/flosch/pongo2"
	"lambda.sx/marcus/lambdago/models"
	"lambda.sx/marcus/lambdago/session"
	"lambda.sx/marcus/lambdago/settings"
	"lambda.sx/marcus/lambdago/sql"
	"mime"
	"net/http"
	"strings"
	"upper.io/db"
)

// Compile the templates on startup for a speed boost
var indexTpl = pongo2.Must(pongo2.FromFile("templates/index.html"))
var aboutTpl = pongo2.Must(pongo2.FromFile("templates/about.html"))
var fourohfourTpl = pongo2.Must(pongo2.FromFile("templates/404.html"))

func HandleIndex(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	rendered_index, err := indexTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_index
}

func HandleAbout(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	rendered_about, err := aboutTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_about
}

// Try to find a paste or image, or serve 404
func HandleDefault(r *http.Request, w http.ResponseWriter) (error, string) {
	url := strings.Split(r.URL.String(), "?")[0]
	if url != "" {
		url = url[1:] // Remove "/" from before request
	}
	url = strings.Split(url, ".")[0] //Ignore any extensions
	for _, ext := range allowedTypes {
		path := "uploads/" + url + "." + ext
		if fileExists(path) {
			mimetype := mime.TypeByExtension("." + ext)
			w.Header().Set("Content-Type", mimetype)
			w.Header().Set("Cache-Control", "public, max-age=259200")
			// Don't let browsers guess a mime type other than the one we say.
			// Prevents users from serving files with a different extension from their true type.
			w.Header().Set("X-Content-Type-Options", "nosniff")
			http.ServeFile(w, r, path)
			return nil, ""
		}
	}

	col, _ := sql.Connection().Collection("pastes")
	result := col.Find(db.Cond{"name": url})
	cnt, err := result.Count()
	if err == nil && cnt > 0 {
		var paste models.Paste
		result.One(&paste)
		return HandleViewPaste(r, w, paste.ContentJson)
	}

	user := session.GetUser(r, w)
	rendered_404, err := fourohfourTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	w.WriteHeader(404)
	return nil, rendered_404
}
<commit_msg>Give 304 if the client already has gotten an image before<commit_after>package views

import (
	"github.com/flosch/pongo2"
	"lambda.sx/marcus/lambdago/models"
	"lambda.sx/marcus/lambdago/session"
	"lambda.sx/marcus/lambdago/settings"
	"lambda.sx/marcus/lambdago/sql"
	"mime"
	"net/http"
	"strings"
	"upper.io/db"
)

// Compile the templates on startup for a speed boost
var indexTpl = pongo2.Must(pongo2.FromFile("templates/index.html"))
var aboutTpl = pongo2.Must(pongo2.FromFile("templates/about.html"))
var fourohfourTpl = pongo2.Must(pongo2.FromFile("templates/404.html"))

func HandleIndex(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	rendered_index, err := indexTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_index
}

func HandleAbout(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	rendered_about, err := aboutTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_about
}

// Try to find a paste or image, or serve 404
func HandleDefault(r *http.Request, w http.ResponseWriter) (error, string) {
	url := strings.Split(r.URL.String(), "?")[0]
	if url != "" {
		url = url[1:] // Remove "/" from before request
	}
	url = strings.Split(url, ".")[0] //Ignore any extensions
	for _, ext := range allowedTypes {
		path := "uploads/" + url + "." + ext
		if fileExists(path) {
			if r.Header.Get("If-Modified-Since") != "" { //They already have the image
				w.WriteHeader(304)
				return nil, ""
			}
			mimetype := mime.TypeByExtension("." + ext)
			w.Header().Set("Content-Type", mimetype)
			w.Header().Set("Cache-Control", "public, max-age=259200")
			// Don't let browsers guess a mime type other than the one we say.
			// Prevents users from serving files with a different extension from their true type.
			w.Header().Set("X-Content-Type-Options", "nosniff")
			http.ServeFile(w, r, path)
			return nil, ""
		}
	}

	col, _ := sql.Connection().Collection("pastes")
	result := col.Find(db.Cond{"name": url})
	cnt, err := result.Count()
	if err == nil && cnt > 0 {
		var paste models.Paste
		result.One(&paste)
		return HandleViewPaste(r, w, paste.ContentJson)
	}

	user := session.GetUser(r, w)
	rendered_404, err := fourohfourTpl.Execute(pongo2.Context{
		"user":  user,
		"nocdn": !settings.UseCDN,
	})
	if err != nil {
		return err, ""
	}
	w.WriteHeader(404)
	return nil, rendered_404
}
<|endoftext|>


<commit_before>package parcels

import (
	"bytes"
	"io"
)

type internalBytesParcel struct {
	content []byte
}

// ParcelFromBytes returns a new Parcel whose contents is the value of the []byte.
// Note that the contents the input []byte 'b' is copied, and the new Parcel
// is not attached to the original backing memory.
//
// Example
//
//      parcel := parcels.ParcelFromBytes( []byte{72, 69, 76, 76, 79} )
//
//      asBytes  := parcel.Bytes()
//      asReader := parcel.Reader()
//      asRunes  := parcel.Runes()
//      asString := parcel.String()
//
//	n, err := parcel.WriteTo(writer)
func ParcelFromBytes(b []byte) Parcel {

	content := append([]byte(nil), b...)

	parcel := internalBytesParcel{
		content:content,
	}

	return &parcel
}

func (parcel *internalBytesParcel) Bytes() []byte {
	return append([]byte(nil), parcel.content...)
}

func (parcel *internalBytesParcel) Reader() io.Reader {
	return bytes.NewReader(parcel.content)
}

func (parcel *internalBytesParcel) Runes() []rune {
	return []rune(string(parcel.content))
}

func (parcel *internalBytesParcel) String() string {
	return string(parcel.content)
}

func (parcel *internalBytesParcel) Then(fn func(Parcel)Parcel) Parcel {
	return fn(parcel)
}
<commit_msg>added WriteTo(w io.Writer) (int64, error) method to parcels.Parcel interface, and underlying implementations of parcels.Parcel, so that they fit the io.WriterTo interface.<commit_after>package parcels

import (
	"bytes"
	"io"
)

type internalBytesParcel struct {
	content []byte
}

// ParcelFromBytes returns a new Parcel whose contents is the value of the []byte.
// Note that the contents the input []byte 'b' is copied, and the new Parcel
// is not attached to the original backing memory.
//
// Example
//
//	parcel := parcels.ParcelFromBytes( []byte{72, 69, 76, 76, 79} )
//
//	asBytes  := parcel.Bytes()
//	asReader := parcel.Reader()
//	asRunes  := parcel.Runes()
//	asString := parcel.String()
//
//	n, err := parcel.WriteTo(writer)
func ParcelFromBytes(b []byte) Parcel {

	content := append([]byte(nil), b...)

	parcel := internalBytesParcel{
		content:content,
	}

	return &parcel
}

func (parcel *internalBytesParcel) Bytes() []byte {
	return append([]byte(nil), parcel.content...)
}

func (parcel *internalBytesParcel) Reader() io.Reader {
	return bytes.NewReader(parcel.content)
}

func (parcel *internalBytesParcel) Runes() []rune {
	return []rune(string(parcel.content))
}

func (parcel *internalBytesParcel) String() string {
	return string(parcel.content)
}

func (parcel *internalBytesParcel) Then(fn func(Parcel)Parcel) Parcel {
	return fn(parcel)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type config struct {
	InputSeparator  string   `json:"input-separator"`
	OutputSeparator string   `json:"output-separator"`
	SplitSeparator  string   `json:"split-separator"`
	Columns         []Column `json:"columns"`
}

type Column struct {
	From    int    `json:"from"`
	To      int    `json:"to"`
	Title   string `json:"title"`
	SplitBy string `json:"split-by"`
}

func printSlice(xs []string, sep string, out io.Writer) {
	l := len(xs)
	for i, e := range xs {
		fmt.Fprint(out, e)
		if (i + 1) < l {
			fmt.Print(sep)
		}
	}
	fmt.Fprintln(out)
}

func makeHeader(cols []Column) []string {
	target := make([]string, len(cols))

	for _, e := range cols {
		target[e.To] = e.Title
	}
	return target
}

func processRecord(record []string, cols []Column, splitter string, sep string) []string {

	target := make([]string, len(cols))
	for _, e := range cols {
		s := record[e.From]

		// replace splitter strings, if necessary
		if e.SplitBy != "" {
			s = strings.Replace(s, e.SplitBy, splitter, -1)
		}
		// escape string, if it contains separator
		if strings.Contains(s, sep) {
			s = "\"" + s + "\""
		}

		target[e.To] = s
	}
	return target
}

func main() {

	if len(os.Args) != 3 {
		fmt.Fprintf(os.Stderr, "Usage: %v config-file input-file\n", filepath.Base(os.Args[0]))
		os.Exit(1)
	}

	// read config
	var conf config
	f, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read config file: %v\n", err)
		os.Exit(1)
	}
	err = json.Unmarshal(f, &conf)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot parse config file: %v\n", err)
		os.Exit(1)
	}

	// set defaults, if not specified in configuration
	if conf.InputSeparator == "" {
		conf.InputSeparator = ";"
	}
	if conf.OutputSeparator == "" {
		conf.OutputSeparator = ";"
	}
	if conf.SplitSeparator == "" {
		conf.SplitSeparator = ";"
	}

	fmt.Println(conf)

	// process file
	fn := os.Args[2]

	infile, err := os.Open(fn)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot open input file: %v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	r := csv.NewReader(infile)
	r.Comma = ';' //utf8.DecodeRuneInString(conf.InputSeparator) // potentially discard characters // TODO Fixme
	r.LazyQuotes = true

	records, err := r.ReadAll()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read records: %v\n", err)
		os.Exit(1)
	}

	printSlice(makeHeader(conf.Columns), conf.OutputSeparator, os.Stdout)

	// process records
	for _, e := range records {
		printSlice(processRecord(e, conf.Columns, conf.SplitSeparator, conf.OutputSeparator),
			conf.OutputSeparator, os.Stdout)
	}
}
<commit_msg>Remote debugging output, 'fix' separator input<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type config struct {
	InputSeparator  string   `json:"input-separator"`
	OutputSeparator string   `json:"output-separator"`
	SplitSeparator  string   `json:"split-separator"`
	Columns         []Column `json:"columns"`
}

type Column struct {
	From    int    `json:"from"`
	To      int    `json:"to"`
	Title   string `json:"title"`
	SplitBy string `json:"split-by"`
}

func printSlice(xs []string, sep string, out io.Writer) {
	l := len(xs)
	for i, e := range xs {
		fmt.Fprint(out, e)
		if (i + 1) < l {
			fmt.Print(sep)
		}
	}
	fmt.Fprintln(out)
}

func makeHeader(cols []Column) []string {
	target := make([]string, len(cols))

	for _, e := range cols {
		target[e.To] = e.Title
	}
	return target
}

func processRecord(record []string, cols []Column, splitter string, sep string) []string {

	target := make([]string, len(cols))
	for _, e := range cols {
		s := record[e.From]

		// replace splitter strings, if necessary
		if e.SplitBy != "" {
			s = strings.Replace(s, e.SplitBy, splitter, -1)
		}
		// escape string, if it contains separator
		if strings.Contains(s, sep) {
			s = "\"" + s + "\""
		}

		target[e.To] = s
	}
	return target
}

func main() {

	if len(os.Args) != 3 {
		fmt.Fprintf(os.Stderr, "Usage: %v config-file input-file\n", filepath.Base(os.Args[0]))
		os.Exit(1)
	}

	// read config
	var conf config
	f, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read config file: %v\n", err)
		os.Exit(1)
	}
	err = json.Unmarshal(f, &conf)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot parse config file: %v\n", err)
		os.Exit(1)
	}

	// set defaults, if not specified in configuration
	if conf.InputSeparator == "" {
		conf.InputSeparator = ";"
	}
	if conf.OutputSeparator == "" {
		conf.OutputSeparator = ";"
	}
	if conf.SplitSeparator == "" {
		conf.SplitSeparator = ";"
	}

	// process file
	fn := os.Args[2]

	infile, err := os.Open(fn)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot open input file: %v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	r := csv.NewReader(infile)
	r.Comma = rune(conf.InputSeparator[0]) // TODO works only with 8-bits
	r.LazyQuotes = true

	records, err := r.ReadAll()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read records: %v\n", err)
		os.Exit(1)
	}

	printSlice(makeHeader(conf.Columns), conf.OutputSeparator, os.Stdout)

	// process records
	for _, e := range records {
		printSlice(processRecord(e, conf.Columns, conf.SplitSeparator, conf.OutputSeparator),
			conf.OutputSeparator, os.Stdout)
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"
)

var ErrCacheMiss = errors.New("cache: key is missing")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group        singleflight.Group
	hits, misses int64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	if item.Expiration >= 0 && item.Expiration < time.Second {
		item.Expiration = time.Hour
	} else if item.Expiration == -1 {
		item.Expiration = 0
	}

	object, err := item.object()
	if err != nil {
		return err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	err = cd.Redis.Set(item.Key, b, item.Expiration).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return err
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(key, object, false)
}

func (cd *Codec) get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		v, ok := cd.localCache.Get(key)
		if ok {
			b, ok := v.([]byte)
			if ok {
				atomic.AddInt64(&cd.hits, 1)
				return b, nil
			}
		}
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddInt64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) (interface{}, error) {
	if cd.localCache != nil {
		if err := cd.getItemFast(item); err == nil {
			return item.Object, nil
		}
	}
	return cd.group.Do(item.Key, func() (interface{}, error) {
		if err := cd.getItem(item); err == nil {
			return item.Object, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		cd.Set(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})

		err = cd.getItem(item)
		if err != nil {
			return nil, err
		}
		return item.Object, nil
	})
}

func (cd *Codec) getItem(item *Item) error {
	return cd._getItem(item, false)
}

func (cd *Codec) getItemFast(item *Item) error {
	return cd._getItem(item, true)
}

func (cd *Codec) _getItem(item *Item, onlyLocalCache bool) error {
	if item.Object != nil {
		return cd.get(item.Key, item.Object, onlyLocalCache)
	} else {
		return cd.get(item.Key, &item.Object, onlyLocalCache)
	}
}

func (cd *Codec) Delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Codec) Hits() int {
	return int(atomic.LoadInt64(&cd.hits))
}

func (cd *Codec) Misses() int {
	return int(atomic.LoadInt64(&cd.misses))
}
<commit_msg>Extract item.exp<commit_after>package cache

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"
)

var ErrCacheMiss = errors.New("cache: key is missing")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group        singleflight.Group
	hits, misses int64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	object, err := item.object()
	if err != nil {
		return err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return err
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(key, object, false)
}

func (cd *Codec) get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		v, ok := cd.localCache.Get(key)
		if ok {
			b, ok := v.([]byte)
			if ok {
				atomic.AddInt64(&cd.hits, 1)
				return b, nil
			}
		}
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddInt64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) (interface{}, error) {
	if cd.localCache != nil {
		if err := cd.getItemFast(item); err == nil {
			return item.Object, nil
		}
	}

	return cd.group.Do(item.Key, func() (interface{}, error) {
		if err := cd.getItem(item); err == nil {
			return item.Object, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		cd.Set(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})

		err = cd.getItem(item)
		if err != nil {
			return nil, err
		}
		return item.Object, nil
	})
}

func (cd *Codec) getItem(item *Item) error {
	return cd._getItem(item, false)
}

func (cd *Codec) getItemFast(item *Item) error {
	return cd._getItem(item, true)
}

func (cd *Codec) _getItem(item *Item, onlyLocalCache bool) error {
	if item.Object != nil {
		return cd.get(item.Key, item.Object, onlyLocalCache)
	} else {
		return cd.get(item.Key, &item.Object, onlyLocalCache)
	}
}

func (cd *Codec) Delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Codec) Hits() int {
	return int(atomic.LoadInt64(&cd.hits))
}

func (cd *Codec) Misses() int {
	return int(atomic.LoadInt64(&cd.misses))
}
<|endoftext|>


<commit_before>package main

import (
	"hash/fnv"
	"log"
	"math"
	"strconv"
	"time"

	"github.com/dustin/gojson"
	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var cacheInput chan *processIn
var cacheInputSet chan *processOut

func cacheReceiveLoop(client *memcached.Client, out chan *processOut) {
	for {
		res, err := client.Receive()
		if err != nil {
			log.Printf("Error receiving from memcache: %v", err)
			return
		}

		po := processOut{cacheOpaque: res.Opaque}

		if res.Opcode == gomemcached.GET && res.Status == gomemcached.SUCCESS {
			rv := map[string][]interface{}{}
			err = json.Unmarshal(res.Body, &rv)
			if err == nil {
				po.value = rv["v"]
			} else {
				log.Printf("Decoding error:  %v\n%s", err, res.Body)
				po.err = err
			}
		} else {
			po.err = res
		}

		out <- &po
	}
}

func cacheOpener(ch chan<- *memcached.Client) {
	log.Printf("Connecting to memcached")
	client, err := memcached.Connect("tcp", *cacheAddr)
	if err != nil {
		log.Printf("Failed to connect to memcached: %v", err)
	}
	ch <- client
}

func cacheProcessor(ch <-chan *processIn, chset <-chan *processOut) {
	omap := map[uint32]*processIn{}
	opaque := uint32(100)

	out := make(chan *processOut)
	connch := make(chan *memcached.Client)

	go cacheOpener(connch)

	var client *memcached.Client

	for {
		select {
		case client = <-connch:
			if client == nil {
				time.AfterFunc(time.Second, func() {
					cacheOpener(connch)
				})
			} else {
				go cacheReceiveLoop(client, out)
			}
		case pi := <-ch:
			switch {
			case client == nil:
				// No connection, pass through
				processorInput <- pi
			case time.Now().Before(pi.before):
				newOpaque := opaque
				opaque++
				if opaque == math.MaxUint32 {
					opaque = 100
				}
				omap[newOpaque] = pi
				pi.cacheKey = cacheKey(pi)

				req := &gomemcached.MCRequest{
					Opcode: gomemcached.GET,
					Opaque: newOpaque,
					Key:    []byte(pi.cacheKey),
				}

				err := client.Transmit(req)
				if err != nil {
					log.Printf("Error transmitting!: %v", err)
					client.Close()
					client = nil
					go cacheOpener(connch)
					for _, v := range omap {
						processorInput <- v
					}
					omap = map[uint32]*processIn{}
				}
			default:
				// Too old.
				pi.out <- &processOut{"", pi.key, nil, timeoutError, 0}
			}
		case po := <-out:
			pi, ok := omap[po.cacheOpaque]
			if ok {
				delete(omap, po.cacheOpaque)
				po.key = pi.key
				if po.err == nil {
					po.cacheKey = pi.cacheKey
					pi.out <- po
				} else {
					processorInput <- pi
				}
			} else {
				if po.cacheOpaque > 10 {
					log.Printf("Unknown opaque:  %v", po.cacheOpaque)
				}
			}
		case po := <-chset:
			if client != nil {
				bytes, err := json.Marshal(po)
				if err == nil {
					req := &gomemcached.MCRequest{
						Opcode: gomemcached.SETQ,
						Key:    []byte(po.cacheKey),
						Cas:    0,
						Opaque: 1,
						Extras: []byte{0, 0, 0, 0, 0, 0, 0, 0},
						Body:   bytes}

					err = client.Transmit(req)
					if err != nil {
						log.Printf("Error transmitting!: %v", err)
						client.Close()
						client = nil
						go cacheOpener(connch)
						for _, v := range omap {
							processorInput <- v
						}
						omap = map[uint32]*processIn{}
					}
				} else {
					log.Fatalf("Error marshaling %v: %v", po, err)
				}
			} // has client, will cache
		}
	}
}

func cacheKey(p *processIn) string {
	h := fnv.New64()
	for _, i := range p.infos {
		i.WriteIDTo(h)
	}
	for i := range p.ptrs {
		h.Write([]byte(p.ptrs[i]))
		h.Write([]byte(p.reds[i]))
	}
	for i := range p.filters {
		h.Write([]byte(p.filters[i]))
		h.Write([]byte(p.filtervals[i]))
	}
	return p.dbname + "#" + strconv.FormatInt(p.key, 10) +
		"#" + strconv.FormatUint(h.Sum64(), 10)
}
<commit_msg>Cache fix from upstream API change.<commit_after>package main

import (
	"hash/fnv"
	"log"
	"math"
	"strconv"
	"time"

	"github.com/dustin/gojson"
	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var cacheInput chan *processIn
var cacheInputSet chan *processOut

func cacheReceiveLoop(client *memcached.Client, out chan *processOut) {
	for {
		res, err := memcached.UnwrapMemcachedError(client.Receive())
		if err != nil {
			log.Printf("Error receiving from memcache: %v", err)
			return
		}

		po := processOut{cacheOpaque: res.Opaque}

		if res.Opcode == gomemcached.GET && res.Status == gomemcached.SUCCESS {
			rv := map[string][]interface{}{}
			err = json.Unmarshal(res.Body, &rv)
			if err == nil {
				po.value = rv["v"]
			} else {
				log.Printf("Decoding error:  %v\n%s", err, res.Body)
				po.err = err
			}
		} else {
			po.err = res
		}

		out <- &po
	}
}

func cacheOpener(ch chan<- *memcached.Client) {
	log.Printf("Connecting to memcached")
	client, err := memcached.Connect("tcp", *cacheAddr)
	if err != nil {
		log.Printf("Failed to connect to memcached: %v", err)
	}
	ch <- client
}

func cacheProcessor(ch <-chan *processIn, chset <-chan *processOut) {
	omap := map[uint32]*processIn{}
	opaque := uint32(100)

	out := make(chan *processOut)
	connch := make(chan *memcached.Client)

	go cacheOpener(connch)

	var client *memcached.Client

	for {
		select {
		case client = <-connch:
			if client == nil {
				time.AfterFunc(time.Second, func() {
					cacheOpener(connch)
				})
			} else {
				go cacheReceiveLoop(client, out)
			}
		case pi := <-ch:
			switch {
			case client == nil:
				// No connection, pass through
				processorInput <- pi
			case time.Now().Before(pi.before):
				newOpaque := opaque
				opaque++
				if opaque == math.MaxUint32 {
					opaque = 100
				}
				omap[newOpaque] = pi
				pi.cacheKey = cacheKey(pi)

				req := &gomemcached.MCRequest{
					Opcode: gomemcached.GET,
					Opaque: newOpaque,
					Key:    []byte(pi.cacheKey),
				}

				err := client.Transmit(req)
				if err != nil {
					log.Printf("Error transmitting!: %v", err)
					client.Close()
					client = nil
					go cacheOpener(connch)
					for _, v := range omap {
						processorInput <- v
					}
					omap = map[uint32]*processIn{}
				}
			default:
				// Too old.
				pi.out <- &processOut{"", pi.key, nil, timeoutError, 0}
			}
		case po := <-out:
			pi, ok := omap[po.cacheOpaque]
			if ok {
				delete(omap, po.cacheOpaque)
				po.key = pi.key
				if po.err == nil {
					po.cacheKey = pi.cacheKey
					pi.out <- po
				} else {
					processorInput <- pi
				}
			} else {
				if po.cacheOpaque > 10 {
					log.Printf("Unknown opaque:  %v", po.cacheOpaque)
				}
			}
		case po := <-chset:
			if client != nil {
				bytes, err := json.Marshal(po)
				if err == nil {
					req := &gomemcached.MCRequest{
						Opcode: gomemcached.SETQ,
						Key:    []byte(po.cacheKey),
						Cas:    0,
						Opaque: 1,
						Extras: []byte{0, 0, 0, 0, 0, 0, 0, 0},
						Body:   bytes}

					err = client.Transmit(req)
					if err != nil {
						log.Printf("Error transmitting!: %v", err)
						client.Close()
						client = nil
						go cacheOpener(connch)
						for _, v := range omap {
							processorInput <- v
						}
						omap = map[uint32]*processIn{}
					}
				} else {
					log.Fatalf("Error marshaling %v: %v", po, err)
				}
			} // has client, will cache
		}
	}
}

func cacheKey(p *processIn) string {
	h := fnv.New64()
	for _, i := range p.infos {
		i.WriteIDTo(h)
	}
	for i := range p.ptrs {
		h.Write([]byte(p.ptrs[i]))
		h.Write([]byte(p.reds[i]))
	}
	for i := range p.filters {
		h.Write([]byte(p.filters[i]))
		h.Write([]byte(p.filtervals[i]))
	}
	return p.dbname + "#" + strconv.FormatInt(p.key, 10) +
		"#" + strconv.FormatUint(h.Sum64(), 10)
}
<|endoftext|>


<commit_before>// redis cache backend implementation for persistent storage

package main

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
)

const requestsBucketName = "rqbucket"

type Cache struct {
	db             *bolt.DB
	requestsBucket []byte
}

func getDB(name string) *bolt.DB {
	log.WithFields(log.Fields{
		"databaseName": name,
		"Mode":         AppConfig.mode,
	}).Info("Initiating database")
	db, err := bolt.Open(name, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	return db
}

func (c *Cache) Set(key, value []byte) error {
	err := c.db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(c.requestsBucket)
		if err != nil {
			return err
		}
		err = bucket.Put(key, value)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

func (c *Cache) Get(key []byte) (value []byte, err error) {
	err = c.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(c.requestsBucket)
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", c.requestsBucket)
		}
		value = bucket.Get(key)
		return nil
	})

	return
}

func (c *Cache) GetAllRequests() (payloads []Payload, err error) {
	err = c.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(c.requestsBucket)
		if b == nil {
			// bucket doesn't exist
			return nil
		}
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			var pl Payload
			err = json.Unmarshal(v, &pl)

			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"json":  v,
				}).Warning("Failed to deserialize json")
			} else {
				payloads = append(payloads, pl)
			}
		}
		return nil
	})
	return
}

func (c *Cache) DeleteBucket(name []byte) (err error) {
	err = c.db.Update(func(tx *bolt.Tx) error {
		err = tx.DeleteBucket(name)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
				"name":  string(name),
			}).Warning("Failed to delete bucket")
			return err
		} else {
			return nil
		}
	})
	return
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/boltdb/bolt"
)

const requestsBucketName = "rqbucket"

type Cache struct {
	db             *bolt.DB
	requestsBucket []byte
}

func getDB(name string) *bolt.DB {
	log.WithFields(log.Fields{
		"databaseName": name,
	}).Info("Initiating database")
	db, err := bolt.Open(name, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	return db
}

func (c *Cache) Set(key, value []byte) error {
	err := c.db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(c.requestsBucket)
		if err != nil {
			return err
		}
		err = bucket.Put(key, value)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

func (c *Cache) Get(key []byte) (value []byte, err error) {
	err = c.db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(c.requestsBucket)
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", c.requestsBucket)
		}
		value = bucket.Get(key)
		return nil
	})

	return
}

func (c *Cache) GetAllRequests() (payloads []Payload, err error) {
	err = c.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(c.requestsBucket)
		if b == nil {
			// bucket doesn't exist
			return nil
		}
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			var pl Payload
			err = json.Unmarshal(v, &pl)

			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"json":  v,
				}).Warning("Failed to deserialize json")
			} else {
				payloads = append(payloads, pl)
			}
		}
		return nil
	})
	return
}

func (c *Cache) DeleteBucket(name []byte) (err error) {
	err = c.db.Update(func(tx *bolt.Tx) error {
		err = tx.DeleteBucket(name)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
				"name":  string(name),
			}).Warning("Failed to delete bucket")
			return err
		} else {
			return nil
		}
	})
	return
}
<|endoftext|>


<commit_before>package mango

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"

	"github.com/singpenguin/mango/utils"
)

type HTTPRequest struct {
	Args       []string //route params
	Method     string   //HTTP method
	RemoteAddr string   //ip
	Path       string
	Host       string
	Params     url.Values                         //GET or POST params
	Body       []byte                             //Content-Type=application/octet-stream, Body=http body
	Files      map[string][]*multipart.FileHeader //Content-Type=multipart/form-data
	Length     int                                //response length
	StatusCode int

	W http.ResponseWriter
	R *http.Request
}

func (self *HTTPRequest) Init(w http.ResponseWriter, r *http.Request, a []string) {
	self.W = w
	self.R = r
	//self.Method = r.Method
	self.Path = r.URL.Path
	self.Args = a
	self.Host = r.Host
	self.StatusCode = http.StatusOK

	ip := r.Header.Get("X-Forwarded-For")
	if ip == "" {
		ip = r.RemoteAddr
	}
	self.RemoteAddr = strings.Split(ip, ":")[0]
	self.parseBodyArguments()
}

func (self *HTTPRequest) UserAgent() string {
	return self.R.Header.Get("User-Agent")
}

func (self *HTTPRequest) Header(s string) string {
	return self.R.Header.Get(s)
}

func (self *HTTPRequest) SetHeader(k, v string) {
	self.W.Header().Set(k, v)
}

func (self *HTTPRequest) SetStatus(st int) {
	self.StatusCode = st
	self.W.WriteHeader(st)
}

func (self *HTTPRequest) Cookie(k string) (string, error) {
	ck, err := self.R.Cookie(k)
	if err != nil {
		return "", err
	}
	return ck.Value, nil
}

var cks = strings.NewReplacer("\n", "-", "\r", "-")
var cvs = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func (self *HTTPRequest) SetSecureCookie(k, v string, params ...interface{}) {
	bv := []byte(v)
	ts := utils.IntToStr(utils.Timestamp())
	v = utils.Base64Encode(bv)
	sa := []string{}
	sa = append(sa, "2|1:0")
	sa = append(sa, fmt.Sprintf("%d:%s", len(ts), ts))
	sa = append(sa, fmt.Sprintf("%d:%s", len(k), k))
	sa = append(sa, fmt.Sprintf("%d:%s", len(v), v))
	sa = append(sa, "")
	to_sign := strings.Join(sa, "|")
	signature := utils.HmacSha256(to_sign, CookieSecret)

	self.SetCookie(k, to_sign+signature, params...)
}

func (self *HTTPRequest) GetSecureCookie(k string) (string, error) {
	value, err := self.Cookie(k)
	if err != nil || len(value) < 64 {
		return "", err
	}
	_rest := value[2:]
	_, rest, err := consumeField(_rest)
	if err != nil {
		return "", err
	}
	timestamp_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	name_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	value_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	passed_sig := rest
	signed_str := value[:len(value)-len(passed_sig)]
	signature := utils.HmacSha256(signed_str, CookieSecret)
	if passed_sig != signature {
		return "", errors.New("signature invalid")
	}
	if name_field != k {
		return "", errors.New("cookie name invalid")
	}
	timestamp, _ := utils.StrToInt64(timestamp_field)
	if timestamp < utils.Timestamp()-2678400 {
		return "", errors.New("signature has expired")
	}
	devalue, err := utils.Base64Decode(value_field)
	if err != nil {
		return "", errors.New("value cannot be decode")
	}
	return devalue, nil
}

func consumeField(value string) (string, string, error) {
	var length, rest string
	utils.Unpack(strings.SplitN(value, ":", 2), &length, &rest)

	n, err := utils.StrToInt64(length)
	if err != nil {
		return "", "", errors.New("length field invalid")
	}
	key_field := rest[:n]
	if rest[n:n+1] != "|" {
		return "", "", errors.New("malformed signed value")
	}
	rest = rest[n+1:]
	return key_field, rest, nil
}

func (self *HTTPRequest) SetCookie(k, v string, params ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", cks.Replace(k), cvs.Replace(v))
	if len(params) > 0 {
		switch v := params[0].(type) {
		case int:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int64:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int32:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		}
	}
	// Path, Domain, Secure, HttpOnly

	// default "/"
	if len(params) > 1 {
		if v, ok := params[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", cvs.Replace(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(params) > 2 {
		if v, ok := params[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", cvs.Replace(v))
		}
	}

	// default empty
	if len(params) > 3 {
		var secure bool
		switch v := params[3].(type) {
		case bool:
			secure = v
		default:
			if params[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false.
	httponly := false
	if len(params) > 4 {
		if v, ok := params[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	self.W.Header().Add("Set-Cookie", b.String())
}

func (self *HTTPRequest) Scheme() string {
	if self.R.URL.Scheme != "" {
		return self.R.URL.Scheme
	} else if self.R.TLS == nil {
		return "http"
	} else {
		return "https"
	}
}

func (self *HTTPRequest) Redirect(newloc string) {
	self.StatusCode = http.StatusFound
	self.SetHeader("Location", newloc)
	self.SetStatus(http.StatusFound)
}

func (self *HTTPRequest) Query() map[string][]string {
	return self.R.URL.Query()
}

func (self *HTTPRequest) QueryString() string {
	if s := strings.Split(self.R.RequestURI, "?"); len(s) > 1 {
		return s[1]
	} else {
		return ""
	}
}

func (self *HTTPRequest) parseBodyArguments() {
	qs := self.R.URL.Query()
	var pt url.Values
	ct := self.Header("Content-Type")
	if strings.HasPrefix(ct, "application/x-www-form-urlencoded") {
		if err := self.R.ParseForm(); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			pt = self.R.Form
		}
	} else if strings.HasPrefix(ct, "multipart/form-data") {
		// TODO: Extract the multipart form param so app can set it.
		if err := self.R.ParseMultipartForm(32 << 20 /* 32 MB */); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			self.Files = self.R.MultipartForm.File
		}
	} else if strings.HasPrefix(ct, "application/octet-stream") {
		self.Body = self.Data()
	}

	num := len(qs) + len(pt)

	if num == 0 {
		self.Params = make(url.Values, 0)
		return
	}
	values := make(url.Values, num)
	for k, v := range qs {
		values[k] = append(values[k], v...)
	}
	for k, v := range pt {
		values[k] = append(values[k], v...)
	}
	self.Params = values
}

func (self *HTTPRequest) Data() []byte {
	body, _ := ioutil.ReadAll(self.R.Body)
	self.R.Body.Close()
	bf := bytes.NewBuffer(body)
	self.R.Body = ioutil.NopCloser(bf)
	return body
}

func (self *HTTPRequest) Write(s []byte) {
	n, _ := self.W.Write(s)
	self.Length += n
}

func (self *HTTPRequest) Render(name string, data interface{}) {
	Template[name].Execute(self.W, data)
}

var jsonContentType = "application/json; charset=utf-8"

func (self *HTTPRequest) Send(data interface{}) error {
	self.SetHeader("Content-Type", jsonContentType)
	return json.NewEncoder(self.W).Encode(data)
}

type Handler func(ctx *HTTPRequest)
<commit_msg>fix form-data bug<commit_after>package mango

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"

	"github.com/singpenguin/mango/utils"
)

type HTTPRequest struct {
	Args       []string //route params
	Method     string   //HTTP method
	RemoteAddr string   //ip
	Path       string
	Host       string
	Params     url.Values                         //GET or POST params
	Body       []byte                             //Content-Type=application/octet-stream, Body=http body
	Files      map[string][]*multipart.FileHeader //Content-Type=multipart/form-data
	Length     int                                //response length
	StatusCode int

	W http.ResponseWriter
	R *http.Request
}

func (self *HTTPRequest) Init(w http.ResponseWriter, r *http.Request, a []string) {
	self.W = w
	self.R = r
	//self.Method = r.Method
	self.Path = r.URL.Path
	self.Args = a
	self.Host = r.Host
	self.StatusCode = http.StatusOK

	ip := r.Header.Get("X-Forwarded-For")
	if ip == "" {
		ip = r.RemoteAddr
	}
	self.RemoteAddr = strings.Split(ip, ":")[0]
	self.parseBodyArguments()
}

func (self *HTTPRequest) UserAgent() string {
	return self.R.Header.Get("User-Agent")
}

func (self *HTTPRequest) Header(s string) string {
	return self.R.Header.Get(s)
}

func (self *HTTPRequest) SetHeader(k, v string) {
	self.W.Header().Set(k, v)
}

func (self *HTTPRequest) SetStatus(st int) {
	self.StatusCode = st
	self.W.WriteHeader(st)
}

func (self *HTTPRequest) Cookie(k string) (string, error) {
	ck, err := self.R.Cookie(k)
	if err != nil {
		return "", err
	}
	return ck.Value, nil
}

var cks = strings.NewReplacer("\n", "-", "\r", "-")
var cvs = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func (self *HTTPRequest) SetSecureCookie(k, v string, params ...interface{}) {
	bv := []byte(v)
	ts := utils.IntToStr(utils.Timestamp())
	v = utils.Base64Encode(bv)
	sa := []string{}
	sa = append(sa, "2|1:0")
	sa = append(sa, fmt.Sprintf("%d:%s", len(ts), ts))
	sa = append(sa, fmt.Sprintf("%d:%s", len(k), k))
	sa = append(sa, fmt.Sprintf("%d:%s", len(v), v))
	sa = append(sa, "")
	to_sign := strings.Join(sa, "|")
	signature := utils.HmacSha256(to_sign, CookieSecret)

	self.SetCookie(k, to_sign+signature, params...)
}

func (self *HTTPRequest) GetSecureCookie(k string) (string, error) {
	value, err := self.Cookie(k)
	if err != nil || len(value) < 64 {
		return "", err
	}
	_rest := value[2:]
	_, rest, err := consumeField(_rest)
	if err != nil {
		return "", err
	}
	timestamp_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	name_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	value_field, rest, err := consumeField(rest)
	if err != nil {
		return "", err
	}
	passed_sig := rest
	signed_str := value[:len(value)-len(passed_sig)]
	signature := utils.HmacSha256(signed_str, CookieSecret)
	if passed_sig != signature {
		return "", errors.New("signature invalid")
	}
	if name_field != k {
		return "", errors.New("cookie name invalid")
	}
	timestamp, _ := utils.StrToInt64(timestamp_field)
	if timestamp < utils.Timestamp()-2678400 {
		return "", errors.New("signature has expired")
	}
	devalue, err := utils.Base64Decode(value_field)
	if err != nil {
		return "", errors.New("value cannot be decode")
	}
	return devalue, nil
}

func consumeField(value string) (string, string, error) {
	var length, rest string
	utils.Unpack(strings.SplitN(value, ":", 2), &length, &rest)

	n, err := utils.StrToInt64(length)
	if err != nil {
		return "", "", errors.New("length field invalid")
	}
	key_field := rest[:n]
	if rest[n:n+1] != "|" {
		return "", "", errors.New("malformed signed value")
	}
	rest = rest[n+1:]
	return key_field, rest, nil
}

func (self *HTTPRequest) SetCookie(k, v string, params ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", cks.Replace(k), cvs.Replace(v))
	if len(params) > 0 {
		switch v := params[0].(type) {
		case int:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int64:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int32:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		}
	}
	// Path, Domain, Secure, HttpOnly

	// default "/"
	if len(params) > 1 {
		if v, ok := params[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", cvs.Replace(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(params) > 2 {
		if v, ok := params[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", cvs.Replace(v))
		}
	}

	// default empty
	if len(params) > 3 {
		var secure bool
		switch v := params[3].(type) {
		case bool:
			secure = v
		default:
			if params[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false.
	httponly := false
	if len(params) > 4 {
		if v, ok := params[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	self.W.Header().Add("Set-Cookie", b.String())
}

func (self *HTTPRequest) Scheme() string {
	if self.R.URL.Scheme != "" {
		return self.R.URL.Scheme
	} else if self.R.TLS == nil {
		return "http"
	} else {
		return "https"
	}
}

func (self *HTTPRequest) Redirect(newloc string) {
	self.StatusCode = http.StatusFound
	self.SetHeader("Location", newloc)
	self.SetStatus(http.StatusFound)
}

func (self *HTTPRequest) Query() map[string][]string {
	return self.R.URL.Query()
}

func (self *HTTPRequest) QueryString() string {
	if s := strings.Split(self.R.RequestURI, "?"); len(s) > 1 {
		return s[1]
	} else {
		return ""
	}
}

func (self *HTTPRequest) parseBodyArguments() {
	qs := self.R.URL.Query()
	var pt url.Values
	ct := self.Header("Content-Type")
	if strings.HasPrefix(ct, "application/x-www-form-urlencoded") {
		if err := self.R.ParseForm(); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			pt = self.R.Form
		}
	} else if strings.HasPrefix(ct, "multipart/form-data") {
		// TODO: Extract the multipart form param so app can set it.
		if err := self.R.ParseMultipartForm(32 << 20 /* 32 MB */); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			self.Files = self.R.MultipartForm.File
			pt = self.R.Form
		}
	} else if strings.HasPrefix(ct, "application/octet-stream") {
		self.Body = self.Data()
	}

	num := len(qs) + len(pt)

	if num == 0 {
		self.Params = make(url.Values, 0)
		return
	}
	values := make(url.Values, num)
	for k, v := range qs {
		values[k] = append(values[k], v...)
	}
	for k, v := range pt {
		values[k] = append(values[k], v...)
	}
	self.Params = values
}

func (self *HTTPRequest) Data() []byte {
	body, _ := ioutil.ReadAll(self.R.Body)
	self.R.Body.Close()
	bf := bytes.NewBuffer(body)
	self.R.Body = ioutil.NopCloser(bf)
	return body
}

func (self *HTTPRequest) Write(s []byte) {
	n, _ := self.W.Write(s)
	self.Length += n
}

func (self *HTTPRequest) Render(name string, data interface{}) {
	Template[name].Execute(self.W, data)
}

var jsonContentType = "application/json; charset=utf-8"

func (self *HTTPRequest) Send(data interface{}) error {
	self.SetHeader("Content-Type", jsonContentType)
	return json.NewEncoder(self.W).Encode(data)
}

type Handler func(ctx *HTTPRequest)
<|endoftext|>


<commit_before><commit_msg>implement "Set" method of xml provider.<commit_after><|endoftext|>


<commit_before><commit_msg>Stop after doing the current work, not when we see the end.<commit_after><|endoftext|>


<commit_before>package glc

import (
	zmq "github.com/pebbe/zmq4"
	"log"
)

func BindProxy(frontend string, backend string) {
	// create XSUB for publishers to connect to
	xSub, _ := zmq.NewSocket(zmq.XSUB)
	defer xSub.Close()
	err := xSub.Bind(frontend)
	if err != nil {
        log.Fatal(err)
    }

	// create XPUB for subscribers to connect to
	xPub, _ := zmq.NewSocket(zmq.XPUB)
	defer xPub.Close()
	err = xPub.Bind(backend)
	if err != nil {
        log.Fatal(err)
    }

	err = zmq.Proxy(xSub, xPub, nil)
	log.Fatalln("Proxy interrupted:", err)
}

func BindPublisher(endpoint string) *zmq.Socket {
	socket, err := zmq.NewSocket(zmq.PUB)
	defer socket.Close()

	if err != nil {
        log.Fatal(err)
    }

	err = socket.Connect(endpoint)

	if err != nil {
        log.Fatal(err)
    }

	return socket
}

func BindSubscriber(endpoint string, filter string, callback func(message string)) *zmq.Socket {
	socket, err := zmq.NewSocket(zmq.PUB)
	defer socket.Close()

	socket.SetSubscribe(filter)

	if err != nil {
        log.Fatal(err)
    }

	err = socket.Connect(endpoint)

	if err != nil {
        log.Fatal(err)
    }

	for {
		msg, _ := socket.Recv(zmq.DONTWAIT)

		if msg != "" {
			callback(msg)
		}
	}

	return socket
}

func PublishMessage(message string, socket *zmq.Socket) {
	socket.Send(message, zmq.DONTWAIT)
}<commit_msg>Improvements to abstraction<commit_after>package glc

import (
	zmq "github.com/pebbe/zmq4"
	"log"
)

type GlcZmq struct {
	Socket *zmq.Socket
}

func BindProxy(frontend string, backend string) {
	// create XSUB for publishers to connect to
	xSub, _ := zmq.NewSocket(zmq.XSUB)
	defer xSub.Close()
	err := xSub.Bind(frontend)
	if err != nil {
        log.Fatal(err)
    }

	// create XPUB for subscribers to connect to
	xPub, _ := zmq.NewSocket(zmq.XPUB)
	defer xPub.Close()
	err = xPub.Bind(backend)
	if err != nil {
        log.Fatal(err)
    }

	err = zmq.Proxy(xSub, xPub, nil)
	log.Fatalln("Proxy interrupted:", err)
}

func BindPublisher(endpoint string) *GlcZmq {
	socket, err := zmq.NewSocket(zmq.PUB)
	defer socket.Close()

	if err != nil {
        log.Fatal(err)
    }

	err = socket.Connect(endpoint)

	if err != nil {
        log.Fatal(err)
    }

	return GlcZmq{socket}
}

func BindSubscriber(endpoint string, filter string, callback func(message string)) *zmq.Socket {
	socket, err := zmq.NewSocket(zmq.PUB)
	defer socket.Close()

	socket.SetSubscribe(filter)

	if err != nil {
        log.Fatal(err)
    }

	err = socket.Connect(endpoint)

	if err != nil {
        log.Fatal(err)
    }

	for {
		msg, _ := socket.Recv(zmq.DONTWAIT)

		if msg != "" {
			callback(msg)
		}
	}

	return GlcZmq{socket}
}

func PublishMessage(message string, socket *GlcZmq) {
	GlcZmq.Socket.Send(message, zmq.DONTWAIT)
}<|endoftext|>


<commit_before>package static

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"time"
)

// DirFile contains the static directory and file content info
type DirFile struct {
	Path       string
	Name       string
	Size       int64
	Mode       os.FileMode
	ModTime    int64
	IsDir      bool
	Compressed string
	Files      []*DirFile
}

// Files contains a full instance of a static file collection
type Files struct {
	dir Dir
}

// File contains the static FileInfo
type File struct {
	data    []byte
	path    string
	name    string
	size    int64
	mode    os.FileMode
	modTime int64
	isDir   bool
	files   []*File
}

// Dir implements the FileSystem interface
type Dir struct {
	name             string
	isProductionMode bool
	files            map[string]*File
}

type httpFile struct {
	*bytes.Reader
	*File
}

// Config contains information about how extracting the data should behave
type Config struct {
	IsProductionMode bool
	Name             string
}

// Open returns the FileSystem DIR
func (dir Dir) Open(name string) (http.File, error) {

	if dir.isProductionMode {
		f, found := dir.files[path.Clean(name)]
		if !found {
			return nil, os.ErrNotExist
		}

		return f.File()
	}

	return os.Open(name)
}

// File returns an http.File or error
func (f File) File() (http.File, error) {

	// if production read filesystem file
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		File:   &f,
	}, nil
}

// Close closes the File, rendering it unusable for I/O. It returns an error, if any.
func (f File) Close() error {
	return nil
}

// Readdir returns nil fileinfo and an error because the static FileSystem does not store directories
func (f File) Readdir(count int) ([]os.FileInfo, error) {

	if !f.IsDir() {
		return nil, errors.New("not a directory")
	}

	files := []os.FileInfo{}

	for _, file := range f.files {
		files = append(files, *file)
	}

	return files, nil
}

// Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError.
func (f File) Stat() (os.FileInfo, error) {
	return f, nil
}

// Name returns the name of the file as presented to Open.
func (f File) Name() string {
	return f.name
}

// Size length in bytes for regular files; system-dependent for others
func (f File) Size() int64 {
	return f.size
}

// Mode returns file mode bits
func (f File) Mode() os.FileMode {
	mode := os.FileMode(0644)
	if f.IsDir() {
		return mode | os.ModeDir
	}
	return mode
	// return 0
}

// ModTime returns the files modification time
func (f File) ModTime() time.Time {
	return time.Unix(f.modTime, 0)
}

// IsDir reports whether f describes a directory.
func (f File) IsDir() bool {
	return f.isDir
}

// Sys returns the underlying data source (can return nil)
func (f File) Sys() interface{} {
	return f
}

// New create a new static file instance.
func New(config *Config, file *DirFile) (*Files, error) {
	files := map[string]*File{}

	if config.IsProductionMode {
		processFiles(files, file)
	}

	return &Files{
		dir: Dir{
			name:             config.Name,
			isProductionMode: config.IsProductionMode,
			files:            files,
		},
	}, nil
}

func processFiles(files map[string]*File, file *DirFile) *File {

	f := &File{
		path:    file.Path,
		name:    file.Name,
		size:    file.Size,
		mode:    file.Mode,
		modTime: file.ModTime,
		isDir:   file.IsDir,
		files:   []*File{},
	}

	files[f.path] = f

	if file.IsDir {
		for _, dirFile := range file.Files {
			resultFile := processFiles(files, dirFile)
			f.files = append(f.files, resultFile)
		}

		return f
	}

	// decompress file contents

	b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(file.Compressed))
	reader, err := gzip.NewReader(b64)
	if err != nil {
		log.Fatal(err)
	}

	f.data, err = ioutil.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}

	return f
}

// FS returns an http.FileSystem object for serving files over http
func (s *Files) FS() http.FileSystem {
	return s.dir
}

// GetFile return a files contents as []byte from the filesystem, static or local
func (s *Files) GetFile(name string) ([]byte, error) {
	f, err := s.dir.Open(name)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// GetFileString return a files contents as string from the filesystem, static or local
func (s *Files) GetFileString(name string) (string, error) {
	f, err := s.dir.Open(name)
	if err != nil {
		return "", err
	}

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}

	return string(b), nil
}
<commit_msg>Update Readdir function to return subsequent files<commit_after>package static

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"time"
)

// DirFile contains the static directory and file content info
type DirFile struct {
	Path       string
	Name       string
	Size       int64
	Mode       os.FileMode
	ModTime    int64
	IsDir      bool
	Compressed string
	Files      []*DirFile
}

// Files contains a full instance of a static file collection
type Files struct {
	dir Dir
}

// File contains the static FileInfo
type File struct {
	data         []byte
	path         string
	name         string
	size         int64
	mode         os.FileMode
	modTime      int64
	isDir        bool
	files        []*File
	lastDirIndex int
}

// Dir implements the FileSystem interface
type Dir struct {
	name             string
	isProductionMode bool
	files            map[string]*File
}

type httpFile struct {
	*bytes.Reader
	*File
}

// Config contains information about how extracting the data should behave
type Config struct {
	IsProductionMode bool
	Name             string
}

// Open returns the FileSystem DIR
func (dir Dir) Open(name string) (http.File, error) {

	if dir.isProductionMode {
		f, found := dir.files[path.Clean(name)]
		if !found {
			return nil, os.ErrNotExist
		}

		return f.File()
	}

	return os.Open(name)
}

// File returns an http.File or error
func (f File) File() (http.File, error) {

	// if production read filesystem file
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		File:   &f,
	}, nil
}

// Close closes the File, rendering it unusable for I/O. It returns an error, if any.
func (f File) Close() error {
	return nil
}

// Readdir returns nil fileinfo and an error because the static FileSystem does not store directories
func (f File) Readdir(count int) ([]os.FileInfo, error) {

	if !f.IsDir() {
		return nil, errors.New("not a directory")
	}

	var files []os.FileInfo

	if count <= 0 {
		files = make([]os.FileInfo, len(f.files))
		count = len(f.files)
		f.lastDirIndex = 0
	} else {
		files = make([]os.FileInfo, count)
	}

	if f.lastDirIndex >= len(f.files) {
		return nil, io.EOF
	}

	if count+f.lastDirIndex >= len(f.files) {
		count = len(f.files)
	}

	for i := f.lastDirIndex; i < count; i++ {
		files = append(files, *f.files[i])
	}

	return files, nil
}

// Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError.
func (f File) Stat() (os.FileInfo, error) {
	return f, nil
}

// Name returns the name of the file as presented to Open.
func (f File) Name() string {
	return f.name
}

// Size length in bytes for regular files; system-dependent for others
func (f File) Size() int64 {
	return f.size
}

// Mode returns file mode bits
func (f File) Mode() os.FileMode {
	mode := os.FileMode(0644)
	if f.IsDir() {
		return mode | os.ModeDir
	}
	return mode
	// return 0
}

// ModTime returns the files modification time
func (f File) ModTime() time.Time {
	return time.Unix(f.modTime, 0)
}

// IsDir reports whether f describes a directory.
func (f File) IsDir() bool {
	return f.isDir
}

// Sys returns the underlying data source (can return nil)
func (f File) Sys() interface{} {
	return f
}

// New create a new static file instance.
func New(config *Config, file *DirFile) (*Files, error) {
	files := map[string]*File{}

	if config.IsProductionMode {
		processFiles(files, file)
	}

	return &Files{
		dir: Dir{
			name:             config.Name,
			isProductionMode: config.IsProductionMode,
			files:            files,
		},
	}, nil
}

func processFiles(files map[string]*File, file *DirFile) *File {

	f := &File{
		path:    file.Path,
		name:    file.Name,
		size:    file.Size,
		mode:    file.Mode,
		modTime: file.ModTime,
		isDir:   file.IsDir,
		files:   []*File{},
	}

	files[f.path] = f

	if file.IsDir {
		for _, dirFile := range file.Files {
			resultFile := processFiles(files, dirFile)
			f.files = append(f.files, resultFile)
		}

		return f
	}

	// decompress file contents

	b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(file.Compressed))
	reader, err := gzip.NewReader(b64)
	if err != nil {
		log.Fatal(err)
	}

	f.data, err = ioutil.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}

	return f
}

// FS returns an http.FileSystem object for serving files over http
func (s *Files) FS() http.FileSystem {
	return s.dir
}

// GetFile return a files contents as []byte from the filesystem, static or local
func (s *Files) GetFile(name string) ([]byte, error) {
	f, err := s.dir.Open(name)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// GetFileString return a files contents as string from the filesystem, static or local
func (s *Files) GetFileString(name string) (string, error) {
	f, err := s.dir.Open(name)
	if err != nil {
		return "", err
	}

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}

	return string(b), nil
}
<|endoftext|>