<commit_before>package main

import (
	"log"
	"os"
	"text/template"
)

type Source struct {
	tmpl *template.Template
	api  *Api
}

func NewSource(api *Api, templateFilename string) *Source {
	source := new(Source)

	source.api = api
	source.tmpl = template.Must(template.ParseGlob(templateFilename))

	return source
}

func (s *Source) SaveToFile(filename string) error {
	type TemplateData struct {
		ServiceName string
		MethodList  []string
	}

	templateData := TemplateData{
		ServiceName: "myservice",
	}

	for k := range s.api.swagger.Paths {
		templateData.MethodList = append(templateData.MethodList, "service_"+k[1:])
	}

	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("create file: %s", err)
	}

	err = s.tmpl.Execute(f, templateData)
	if err != nil {
		log.Fatalf("template execution: %s", err)
	}

	return nil
}
<commit_msg>Source base path must be created when necessary.<commit_after>package main

import (
	"log"
	"os"
	"text/template"
)

type Source struct {
	tmpl *template.Template
	api  *Api
}

func NewSource(api *Api, templateFilename string) *Source {
	source := new(Source)

	source.api = api
	source.tmpl = template.Must(template.ParseGlob(templateFilename))

	return source
}

func (s *Source) SaveToFile(filename string) error {
	type TemplateData struct {
		ServiceName string
		MethodList  []string
	}

	templateData := TemplateData{
		ServiceName: "myservice",
	}

	for k := range s.api.swagger.Paths {
		templateData.MethodList = append(templateData.MethodList, "service_"+k[1:])
	}

	if err := CreateBasePath(filename); err != nil {
		log.Fatalf("creating base for %s: %s", filename, err)
	}

	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("create file: %s", err)
	}

	defer f.Close()

	err = s.tmpl.Execute(f, templateData)
	if err != nil {
		log.Fatalf("template execution: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package gopdf

//Current current state
type Current struct {
	setXCount int //many times we go func SetX()
	X         float64
	Y         float64

	//font
	IndexOfFontObj int
	CountOfFont    int
	CountOfL       int

	FontSize      int
	FontStyle     int // Regular|Bold|Italic|Underline
	FontFontCount int
	FontType      int // CURRENT_FONT_TYPE_IFONT or  CURRENT_FONT_TYPE_SUBSET

	FontISubset *SubsetFontObj // FontType == CURRENT_FONT_TYPE_SUBSET

	//page
	IndexOfPageObj int

	//img
	CountOfImg int
	//cache of image in pdf file
	ImgCaches []ImageCache

	//text color mode
	txtColorMode string //color, gray

	//text color
	txtColor Rgb

	//text grayscale
	grayFill float64
	//draw grayscale
	grayStroke float64

	lineWidth float64

	//current page size
	pageSize *Rect

	//current trim size
	trimBox *Box

	transparency    Transparency
	transparencyMap map[string]Transparency
}

func (c *Current) setTextColor(rgb Rgb) {
	c.txtColor = rgb
}

func (c *Current) textColor() Rgb {
	return c.txtColor
}

// ImageCache is metadata for caching images.
type ImageCache struct {
	Path  string //ID or Path
	Index int
	Rect  *Rect
}

//Rgb  rgb color
type Rgb struct {
	r uint8
	g uint8
	b uint8
}

//SetR set red
func (rgb *Rgb) SetR(r uint8) {
	rgb.r = r
}

//SetG set green
func (rgb *Rgb) SetG(g uint8) {
	rgb.g = g
}

//SetB set blue
func (rgb *Rgb) SetB(b uint8) {
	rgb.b = b
}

func (rgb Rgb) equal(obj Rgb) bool {
	if rgb.r == obj.r && rgb.g == obj.g && rgb.b == obj.b {
		return true
	}
	return false
}

// Transparency defines an object alpha.
type Transparency struct {
	IndexOfExtGState int
}
<commit_msg>clean up<commit_after>package gopdf

//Current current state
type Current struct {
	setXCount int //many times we go func SetX()
	X         float64
	Y         float64

	//font
	IndexOfFontObj int
	CountOfFont    int
	CountOfL       int

	FontSize      int
	FontStyle     int // Regular|Bold|Italic|Underline
	FontFontCount int
	FontType      int // CURRENT_FONT_TYPE_IFONT or  CURRENT_FONT_TYPE_SUBSET

	FontISubset *SubsetFontObj // FontType == CURRENT_FONT_TYPE_SUBSET

	//page
	IndexOfPageObj int

	//img
	CountOfImg int
	//cache of image in pdf file
	ImgCaches []ImageCache

	//text color mode
	txtColorMode string //color, gray

	//text color
	txtColor Rgb

	//text grayscale
	grayFill float64
	//draw grayscale
	grayStroke float64

	lineWidth float64

	//current page size
	pageSize *Rect

	//current trim box
	trimBox *Box

	transparency    Transparency
	transparencyMap map[string]Transparency
}

func (c *Current) setTextColor(rgb Rgb) {
	c.txtColor = rgb
}

func (c *Current) textColor() Rgb {
	return c.txtColor
}

// ImageCache is metadata for caching images.
type ImageCache struct {
	Path  string //ID or Path
	Index int
	Rect  *Rect
}

//Rgb  rgb color
type Rgb struct {
	r uint8
	g uint8
	b uint8
}

//SetR set red
func (rgb *Rgb) SetR(r uint8) {
	rgb.r = r
}

//SetG set green
func (rgb *Rgb) SetG(g uint8) {
	rgb.g = g
}

//SetB set blue
func (rgb *Rgb) SetB(b uint8) {
	rgb.b = b
}

func (rgb Rgb) equal(obj Rgb) bool {
	if rgb.r == obj.r && rgb.g == obj.g && rgb.b == obj.b {
		return true
	}
	return false
}

// Transparency defines an object alpha.
type Transparency struct {
	IndexOfExtGState int
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"strconv"
	"strings"
	"time"

	"github.com/zorkian/go-datadog-api"
)

var homeTildeShortcut = "~" + string(os.PathSeparator)

type datadogKeys struct {
	ApiKey string `json:"api_key"`
	AppKey string `json:"app_key"`
}

func init() {
	log.SetOutput(os.Stderr)
}

func expandPath(path string) string {
	if path[:2] == homeTildeShortcut {
		currentUser, err := user.Current()
		if err != nil {
			panic(err)
		}
		return strings.Replace(path, homeTildeShortcut, currentUser.HomeDir+string(os.PathSeparator), 1)
	} else {
		return path
	}
}

func readDatadogKeys(configFilePath string, keysChan chan<- string) {
	defer func() {
		if err := recover(); err != nil {
			log.Printf("error: datadog configuration missing or inaccessible.")
			log.Fatalf("error: %v", err)
		}
	}()

	keys := &datadogKeys{
		ApiKey: os.Getenv("DATADOG_API_KEY"),
		AppKey: os.Getenv("DATADOG_APP_KEY"),
	}

	if keys.ApiKey == "" || keys.AppKey == "" {
		f, err := os.Open(expandPath(configFilePath))
		if err != nil {
			panic(err)
		}
		defer f.Close()
		json.NewDecoder(f).Decode(keys)
	}

	keysChan <- keys.ApiKey
	keysChan <- keys.AppKey
}

func validateType(metricType string) string {
	switch metricType {
	case "increment", "incr", "i", "c", "counter":
		return "counter"
	case "gauge", "g":
		return "gauge"
	default:
		panic(fmt.Sprintf("'%s' is not a valid metric type. must be one of 'counter', or 'gauge'.", metricType))
	}
}

func createDataPoint(value string) datadog.DataPoint {
	converted, err := strconv.ParseFloat(value, 64)
	if err != nil {
		panic(err)
	}
	now := float64(time.Now().Unix())
	return datadog.DataPoint([2]float64{now, converted})
}

func validateAndConvertPoints(data []string) (points []datadog.DataPoint) {
	switch len(data) {
	case 0:
		panic("no value(s)")
	default:
		for _, v := range data {
			points = append(points, createDataPoint(v))
		}
	}
	return points
}

func parseArgs(args []string, tags string) datadog.Metric {
	if len(args) < 3 {
		panic("not enough arguments. usage: datadog TYPE METRIC VALUE(S)...")
	}

	return datadog.Metric{
		Type:   validateType(args[0]),
		Metric: args[1],
		Points: validateAndConvertPoints(args[2:]),
		Tags:   strings.Split(tags, ","),
	}
}

func main() {
	defer func() {
		if err := recover(); err != nil {
			log.Fatalf("error: %v", err)
		}
	}()

	// datadog gauge my_metric values...
	var dryRun bool
	flag.BoolVar(&dryRun, "dry-run", false, "Don't send data to datadog.")
	var tags string
	flag.StringVar(&tags, "tags", "", "Tags to add to this metric, e.g. 'key:value,key2:value2'.")
	var configFile string
	flag.StringVar(&configFile, "conf", "~/.datadogrc", "Datadog app and api keys")
	flag.Parse()

	keysChan := make(chan string, 2)
	go readDatadogKeys(configFile, keysChan)

	metric := parseArgs(flag.Args(), tags)

	if dryRun {
		encoded, err := json.Marshal(metric)
		if err != nil {
			panic(err)
		}
		log.Println(string(encoded))
	} else {
		client := datadog.NewClient(<-keysChan, <-keysChan)
		err := client.PostMetrics([]datadog.Metric{metric})
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>readDatadogKeys: if the JSON file is botched, error out.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"strconv"
	"strings"
	"time"

	"github.com/zorkian/go-datadog-api"
)

var homeTildeShortcut = "~" + string(os.PathSeparator)

type datadogKeys struct {
	ApiKey string `json:"api_key"`
	AppKey string `json:"app_key"`
}

func init() {
	log.SetOutput(os.Stderr)
}

func expandPath(path string) string {
	if path[:2] == homeTildeShortcut {
		currentUser, err := user.Current()
		if err != nil {
			panic(err)
		}
		return strings.Replace(path, homeTildeShortcut, currentUser.HomeDir+string(os.PathSeparator), 1)
	} else {
		return path
	}
}

func readDatadogKeys(configFilePath string, keysChan chan<- string) {
	defer func() {
		if err := recover(); err != nil {
			log.Printf("error: datadog configuration missing or inaccessible.")
			log.Fatalf("error: %v", err)
		}
	}()

	keys := &datadogKeys{
		ApiKey: os.Getenv("DATADOG_API_KEY"),
		AppKey: os.Getenv("DATADOG_APP_KEY"),
	}

	if keys.ApiKey == "" || keys.AppKey == "" {
		f, err := os.Open(expandPath(configFilePath))
		if err != nil {
			panic(err)
		}
		defer f.Close()

		err = json.NewDecoder(f).Decode(keys)
		if err != nil {
			panic(err)
		}
	}

	keysChan <- keys.ApiKey
	keysChan <- keys.AppKey
}

func validateType(metricType string) string {
	switch metricType {
	case "increment", "incr", "i", "c", "counter":
		return "counter"
	case "gauge", "g":
		return "gauge"
	default:
		panic(fmt.Sprintf("'%s' is not a valid metric type. must be one of 'counter', or 'gauge'.", metricType))
	}
}

func createDataPoint(value string) datadog.DataPoint {
	converted, err := strconv.ParseFloat(value, 64)
	if err != nil {
		panic(err)
	}
	now := float64(time.Now().Unix())
	return datadog.DataPoint([2]float64{now, converted})
}

func validateAndConvertPoints(data []string) (points []datadog.DataPoint) {
	switch len(data) {
	case 0:
		panic("no value(s)")
	default:
		for _, v := range data {
			points = append(points, createDataPoint(v))
		}
	}
	return points
}

func parseArgs(args []string, tags string) datadog.Metric {
	if len(args) < 3 {
		panic("not enough arguments. usage: datadog TYPE METRIC VALUE(S)...")
	}

	return datadog.Metric{
		Type:   validateType(args[0]),
		Metric: args[1],
		Points: validateAndConvertPoints(args[2:]),
		Tags:   strings.Split(tags, ","),
	}
}

func main() {
	defer func() {
		if err := recover(); err != nil {
			log.Fatalf("error: %v", err)
		}
	}()

	// datadog gauge my_metric values...
	var dryRun bool
	flag.BoolVar(&dryRun, "dry-run", false, "Don't send data to datadog.")
	var tags string
	flag.StringVar(&tags, "tags", "", "Tags to add to this metric, e.g. 'key:value,key2:value2'.")
	var configFile string
	flag.StringVar(&configFile, "conf", "~/.datadogrc", "Datadog app and api keys")
	flag.Parse()

	keysChan := make(chan string, 2)
	go readDatadogKeys(configFile, keysChan)

	metric := parseArgs(flag.Args(), tags)

	if dryRun {
		encoded, err := json.Marshal(metric)
		if err != nil {
			panic(err)
		}
		log.Println(string(encoded))
	} else {
		client := datadog.NewClient(<-keysChan, <-keysChan)
		err := client.PostMetrics([]datadog.Metric{metric})
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package sdsmeta

import (
	//"code.google.com/p/leveldb-go/leveldb"
	//"code.google.com/p/leveldb-go/leveldb/db"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"strconv"
)

// List of column buffers
type SDataFramePartitionCols struct {
	Rows       int
	path       string
	colBuffers []SDataFrameColBuffer
}

// Single Buffer for a Partition of a Column
type SDataFrameColBuffer struct {
	Column           Sdscolumndef
	Path             string
	PartitionKey     string
	IsNA             string
	DataBufferInt32  []int32
	DataBufferFloat  []float32
	DataBufferDouble []float64
	DataBufferInt64  []int64
	DataBufferByte   []byte
	DataBufferFactor []int32
	DataBufferBool   []bool
}

func (pCols *SDataFramePartitionCols) setRow(row int, record []string) (err error) {
	//fmt.Printf("Set row ... %d\n", row)
	for i := 0; i < len(pCols.colBuffers); i++ {
		err = pCols.colBuffers[i].setCol(row, record[i])
		if err != nil {
			return err
		}
	}
	return nil
}

func (pCols *SDataFramePartitionCols) FlushToDisk() (err error) {
	err = nil

	if pCols.Rows == 0 {
		return
	}

	for _, colBuffer := range pCols.colBuffers {
		err = colBuffer.FlushToDisk(pCols.Rows)
		if err != nil {
			return err
		}
	}

	return pCols.createPartitionDB()
}

func (pCols *SDataFramePartitionCols) createPartitionDB() (err error) {
	err = nil

	var dbh partitionStorer

	//dbh, err = openLevelDBStore(pCols.path)
	dbh, err = openMsgPackStore(pCols.path)

	//fname := pCols.path + DF_SEP + DF_PDB

	//var dbh *leveldb.DB
	//dbh, err = leveldb.Open(fname, nil)
	if err != nil {
		return err
	}

	defer func() {
		if err = dbh.close(); err != nil {
			panic(err)
		}
	}()

	//nrows := make([]byte, 8)
	//binary.PutVarint(nrows, int64(pCols.Rows))
	dbh.put(DB_NROW, int64(pCols.Rows))
	//opts := db.WriteOptions{Sync: true}
	//err = dbh.Set([]byte(DB_NROW), nrows, &opts)

	return err
}

func (pCols *SDataFramePartitionCols) String() (s string) {
	s = fmt.Sprintf("List of columns of length %d", len(pCols.colBuffers))
	return
}

func (pCols *SDataFrameColBuffer) String() (s string) {
	s = fmt.Sprintf("Column: %s (%s) Attributes: %s. Partition %s:%s",
		pCols.Column.Colname,
		pCols.Column.Coltype,
		pCols.Column.Attributes,
		pCols.PartitionKey,
		pCols.Path)
	return
}

func (colBuffer *SDataFrameColBuffer) allocateBuffer(nrows int32) {
	switch SDF_ColType_Keywords[colBuffer.Column.Coltype] {
	case SDFK_Integer32:
		fallthrough
	case SDFK_Factor:
		colBuffer.DataBufferInt32 = make([]int32, nrows)
	case SDFK_Float:
		colBuffer.DataBufferFloat = make([]float32, nrows)
	case SDFK_Double:
		colBuffer.DataBufferDouble = make([]float64, nrows)
	case SDFK_Date:
		fallthrough
	case SDFK_Integer64:
		colBuffer.DataBufferInt64 = make([]int64, nrows)
	case SDFK_Character:
		colBuffer.DataBufferByte = make([]byte, nrows)
	case SDFK_Boolean:
		colBuffer.DataBufferBool = make([]bool, nrows)
	default:
		panic(fmt.Sprintf("Unknown column type %s\n",
			colBuffer.String()))
	}
	return
}

func (col *SDataFrameColBuffer) setCol(row int, value string) (err error) {

	err = nil

	if value == col.IsNA {
		// For now just ignore
		return nil
	}

	switch SDF_ColType_Keywords[col.Column.Coltype] {
	case SDFK_Integer32:
		var i int64
		i, err = strconv.ParseInt(value, 10, 32)
		col.DataBufferInt32[row] = int32(i)
	case SDFK_Factor:
		// Not implemented
	case SDFK_Float:
		var f float64
		f, err = strconv.ParseFloat(value, 32)
		col.DataBufferFloat[row] = float32(f)
	case SDFK_Double:
		col.DataBufferDouble[row], err = strconv.ParseFloat(value, 64)
	case SDFK_Date:
		//
	case SDFK_Integer64:
		col.DataBufferInt64[row], err = strconv.ParseInt(value, 10, 64)
	case SDFK_Character:
		//
	case SDFK_Boolean:
		//
	default:
		panic(fmt.Sprintf("Unknown column type %s for row %d value %s of %s\n",
			col.Column.Coltype,
			row,
			value,
			col))
	}

	if err != nil {
		var serr SError
		serr.msg = fmt.Sprintf("%s: error: %s",
			col.String(),
			err)
		err = &serr
	}

	return
}

func (colBuffer *SDataFrameColBuffer) FlushToDisk(rows int) (err error) {
	err = nil

	var fo *os.File
	fname := colBuffer.Path + DF_SEP + colBuffer.Column.Colname
	//fmt.Printf("write ... %s\n", fname)
	fo, err = os.Create(fname)
	if err != nil {
		return err
	}
	defer func() {
		if err = fo.Close(); err != nil {
			panic(err)
		}
	}()
	var out io.Writer
	out = io.Writer(fo)

	switch SDF_ColType_Keywords[colBuffer.Column.Coltype] {
	case SDFK_Integer32:
		buff := colBuffer.DataBufferInt32[0:rows]
		binary.Write(out, binary.LittleEndian, buff)
	case SDFK_Factor:
		//
	case SDFK_Float:
		//colBuffer.DataBufferFloat = make([]float32, nrows)
	case SDFK_Double:
		//colBuffer.DataBufferDouble = make([]float64, nrows)
	case SDFK_Date:
		fallthrough
	case SDFK_Integer64:
		//colBuffer.DataBufferInt64 = make([]int64, nrows)
	case SDFK_Character:
		//colBuffer.DataBufferByte = make([]byte, nrows)
	case SDFK_Boolean:
		//colBuffer.DataBufferBool = make([]bool, nrows)
	default:
		panic(fmt.Sprintf("Unknown column type %s\n",
			colBuffer.String()))
	}

	return err
}
<commit_msg>Added space for R headers to data sections<commit_after>package sdsmeta

import (
	//"code.google.com/p/leveldb-go/leveldb"
	//"code.google.com/p/leveldb-go/leveldb/db"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"strconv"
)

const HEADER_PAD_BYTES = 128 // Used for R SEXPREC_ALIGN pad

// List of column buffers
type SDataFramePartitionCols struct {
	Rows       int
	path       string
	colBuffers []SDataFrameColBuffer
}

// Single Buffer for a Partition of a Column
type SDataFrameColBuffer struct {
	Column           Sdscolumndef
	Path             string
	PartitionKey     string
	IsNA             string
	DataBufferInt32  []int32
	DataBufferFloat  []float32
	DataBufferDouble []float64
	DataBufferInt64  []int64
	DataBufferByte   []byte
	DataBufferFactor []int32
	DataBufferBool   []bool
}

func (pCols *SDataFramePartitionCols) setRow(row int, record []string) (err error) {
	//fmt.Printf("Set row ... %d\n", row)
	for i := 0; i < len(pCols.colBuffers); i++ {
		err = pCols.colBuffers[i].setCol(row, record[i])
		if err != nil {
			return err
		}
	}
	return nil
}

func (pCols *SDataFramePartitionCols) FlushToDisk() (err error) {
	err = nil

	if pCols.Rows == 0 {
		return
	}

	for _, colBuffer := range pCols.colBuffers {
		err = colBuffer.FlushToDisk(pCols.Rows)
		if err != nil {
			return err
		}
	}

	return pCols.createPartitionDB()
}

func (pCols *SDataFramePartitionCols) createPartitionDB() (err error) {
	err = nil

	var dbh partitionStorer

	//dbh, err = openLevelDBStore(pCols.path)
	dbh, err = openMsgPackStore(pCols.path)

	//fname := pCols.path + DF_SEP + DF_PDB

	//var dbh *leveldb.DB
	//dbh, err = leveldb.Open(fname, nil)
	if err != nil {
		return err
	}

	defer func() {
		if err = dbh.close(); err != nil {
			panic(err)
		}
	}()

	//nrows := make([]byte, 8)
	//binary.PutVarint(nrows, int64(pCols.Rows))
	dbh.put(DB_NROW, int64(pCols.Rows))
	//opts := db.WriteOptions{Sync: true}
	//err = dbh.Set([]byte(DB_NROW), nrows, &opts)

	return err
}

func (pCols *SDataFramePartitionCols) String() (s string) {
	s = fmt.Sprintf("List of columns of length %d", len(pCols.colBuffers))
	return
}

func (pCols *SDataFrameColBuffer) String() (s string) {
	s = fmt.Sprintf("Column: %s (%s) Attributes: %s. Partition %s:%s",
		pCols.Column.Colname,
		pCols.Column.Coltype,
		pCols.Column.Attributes,
		pCols.PartitionKey,
		pCols.Path)
	return
}

func (colBuffer *SDataFrameColBuffer) allocateBuffer(nrows int32) {
	switch SDF_ColType_Keywords[colBuffer.Column.Coltype] {
	case SDFK_Integer32:
		fallthrough
	case SDFK_Factor:
		colBuffer.DataBufferInt32 = make([]int32, nrows)
	case SDFK_Float:
		colBuffer.DataBufferFloat = make([]float32, nrows)
	case SDFK_Double:
		colBuffer.DataBufferDouble = make([]float64, nrows)
	case SDFK_Date:
		fallthrough
	case SDFK_Integer64:
		colBuffer.DataBufferInt64 = make([]int64, nrows)
	case SDFK_Character:
		colBuffer.DataBufferByte = make([]byte, nrows)
	case SDFK_Boolean:
		colBuffer.DataBufferBool = make([]bool, nrows)
	default:
		panic(fmt.Sprintf("Unknown column type %s\n",
			colBuffer.String()))
	}
	return
}

func (col *SDataFrameColBuffer) setCol(row int, value string) (err error) {

	err = nil

	if value == col.IsNA {
		// For now just ignore
		return nil
	}

	switch SDF_ColType_Keywords[col.Column.Coltype] {
	case SDFK_Integer32:
		var i int64
		i, err = strconv.ParseInt(value, 10, 32)
		col.DataBufferInt32[row] = int32(i)
	case SDFK_Factor:
		// Not implemented
	case SDFK_Float:
		var f float64
		f, err = strconv.ParseFloat(value, 32)
		col.DataBufferFloat[row] = float32(f)
	case SDFK_Double:
		col.DataBufferDouble[row], err = strconv.ParseFloat(value, 64)
	case SDFK_Date:
		//
	case SDFK_Integer64:
		col.DataBufferInt64[row], err = strconv.ParseInt(value, 10, 64)
	case SDFK_Character:
		//
	case SDFK_Boolean:
		//
	default:
		panic(fmt.Sprintf("Unknown column type %s for row %d value %s of %s\n",
			col.Column.Coltype,
			row,
			value,
			col))
	}

	if err != nil {
		var serr SError
		serr.msg = fmt.Sprintf("%s: error: %s",
			col.String(),
			err)
		err = &serr
	}

	return
}

func (colBuffer *SDataFrameColBuffer) FlushToDisk(rows int) (err error) {
	err = nil

	var fo *os.File
	fname := colBuffer.Path + DF_SEP + colBuffer.Column.Colname
	//fmt.Printf("write ... %s\n", fname)
	fo, err = os.Create(fname)
	if err != nil {
		return err
	}
	defer func() {
		if err = fo.Close(); err != nil {
			panic(err)
		}
	}()
	var out io.Writer
	out = io.Writer(fo)

	// Write out header for R
	var hdrpad [HEADER_PAD_BYTES]byte
	var n int
	n, err = out.Write(hdrpad[:])
	if n != HEADER_PAD_BYTES {
		var serr SError
		serr.msg = "Error while writing out Pad bytes"
		err = &serr
		return err
	}
	if err != nil {
		return err
	}

	switch SDF_ColType_Keywords[colBuffer.Column.Coltype] {
	case SDFK_Integer32:
		buff := colBuffer.DataBufferInt32[0:rows]
		binary.Write(out, binary.LittleEndian, buff)
	case SDFK_Factor:
		//
	case SDFK_Float:
		//colBuffer.DataBufferFloat = make([]float32, nrows)
	case SDFK_Double:
		//colBuffer.DataBufferDouble = make([]float64, nrows)
	case SDFK_Date:
		fallthrough
	case SDFK_Integer64:
		//colBuffer.DataBufferInt64 = make([]int64, nrows)
	case SDFK_Character:
		//colBuffer.DataBufferByte = make([]byte, nrows)
	case SDFK_Boolean:
		//colBuffer.DataBufferBool = make([]bool, nrows)
	default:
		panic(fmt.Sprintf("Unknown column type %s\n",
			colBuffer.String()))
	}

	return err
}
<|endoftext|>


<commit_before>package sparse

import (
	"math"

	"github.com/gonum/floats"
	"github.com/james-bowman/sparse/blas"
	"gonum.org/v1/gonum/mat"
)

var (
	_ Sparser    = (*Vector)(nil)
	_ mat.Matrix = (*Vector)(nil)
	_ mat.Vector = (*Vector)(nil)
)

// Vector is a sparse vector format.  It implements the mat.Vector
// interface but is optimised for sparsely populated vectors where
// most of the elements contain zero values by only storing and
// processing the non-zero values.  The format is similar to the
// triplet format used by COO matrices (and CSR/CSC) but only uses
// 2 arrays because the vector is 1 dimensional rather than 2.
type Vector struct {
	len  int
	ind  []int
	data []float64
}

// NewVector returns a new sparse vector of length len with
// elements specified by ind[] containing the values conatined
// within data.  Vector will reuse the same storage as the slices
// passed in and so any changes to the vector will be reflected
// in the slices and vice versa.
func NewVector(len int, ind []int, data []float64) *Vector {
	return &Vector{
		len:  len,
		ind:  ind,
		data: data,
	}
}

// Dims returns the dimensions of the vector.  This will be
// equivalent to Len(), 1
func (v *Vector) Dims() (r, c int) {
	return v.len, 1
}

// At returns the element at r, c.  At will panic if c != 0.
func (v *Vector) At(r, c int) float64 {
	if c != 0 {
		panic(mat.ErrColAccess)
	}
	return v.AtVec(r)
}

// T returns the transpose of the receiver.
func (v *Vector) T() mat.Matrix {
	return mat.TransposeVec{Vector: v}
}

// NNZ returns the number of non-zero elements in the vector.
func (v *Vector) NNZ() int {
	return len(v.data)
}

// AtVec returns the i'th element of the Vector.
func (v *Vector) AtVec(i int) float64 {
	if i < 0 || i >= v.len {
		panic(mat.ErrRowAccess)
	}

	for j := 0; j < len(v.ind); j++ {
		if v.ind[j] == i {
			return v.data[j]
		}
	}
	return 0.0
}

// Len returns the length of the vector
func (v *Vector) Len() int {
	return v.len
}

// DoNonZero calls the function fn for each of the non-zero elements of the receiver.
// The function fn takes a row/column index and the element value of the receiver at
// (i, j).
func (v *Vector) DoNonZero(fn func(i int, j int, v float64)) {
	for i := 0; i < len(v.ind); i++ {
		fn(v.ind[i], 0, v.data[i])
	}
}

// Gather gathers the entries from the supplied mat.VecDense structure
// that have corresponding non-zero entries in the receiver into the
// receiver.  The method will panic if denseVector is not the same
// length as the receiver.
func (v *Vector) Gather(denseVector *mat.VecDense) {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}
	vec := denseVector.RawVector()
	blas.Dusga(vec.Data, vec.Inc, v.data, v.ind)
}

// GatherAndZero gathers the entries from the supplied mat.VecDense
// structure that have corresponding non-zero entries in the receiver
// into the receiver and then zeros those entries in denseVector.
// The method will panic if denseVector is not the same length
// as the receiver.
func (v *Vector) GatherAndZero(denseVector *mat.VecDense) {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}

	vec := denseVector.RawVector()
	blas.Dusgz(vec.Data, vec.Inc, v.data, v.ind)
}

// Scatter scatters elements from the receiver into the supplied mat.VecDense
// structure, denseVector and returns a pointer to it.  The method will panic
// if denseVector is not the same length as the receiver (unless it is nil)
func (v *Vector) Scatter(denseVector *mat.VecDense) *mat.VecDense {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}
	vec := denseVector.RawVector()
	blas.Dussc(v.data, vec.Data, vec.Inc, v.ind)
	return denseVector
}

// CloneVec clones the supplied mat.Vector, a into the receiver, overwriting
// the previous values of the receiver.  If the receiver is of a different
// length from a, it will be resized to accomodate the values from a.
func (v *Vector) CloneVec(a mat.Vector) {
	if v == a {
		return
	}
	v.len = a.Len()

	if sv, isSparse := a.(*Vector); isSparse {
		size := len(sv.ind)
		if size > cap(v.ind) {
			v.ind = make([]int, size)
			v.data = make([]float64, size)
		} else {
			v.ind = v.ind[:size]
			v.data = v.data[:size]
		}
		for i, val := range sv.ind {
			v.ind[i] = val
			v.data[i] = sv.data[i]
		}
		return
	}

	v.ind = v.ind[:0]
	v.data = v.data[:0]

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i)
		if val != 0 {
			v.ind = append(v.ind, i)
			v.data = append(v.data, val)
		}
	}
}

// ToDense converts the sparse vector to a dense vector
// The returned dense matrix is a new copy of the receiver.
func (v *Vector) ToDense() *mat.VecDense {
	return v.Scatter(mat.NewVecDense(v.len, nil))
}

// AddVec adds the vectors a and b, placing the result in the receiver.
// AddVec will panic if a and b are not the same length.  If a and b
// are both sparse Vector vectors then AddVec will only process the
// non-zero elements.
func (v *Vector) AddVec(a, b mat.Vector) {
	ar := a.Len()
	br := b.Len()

	if ar != br {
		panic(mat.ErrShape)
	}

	v.len = ar

	// Sparse specific optimised implementation
	sa, aIsSparse := a.(*Vector)
	sb, bIsSparse := b.(*Vector)
	if aIsSparse && bIsSparse {
		v.addVecSparse(sa, 1, sb)
		return
	}

	// fallback to basic vector addition with no optimisations
	// for sparseness
	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		p := a.AtVec(i) + b.AtVec(i)
		if p != 0 {
			ind = append(ind, i)
			data = append(data, p)
		}
	}

	v.commitWorkspace(ind, data)
}

// addVecSparse2 adds the vectors a and alpha*b.  This method is
// optimised for processing sparse Vector vectors and only processes
// non-zero elements.
func (v *Vector) addVecSparse(a *Vector, alpha float64, b *Vector) {
	ind, data := v.createWorkspace(0, false)

	spa := NewSPA(a.len)

	spa.Scatter(a.data, a.ind, 1, &ind)
	spa.Scatter(b.data, b.ind, alpha, &ind)
	spa.Gather(&data, &ind)

	v.commitWorkspace(ind, data)
}

// ScaleVec scales the vector a by alpha, placing the result in the
// receiver.
func (v *Vector) ScaleVec(alpha float64, a mat.Vector) {
	v.len = a.Len()

	if alpha == 0 {
		v.ind = v.ind[:0]
		v.data = v.data[:0]
		return
	}

	if s, isSparse := a.(*Vector); isSparse {
		ind, data := v.createWorkspace(s.NNZ(), false)
		copy(ind, s.ind)
		for i, val := range s.data {
			data[i] = alpha * val
		}
		v.commitWorkspace(ind, data)
		return
	}

	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i)
		if val != 0 {
			ind = append(ind, i)
			data = append(data, alpha*val)
		}
	}
	v.commitWorkspace(ind, data)
}

// AddScaledVec adds the vectors a and alpha*b, placing the result
// in the receiver.  AddScaledVec will panic if a and b are not the
// same length.
func (v *Vector) AddScaledVec(a mat.Vector, alpha float64, b mat.Vector) {
	ar := a.Len()
	br := b.Len()

	if ar != br {
		panic(mat.ErrShape)
	}

	v.len = ar

	// Sparse specific optimised implementation
	sa, aIsSparse := a.(*Vector)
	sb, bIsSparse := b.(*Vector)
	if aIsSparse && bIsSparse {
		v.addVecSparse(sa, alpha, sb)
		return
	}

	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i) + alpha*b.AtVec(i)
		if val != 0 {
			ind = append(ind, i)
			data = append(data, val)
		}
	}
	v.commitWorkspace(ind, data)
}

// Norm calculates the Norm of the vector only processing the
// non-zero elements.
// See Normer interface for more details.
func (v *Vector) Norm(L float64) float64 {
	if v.len == 0 || len(v.data) == 0 {
		return 0
	}
	if L == 2 {
		twoNorm := math.Pow(math.Abs(v.data[0]), 2)
		for i := 1; i < len(v.data); i++ {
			twoNorm += math.Pow(v.data[i], 2)
		}
		return math.Sqrt(twoNorm)
	}
	return floats.Norm(v.data, L)
}

// Dot returns the sum of the element-wise product (dot product) of a and b.
// Dot panics if the matrix sizes are unequal.  For sparse vectors, Dot will
// only process non-zero elements otherwise this method simply delegates to
// mat.Dot()
func Dot(a, b mat.Vector) float64 {
	if a.Len() != b.Len() {
		panic(mat.ErrShape)
	}

	as, aIsSparse := a.(*Vector)
	bs, bIsSparse := b.(*Vector)

	if aIsSparse {
		if bIsSparse {
			buf := getFloats(bs.len, true)
			blas.Dussc(bs.data, buf, 1, bs.ind)
			val := blas.Dusdot(as.data, as.ind, buf, 1)
			putFloats(buf)
			return val
		}
		if bdense, bIsDense := b.(mat.RawVectorer); bIsDense {
			raw := bdense.RawVector()
			return blas.Dusdot(as.data, as.ind, raw.Data, raw.Inc)
		}
		return dotSparse(as, b)
	}
	if bIsSparse {
		if adense, aIsDense := a.(mat.RawVectorer); aIsDense {
			raw := adense.RawVector()
			return blas.Dusdot(bs.data, bs.ind, raw.Data, raw.Inc)
		}
		return dotSparse(bs, a)
	}
	return mat.Dot(a, b)
}

// dotSparse returns the sum of the element-wise multiplication
// of a and b where a is sparse and b is any implementation of
// mat.Vector.
func dotSparse(a *Vector, b mat.Vector) float64 {
	var result float64
	for i, ind := range a.ind {
		result += a.data[i] * b.AtVec(ind)
	}
	return result
}

// createWorkspace creates a temporary workspace to store the result
// of vector operations avoiding the issue of mutating operands mid
// operation where they overlap with the receiver
// e.g.
//	result.AddVec(result, a)
// createWorkspace will attempt to reuse previously allocated memory
// for the temporary workspace where ever possible to avoid allocating
// memory and placing strain on GC.
func (v *Vector) createWorkspace(size int, zero bool) ([]int, []float64) {
	ind := getInts(size, zero)
	data := getFloats(size, zero)

	return ind, data
}

// commitWorkspace commits a temporary workspace previously created
// with createWorkspace.  This has the effect of updaing the receiver
// with the values from the temporary workspace and returning the
// memory used by the workspace to the pool for other operations to
// reuse.
func (v *Vector) commitWorkspace(indexes []int, data []float64) {
	v.ind, indexes = indexes, v.ind
	v.data, data = data, v.data
	putInts(indexes)
	putFloats(data)
}
<commit_msg>improved performance of Norm() method<commit_after>package sparse

import (
	"math"

	"github.com/gonum/floats"
	"github.com/james-bowman/sparse/blas"
	"gonum.org/v1/gonum/mat"
)

var (
	_ Sparser    = (*Vector)(nil)
	_ mat.Matrix = (*Vector)(nil)
	_ mat.Vector = (*Vector)(nil)
)

// Vector is a sparse vector format.  It implements the mat.Vector
// interface but is optimised for sparsely populated vectors where
// most of the elements contain zero values by only storing and
// processing the non-zero values.  The format is similar to the
// triplet format used by COO matrices (and CSR/CSC) but only uses
// 2 arrays because the vector is 1 dimensional rather than 2.
type Vector struct {
	len  int
	ind  []int
	data []float64
}

// NewVector returns a new sparse vector of length len with
// elements specified by ind[] containing the values conatined
// within data.  Vector will reuse the same storage as the slices
// passed in and so any changes to the vector will be reflected
// in the slices and vice versa.
func NewVector(len int, ind []int, data []float64) *Vector {
	return &Vector{
		len:  len,
		ind:  ind,
		data: data,
	}
}

// Dims returns the dimensions of the vector.  This will be
// equivalent to Len(), 1
func (v *Vector) Dims() (r, c int) {
	return v.len, 1
}

// At returns the element at r, c.  At will panic if c != 0.
func (v *Vector) At(r, c int) float64 {
	if c != 0 {
		panic(mat.ErrColAccess)
	}
	return v.AtVec(r)
}

// T returns the transpose of the receiver.
func (v *Vector) T() mat.Matrix {
	return mat.TransposeVec{Vector: v}
}

// NNZ returns the number of non-zero elements in the vector.
func (v *Vector) NNZ() int {
	return len(v.data)
}

// AtVec returns the i'th element of the Vector.
func (v *Vector) AtVec(i int) float64 {
	if i < 0 || i >= v.len {
		panic(mat.ErrRowAccess)
	}

	for j := 0; j < len(v.ind); j++ {
		if v.ind[j] == i {
			return v.data[j]
		}
	}
	return 0.0
}

// Len returns the length of the vector
func (v *Vector) Len() int {
	return v.len
}

// DoNonZero calls the function fn for each of the non-zero elements of the receiver.
// The function fn takes a row/column index and the element value of the receiver at
// (i, j).
func (v *Vector) DoNonZero(fn func(i int, j int, v float64)) {
	for i := 0; i < len(v.ind); i++ {
		fn(v.ind[i], 0, v.data[i])
	}
}

// Gather gathers the entries from the supplied mat.VecDense structure
// that have corresponding non-zero entries in the receiver into the
// receiver.  The method will panic if denseVector is not the same
// length as the receiver.
func (v *Vector) Gather(denseVector *mat.VecDense) {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}
	vec := denseVector.RawVector()
	blas.Dusga(vec.Data, vec.Inc, v.data, v.ind)
}

// GatherAndZero gathers the entries from the supplied mat.VecDense
// structure that have corresponding non-zero entries in the receiver
// into the receiver and then zeros those entries in denseVector.
// The method will panic if denseVector is not the same length
// as the receiver.
func (v *Vector) GatherAndZero(denseVector *mat.VecDense) {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}

	vec := denseVector.RawVector()
	blas.Dusgz(vec.Data, vec.Inc, v.data, v.ind)
}

// Scatter scatters elements from the receiver into the supplied mat.VecDense
// structure, denseVector and returns a pointer to it.  The method will panic
// if denseVector is not the same length as the receiver (unless it is nil)
func (v *Vector) Scatter(denseVector *mat.VecDense) *mat.VecDense {
	if v.len != denseVector.Len() {
		panic(mat.ErrShape)
	}
	vec := denseVector.RawVector()
	blas.Dussc(v.data, vec.Data, vec.Inc, v.ind)
	return denseVector
}

// CloneVec clones the supplied mat.Vector, a into the receiver, overwriting
// the previous values of the receiver.  If the receiver is of a different
// length from a, it will be resized to accomodate the values from a.
func (v *Vector) CloneVec(a mat.Vector) {
	if v == a {
		return
	}
	v.len = a.Len()

	if sv, isSparse := a.(*Vector); isSparse {
		size := len(sv.ind)
		if size > cap(v.ind) {
			v.ind = make([]int, size)
			v.data = make([]float64, size)
		} else {
			v.ind = v.ind[:size]
			v.data = v.data[:size]
		}
		for i, val := range sv.ind {
			v.ind[i] = val
			v.data[i] = sv.data[i]
		}
		return
	}

	v.ind = v.ind[:0]
	v.data = v.data[:0]

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i)
		if val != 0 {
			v.ind = append(v.ind, i)
			v.data = append(v.data, val)
		}
	}
}

// ToDense converts the sparse vector to a dense vector
// The returned dense matrix is a new copy of the receiver.
func (v *Vector) ToDense() *mat.VecDense {
	return v.Scatter(mat.NewVecDense(v.len, nil))
}

// AddVec adds the vectors a and b, placing the result in the receiver.
// AddVec will panic if a and b are not the same length.  If a and b
// are both sparse Vector vectors then AddVec will only process the
// non-zero elements.
func (v *Vector) AddVec(a, b mat.Vector) {
	ar := a.Len()
	br := b.Len()

	if ar != br {
		panic(mat.ErrShape)
	}

	v.len = ar

	// Sparse specific optimised implementation
	sa, aIsSparse := a.(*Vector)
	sb, bIsSparse := b.(*Vector)
	if aIsSparse && bIsSparse {
		v.addVecSparse(sa, 1, sb)
		return
	}

	// fallback to basic vector addition with no optimisations
	// for sparseness
	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		p := a.AtVec(i) + b.AtVec(i)
		if p != 0 {
			ind = append(ind, i)
			data = append(data, p)
		}
	}

	v.commitWorkspace(ind, data)
}

// addVecSparse2 adds the vectors a and alpha*b.  This method is
// optimised for processing sparse Vector vectors and only processes
// non-zero elements.
func (v *Vector) addVecSparse(a *Vector, alpha float64, b *Vector) {
	ind, data := v.createWorkspace(0, false)

	spa := NewSPA(a.len)

	spa.Scatter(a.data, a.ind, 1, &ind)
	spa.Scatter(b.data, b.ind, alpha, &ind)
	spa.Gather(&data, &ind)

	v.commitWorkspace(ind, data)
}

// ScaleVec scales the vector a by alpha, placing the result in the
// receiver.
func (v *Vector) ScaleVec(alpha float64, a mat.Vector) {
	v.len = a.Len()

	if alpha == 0 {
		v.ind = v.ind[:0]
		v.data = v.data[:0]
		return
	}

	if s, isSparse := a.(*Vector); isSparse {
		ind, data := v.createWorkspace(s.NNZ(), false)
		copy(ind, s.ind)
		for i, val := range s.data {
			data[i] = alpha * val
		}
		v.commitWorkspace(ind, data)
		return
	}

	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i)
		if val != 0 {
			ind = append(ind, i)
			data = append(data, alpha*val)
		}
	}
	v.commitWorkspace(ind, data)
}

// AddScaledVec adds the vectors a and alpha*b, placing the result
// in the receiver.  AddScaledVec will panic if a and b are not the
// same length.
func (v *Vector) AddScaledVec(a mat.Vector, alpha float64, b mat.Vector) {
	ar := a.Len()
	br := b.Len()

	if ar != br {
		panic(mat.ErrShape)
	}

	v.len = ar

	// Sparse specific optimised implementation
	sa, aIsSparse := a.(*Vector)
	sb, bIsSparse := b.(*Vector)
	if aIsSparse && bIsSparse {
		v.addVecSparse(sa, alpha, sb)
		return
	}

	ind, data := v.createWorkspace(0, false)

	for i := 0; i < v.len; i++ {
		val := a.AtVec(i) + alpha*b.AtVec(i)
		if val != 0 {
			ind = append(ind, i)
			data = append(data, val)
		}
	}
	v.commitWorkspace(ind, data)
}

// Norm calculates the Norm of the vector only processing the
// non-zero elements.
// See Normer interface for more details.
func (v *Vector) Norm(L float64) float64 {
	if L == 2 {
		return math.Sqrt(Dot(v, v))
	}
	return floats.Norm(v.data, L)
}

// Dot returns the sum of the element-wise product (dot product) of a and b.
// Dot panics if the matrix sizes are unequal.  For sparse vectors, Dot will
// only process non-zero elements otherwise this method simply delegates to
// mat.Dot()
func Dot(a, b mat.Vector) float64 {
	if a.Len() != b.Len() {
		panic(mat.ErrShape)
	}

	as, aIsSparse := a.(*Vector)
	bs, bIsSparse := b.(*Vector)

	if aIsSparse {
		if bIsSparse {
			buf := getFloats(bs.len, true)
			blas.Dussc(bs.data, buf, 1, bs.ind)
			val := blas.Dusdot(as.data, as.ind, buf, 1)
			putFloats(buf)
			return val
		}
		if bdense, bIsDense := b.(mat.RawVectorer); bIsDense {
			raw := bdense.RawVector()
			return blas.Dusdot(as.data, as.ind, raw.Data, raw.Inc)
		}
		return dotSparse(as, b)
	}
	if bIsSparse {
		if adense, aIsDense := a.(mat.RawVectorer); aIsDense {
			raw := adense.RawVector()
			return blas.Dusdot(bs.data, bs.ind, raw.Data, raw.Inc)
		}
		return dotSparse(bs, a)
	}
	return mat.Dot(a, b)
}

// dotSparse returns the sum of the element-wise multiplication
// of a and b where a is sparse and b is any implementation of
// mat.Vector.
func dotSparse(a *Vector, b mat.Vector) float64 {
	var result float64
	for i, ind := range a.ind {
		result += a.data[i] * b.AtVec(ind)
	}
	return result
}

// createWorkspace creates a temporary workspace to store the result
// of vector operations avoiding the issue of mutating operands mid
// operation where they overlap with the receiver
// e.g.
//	result.AddVec(result, a)
// createWorkspace will attempt to reuse previously allocated memory
// for the temporary workspace where ever possible to avoid allocating
// memory and placing strain on GC.
func (v *Vector) createWorkspace(size int, zero bool) ([]int, []float64) {
	ind := getInts(size, zero)
	data := getFloats(size, zero)

	return ind, data
}

// commitWorkspace commits a temporary workspace previously created
// with createWorkspace.  This has the effect of updaing the receiver
// with the values from the temporary workspace and returning the
// memory used by the workspace to the pool for other operations to
// reuse.
func (v *Vector) commitWorkspace(indexes []int, data []float64) {
	v.ind, indexes = indexes, v.ind
	v.data, data = data, v.data
	putInts(indexes)
	putFloats(data)
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Since focus would make the exit code to be non-zero,
// following testing is uneffective by default.
var _ = XDescribe("Focused(modify code to run effective testing)", func() {
	Context("FDescribe()", func() {
		var fdescribe = 1
		NewGinkgoBuilder("ToFDescribe()").
			It("Sample It", func() {
				fdescribe = 1
			}).
			ToFDescribe()
		It("Flag for FDescribe() should be 1", func() {
			Expect(fdescribe).To(Equal(1))
		})
	})

	/**
	 * FContext()
	 */
	var fcontext = 0
	NewGinkgoBuilder("ToFContext()").
		It("Sample It", func() {
			fcontext = 1
		}).
		ToFContext()
	It("Flag for FContext() should be 1", func() {
		Expect(fcontext).To(Equal(1))
	})
	// :~)

	Context("FIt()", func() {
		fit := 0

		NewGinkgoBuilder("").
			FIt("For FIt", func() {
				fit = 1
			}).
			Expose()

		It("Flag for FIt() should be set to 1", func() {
			Expect(fit).To(Equal(1))
		})
	})

	Context("FSpecify()", func() {
		fspecify := 0

		NewGinkgoBuilder("").
			FSpecify("For FSpecify", func() {
				fspecify = 1
			}).
			Expose()

		It("Flag for FSpecify() should be set to 1", func() {
			Expect(fspecify).To(Equal(1))
		})
	})

	Context("FMeasure()", func() {
		fmeasure := 0

		NewGinkgoBuilder("").
			FMeasure("For FMeasure", func(b Benchmarker) {
				fmeasure = 1
			}, 1).
			Expose()

		It("Flag for FMeasure() should be set to 1", func() {
			Expect(fmeasure).To(Equal(1))
		})
	})
})

var _ = Describe("GinkgoBuilder: For To?Describe()", func() {
	var describe = 0
	NewGinkgoBuilder("ToDescribe()").
		It("Sample It", func() {
			describe = 1
		}).
		ToDescribe()
	It("Flag for Describe() should be 1", func() {
		Expect(describe).To(Equal(1))
	})

	NewGinkgoBuilder("ToPDescribe()").
		// This should be pending
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToPDescribe()

	NewGinkgoBuilder("ToXDescribe()").
		// This should be pending
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToXDescribe()
})

var _ = Describe("GinkgoBuilder: For To?Context()", func() {
	var context = 0
	NewGinkgoBuilder("ToContext()").
		It("Sample It", func() {
			context = 1
		}).
		ToContext()
	It("Flag for Context() should be 1", func() {
		Expect(context).To(Equal(1))
	})

	NewGinkgoBuilder("ToPContext()").
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToPContext()

	NewGinkgoBuilder("ToXContext()").
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToXContext()
})

var _ = Describe("GinkgoBuilder: For building of testing function", func() {
	Context("?It related functions",
		func() {
			var (
				it = 0
			)

			NewGinkgoBuilder("").
				It("For It", func() {
					it = 1
				}).
				PIt("For PIt", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				XIt("For XIt", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				Expose()

			It("Flag for It() should be set to 1", func() {
				Expect(it).To(Equal(1))
			})
		},
	)

	Context("?Specify related functions",
		func() {
			var (
				specify = 0
			)

			NewGinkgoBuilder("").
				Specify("For Specify", func() {
					specify = 1
				}).
				PSpecify("For PSpecify", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				XSpecify("For XSpecify", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				Expose()

			It("Flag for Specify() should be set to 1", func() {
				Expect(specify).To(Equal(1))
			})
		},
	)

	Context("?Measure related functions",
		func() {
			var (
				measure = 0
			)

			NewGinkgoBuilder("").
				Measure("For Measure", func(b Benchmarker) {
					measure = 1
				}, 1).
				PMeasure("For PMeasure", func(b Benchmarker) {
					// This should be pending
					Expect(1).To(Equal(2))
				}, 1).
				XMeasure("For XMeasure", func(b Benchmarker) {
					// This should be pending
					Expect(1).To(Equal(2))
				}, 1).
				Expose()

			It("Flag for Measure() should be set to 1", func() {
				Expect(measure).To(Equal(1))
			})
		},
	)
})

var _ = Describe("GinkgoBuilder: For BeforeEach/AfterEach/JustBeforeEach", func() {
	var (
		justBeforeEach = 0
		beforeEach     = 0
		afterEach      = 0
	)

	NewGinkgoBuilder("").
		BeforeEach(func() {
			beforeEach++
		}).
		AfterEach(func() {
			afterEach++
		}).
		JustBeforeEach(func() {
			justBeforeEach++
		}).
		Expose()

	It("1st testing. BeforeEach()/JustBeforeEach() get called 1 time", func() {
		Expect(beforeEach).To(Equal(1))
		Expect(justBeforeEach).To(Equal(1))
		Expect(afterEach).To(Equal(0))
	})
	It("2nd testing. BeforeEach()/JustBeforeEach() get called 2 times. AfterEach() get called once", func() {
		Expect(beforeEach).To(Equal(2))
		Expect(justBeforeEach).To(Equal(2))
		Expect(afterEach).To(Equal(1))
	})
})

var _ = Describe("GinkgoBuilder: For BeforeFirst/AfterLast", func() {
	var (
		beforeCalled = 0
		afterCalled  = 0
	)

	NewGinkgoBuilder("").
		BeforeFirst(func() {
			beforeCalled++
		}).
		AfterLast(func() {
			afterCalled++
		}).
		It("1st testing", func() {
			Expect(beforeCalled).To(Equal(1))
			Expect(afterCalled).To(Equal(0))
		}).
		It("2nd testing", func() {
			Expect(beforeCalled).To(Equal(1))
			Expect(afterCalled).To(Equal(0))
		}).
		Expose()

	It("BeforeFirst should get called only once", func() {
		Expect(beforeCalled).To(Equal(1))
	})
	It("AfterLast should get called only once", func() {
		Expect(afterCalled).To(Equal(1))
	})
})

var _ = Describe("GinkgoTable", func() {
	Context("Simple case", func() {
		counter := 0

		NewGinkgoTable().
			Exec(func(v int) {
				counter += v
			}).
			Exec(func(v int) {
				counter += v
			}).
			Case("case 1 for value 2", 2).
			Case(
				func(v int) string {
					return fmt.Sprintf("case 2 for value %d", v)
				}, 3,
			).
			Expose()

		It("The final counter should be #case * #exec_body", func() {
			Expect(counter).To(Equal(10))
		})
	})
})

var _ = Describe("Use GinkgoTable with GinkgoBuilder", func() {
	var (
		beforeFirst  = 0
		beforeEach   = 0
		tableExec    = 0
		externalExec = 0
	)

	NewGinkgoBuilder("Sample Context 1").
		BeforeFirst(func() {
			beforeFirst++
		}).
		BeforeEach(func() {
			beforeEach++
		}).
		Table(NewGinkgoTable().
			Exec(func() {
				tableExec++
			}).
			Case("Exec Case 1").
			Case("Exec Case 2"),
		).
		It("External It 1", func() {
			externalExec++
		}).
		ToContext()

	It("\"before first\" should be 1", func() {
		Expect(beforeFirst).To(Equal(1))
	})
	It("\"before each\" should be 3", func() {
		Expect(beforeEach).To(Equal(3))
	})
	It("\"table exec\" should be 2", func() {
		Expect(tableExec).To(Equal(2))
	})
	It("\"external exec\" should be 1", func() {
		Expect(externalExec).To(Equal(1))
	})
})
<commit_msg>[OWL-1934] Fix a logical error in a test case. Improper initial value.<commit_after>package builder

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Since focus would make the exit code to be non-zero,
// following testing is uneffective by default.
var _ = XDescribe("Focused(modify code to run effective testing)", func() {
	Context("FDescribe()", func() {
		var fdescribe = 0
		NewGinkgoBuilder("ToFDescribe()").
			It("Sample It", func() {
				fdescribe = 1
			}).
			ToFDescribe()
		It("Flag for FDescribe() should be 1", func() {
			Expect(fdescribe).To(Equal(1))
		})
	})

	/**
	 * FContext()
	 */
	var fcontext = 0
	NewGinkgoBuilder("ToFContext()").
		It("Sample It", func() {
			fcontext = 1
		}).
		ToFContext()
	It("Flag for FContext() should be 1", func() {
		Expect(fcontext).To(Equal(1))
	})
	// :~)

	Context("FIt()", func() {
		fit := 0

		NewGinkgoBuilder("").
			FIt("For FIt", func() {
				fit = 1
			}).
			Expose()

		It("Flag for FIt() should be set to 1", func() {
			Expect(fit).To(Equal(1))
		})
	})

	Context("FSpecify()", func() {
		fspecify := 0

		NewGinkgoBuilder("").
			FSpecify("For FSpecify", func() {
				fspecify = 1
			}).
			Expose()

		It("Flag for FSpecify() should be set to 1", func() {
			Expect(fspecify).To(Equal(1))
		})
	})

	Context("FMeasure()", func() {
		fmeasure := 0

		NewGinkgoBuilder("").
			FMeasure("For FMeasure", func(b Benchmarker) {
				fmeasure = 1
			}, 1).
			Expose()

		It("Flag for FMeasure() should be set to 1", func() {
			Expect(fmeasure).To(Equal(1))
		})
	})
})

var _ = Describe("GinkgoBuilder: For To?Describe()", func() {
	var describe = 0
	NewGinkgoBuilder("ToDescribe()").
		It("Sample It", func() {
			describe = 1
		}).
		ToDescribe()
	It("Flag for Describe() should be 1", func() {
		Expect(describe).To(Equal(1))
	})

	NewGinkgoBuilder("ToPDescribe()").
		// This should be pending
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToPDescribe()

	NewGinkgoBuilder("ToXDescribe()").
		// This should be pending
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToXDescribe()
})

var _ = Describe("GinkgoBuilder: For To?Context()", func() {
	var context = 0
	NewGinkgoBuilder("ToContext()").
		It("Sample It", func() {
			context = 1
		}).
		ToContext()
	It("Flag for Context() should be 1", func() {
		Expect(context).To(Equal(1))
	})

	NewGinkgoBuilder("ToPContext()").
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToPContext()

	NewGinkgoBuilder("ToXContext()").
		It("Sample It", func() {
			Expect(1).To(Equal(2))
		}).
		ToXContext()
})

var _ = Describe("GinkgoBuilder: For building of testing function", func() {
	Context("?It related functions",
		func() {
			var (
				it = 0
			)

			NewGinkgoBuilder("").
				It("For It", func() {
					it = 1
				}).
				PIt("For PIt", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				XIt("For XIt", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				Expose()

			It("Flag for It() should be set to 1", func() {
				Expect(it).To(Equal(1))
			})
		},
	)

	Context("?Specify related functions",
		func() {
			var (
				specify = 0
			)

			NewGinkgoBuilder("").
				Specify("For Specify", func() {
					specify = 1
				}).
				PSpecify("For PSpecify", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				XSpecify("For XSpecify", func() {
					// This should be pending
					Expect(1).To(Equal(2))
				}).
				Expose()

			It("Flag for Specify() should be set to 1", func() {
				Expect(specify).To(Equal(1))
			})
		},
	)

	Context("?Measure related functions",
		func() {
			var (
				measure = 0
			)

			NewGinkgoBuilder("").
				Measure("For Measure", func(b Benchmarker) {
					measure = 1
				}, 1).
				PMeasure("For PMeasure", func(b Benchmarker) {
					// This should be pending
					Expect(1).To(Equal(2))
				}, 1).
				XMeasure("For XMeasure", func(b Benchmarker) {
					// This should be pending
					Expect(1).To(Equal(2))
				}, 1).
				Expose()

			It("Flag for Measure() should be set to 1", func() {
				Expect(measure).To(Equal(1))
			})
		},
	)
})

var _ = Describe("GinkgoBuilder: For BeforeEach/AfterEach/JustBeforeEach", func() {
	var (
		justBeforeEach = 0
		beforeEach     = 0
		afterEach      = 0
	)

	NewGinkgoBuilder("").
		BeforeEach(func() {
			beforeEach++
		}).
		AfterEach(func() {
			afterEach++
		}).
		JustBeforeEach(func() {
			justBeforeEach++
		}).
		Expose()

	It("1st testing. BeforeEach()/JustBeforeEach() get called 1 time", func() {
		Expect(beforeEach).To(Equal(1))
		Expect(justBeforeEach).To(Equal(1))
		Expect(afterEach).To(Equal(0))
	})
	It("2nd testing. BeforeEach()/JustBeforeEach() get called 2 times. AfterEach() get called once", func() {
		Expect(beforeEach).To(Equal(2))
		Expect(justBeforeEach).To(Equal(2))
		Expect(afterEach).To(Equal(1))
	})
})

var _ = Describe("GinkgoBuilder: For BeforeFirst/AfterLast", func() {
	var (
		beforeCalled = 0
		afterCalled  = 0
	)

	NewGinkgoBuilder("").
		BeforeFirst(func() {
			beforeCalled++
		}).
		AfterLast(func() {
			afterCalled++
		}).
		It("1st testing", func() {
			Expect(beforeCalled).To(Equal(1))
			Expect(afterCalled).To(Equal(0))
		}).
		It("2nd testing", func() {
			Expect(beforeCalled).To(Equal(1))
			Expect(afterCalled).To(Equal(0))
		}).
		Expose()

	It("BeforeFirst should get called only once", func() {
		Expect(beforeCalled).To(Equal(1))
	})
	It("AfterLast should get called only once", func() {
		Expect(afterCalled).To(Equal(1))
	})
})

var _ = Describe("GinkgoTable", func() {
	Context("Simple case", func() {
		counter := 0

		NewGinkgoTable().
			Exec(func(v int) {
				counter += v
			}).
			Exec(func(v int) {
				counter += v
			}).
			Case("case 1 for value 2", 2).
			Case(
				func(v int) string {
					return fmt.Sprintf("case 2 for value %d", v)
				}, 3,
			).
			Expose()

		It("The final counter should be #case * #exec_body", func() {
			Expect(counter).To(Equal(10))
		})
	})
})

var _ = Describe("Use GinkgoTable with GinkgoBuilder", func() {
	var (
		beforeFirst  = 0
		beforeEach   = 0
		tableExec    = 0
		externalExec = 0
	)

	NewGinkgoBuilder("Sample Context 1").
		BeforeFirst(func() {
			beforeFirst++
		}).
		BeforeEach(func() {
			beforeEach++
		}).
		Table(NewGinkgoTable().
			Exec(func() {
				tableExec++
			}).
			Case("Exec Case 1").
			Case("Exec Case 2"),
		).
		It("External It 1", func() {
			externalExec++
		}).
		ToContext()

	It("\"before first\" should be 1", func() {
		Expect(beforeFirst).To(Equal(1))
	})
	It("\"before each\" should be 3", func() {
		Expect(beforeEach).To(Equal(3))
	})
	It("\"table exec\" should be 2", func() {
		Expect(tableExec).To(Equal(2))
	})
	It("\"external exec\" should be 1", func() {
		Expect(externalExec).To(Equal(1))
	})
})
<|endoftext|>


<commit_before>// Package db encapsulates connection with Riak.
//
// The function Open dials to Riak and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/mrb/riakpbc"
	"github.com/tsuru/config"
	"log"
	"strings"
	"sync"
	"time"
    "encoding/json"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const (
	DefaultRiakURL    = "127.0.0.1:8087"
	DefaultBucketName = "appreqs"
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *riakpbc.Client
	used time.Time
}

// Storage holds the connection with the bucket name.
type Storage struct {
	coder_client *riakpbc.Client
	bktname      string
}

func open(addr []string, bucketname string) (*Storage, error) {
	log.Printf("--> Dialing to %v", addr)
	coder := riakpbc.NewCoder("json", riakpbc.JsonMarshaller, riakpbc.JsonUnmarshaller)
	riakCoder := riakpbc.NewClientWithCoder(addr, coder)
	if err := riakCoder.Dial(); err != nil {
		return nil, err
	}

	// Set Client ID
	/*if _, err := riakCoder.SetClientId("coolio"); err != nil {
		log.Fatalf("Setting client ID failed: %v", err)
	}
	*/
	storage := &Storage{coder_client: riakCoder, bktname: bucketname}

	mut.Lock()
	conn[strings.Join(addr, "::")] = &session{s: riakCoder, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the Riak database, and return a new connection (represented
// by the type Storage).
//
// addr is a Riak connection URI, and bktname is the name of the bucket.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr []string, bktname string) (storage *Storage, err error) {
	log.Printf("--> Connecting to %v", addr)
	defer func() {
		if r := recover(); r != nil {
			log.Printf("--> Recovered from panic")
			storage, err = open(addr, bktname)
		}
	}()
	mut.RLock()
	if session, ok := conn[strings.Join(addr, "::")]; ok {
		mut.RUnlock()
		if _, err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[strings.Join(addr, "::")] = session
			mut.Unlock()
		}
		return open(addr, bktname)
	}
	mut.RUnlock()
	return open(addr, bktname)
}

// Conn reads the megam config and calls Open to get a database connection.
//
// Most megam packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn(bktname string) (*Storage, error) {
	url, _ := config.GetString("riak:url")
	if url == "" {
		url = DefaultRiakURL
	}
	//bktname, _ := config.GetString("riak:bucket")
	if bktname == "" {
		bktname = DefaultBucketName
	}
	tadr := []string{url}
	log.Printf("%v %s", tadr, bktname)
	return Open(tadr, bktname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	log.Printf("---] Closing storage %v", s)
	s.coder_client.Close()
}

// FetchStruct stores a struct  as JSON
//   eg: data := ExampleData{
//        Field1: "ExampleData1",
//        Field2: 1,
//   }
// So the send can pass in 	out := &ExampleData{}
// Apps returns the apps collection from MongoDB.
func (s *Storage) FetchStruct(key string, out interface{}) error {
	if _, err := s.coder_client.FetchStruct(s.bktname, key, out); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	fmt.Println(out)
	//TO-DO:
	//we need to return the fetched json -> to struct interface
	return nil
}

// StoreStruct returns the apps collection from MongoDB.
func (s *Storage) StoreStruct(key string, data interface{}) error {
	if _, err := s.coder_client.StoreStruct(s.bktname, key, data); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	return nil
}

// Fetch raw data (int, string, []byte)
func (s *Storage) FetchObject(key string, out interface{}) error {
	obj, err := s.coder_client.FetchObject(s.bktname, key)
	if err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	  object := "{\"Data\":\"" + string(obj.GetContent()[0].GetValue()) + "\"}"
	  jsonBlob := []byte(object)
      jerr := json.Unmarshal(jsonBlob, out)
	  if jerr != nil {
               return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", jerr)
	  }
	fmt.Println(out)
	//TO-DO:
	//we need to return the fetched json -> to struct interface
	return nil
}

// Store raw data (int, string, []byte)
func (s *Storage) StoreObject(key string, data string) error {
	if _, err := s.coder_client.StoreObject(s.bktname, key, []byte(data)); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	return nil
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {

			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			log.Printf("--> Stale conn[%s]", c)
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<commit_msg>Fix ssh files download<commit_after>// Package db encapsulates connection with Riak.
//
// The function Open dials to Riak and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/mrb/riakpbc"
	"github.com/tsuru/config"
	"log"
	"strings"
	"sync"
	"time"
    "encoding/json"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const (
	DefaultRiakURL    = "127.0.0.1:8087"
	DefaultBucketName = "appreqs"
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *riakpbc.Client
	used time.Time
}

// Storage holds the connection with the bucket name.
type Storage struct {
	coder_client *riakpbc.Client
	bktname      string
}

func open(addr []string, bucketname string) (*Storage, error) {
	log.Printf("--> Dialing to %v", addr)
	coder := riakpbc.NewCoder("json", riakpbc.JsonMarshaller, riakpbc.JsonUnmarshaller)
	riakCoder := riakpbc.NewClientWithCoder(addr, coder)
	if err := riakCoder.Dial(); err != nil {
		return nil, err
	}

	// Set Client ID
	/*if _, err := riakCoder.SetClientId("coolio"); err != nil {
		log.Fatalf("Setting client ID failed: %v", err)
	}
	*/
	storage := &Storage{coder_client: riakCoder, bktname: bucketname}

	mut.Lock()
	conn[strings.Join(addr, "::")] = &session{s: riakCoder, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the Riak database, and return a new connection (represented
// by the type Storage).
//
// addr is a Riak connection URI, and bktname is the name of the bucket.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr []string, bktname string) (storage *Storage, err error) {
	log.Printf("--> Connecting to %v", addr)
	defer func() {
		if r := recover(); r != nil {
			log.Printf("--> Recovered from panic")
			storage, err = open(addr, bktname)
		}
	}()
	mut.RLock()
	if session, ok := conn[strings.Join(addr, "::")]; ok {
		mut.RUnlock()
		if _, err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[strings.Join(addr, "::")] = session
			mut.Unlock()
		}
		return open(addr, bktname)
	}
	mut.RUnlock()
	return open(addr, bktname)
}

// Conn reads the megam config and calls Open to get a database connection.
//
// Most megam packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn(bktname string) (*Storage, error) {
	url, _ := config.GetString("riak:url")
	if url == "" {
		url = DefaultRiakURL
	}
	//bktname, _ := config.GetString("riak:bucket")
	if bktname == "" {
		bktname = DefaultBucketName
	}
	tadr := []string{url}
	log.Printf("%v %s", tadr, bktname)
	return Open(tadr, bktname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	log.Printf("---] Closing storage %v", s)
	s.coder_client.Close()
}

// FetchStruct stores a struct  as JSON
//   eg: data := ExampleData{
//        Field1: "ExampleData1",
//        Field2: 1,
//   }
// So the send can pass in 	out := &ExampleData{}
// Apps returns the apps collection from MongoDB.
func (s *Storage) FetchStruct(key string, out interface{}) error {
	if _, err := s.coder_client.FetchStruct(s.bktname, key, out); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	fmt.Println(out)
	//TO-DO:
	//we need to return the fetched json -> to struct interface
	return nil
}

// StoreStruct returns the apps collection from MongoDB.
func (s *Storage) StoreStruct(key string, data interface{}) error {
	if _, err := s.coder_client.StoreStruct(s.bktname, key, data); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	return nil
}

type SshObject struct{
	  Data string
	}

// Fetch raw data (int, string, []byte)
 func (s *Storage) FetchObject(key string, out *SshObject) error {

	obj, err := s.coder_client.FetchObject(s.bktname, key)
	if err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
    out.Data = string(obj.GetContent()[0].GetValue())
	return nil
}

// Store raw data (int, string, []byte)
func (s *Storage) StoreObject(key string, data string) error {
	if _, err := s.coder_client.StoreObject(s.bktname, key, []byte(data)); err != nil {
		return fmt.Errorf("Convert fetched JSON to the Struct, and return it failed: %s", err)
	}
	return nil
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {

			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			log.Printf("--> Stale conn[%s]", c)
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!solaris

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/go-fsnotify/fsnotify"
)

func cat(p string) error {
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var fileInfo os.FileInfo
	fileInfo, err = f.Stat()
	if err != nil {
		return err
	}

	isdir := (fileInfo.Mode() & os.ModeDir) != 0
	if isdir {
		return fmt.Errorf("cat cannot open %s", p)
	}

	scanner := bufio.NewScanner(f)

	// Clear the screen first
	fmt.Printf("\033[2J\033[;H")

	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	return nil
}

func NewWatcher(p string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()
	done := make(chan bool)

	// Initial cat, at least once
	cat(p)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if (event.Op & fsnotify.Write) == fsnotify.Write {
					cat(event.Name)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	err = watcher.Add(p)
	if err != nil {
		log.Fatal(err)
	}
	<-done
}

func main() {
	flag.Parse()
	argv := flag.Args()
	argc := len(argv)
	if argc < 1 {
		fmt.Fprintf(os.Stderr, "expecting a file path at least")
		os.Exit(-1)
	}

	absPath, err := filepath.Abs(argv[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(-2)
	}
	NewWatcher(absPath)
}
<commit_msg>add "\n" for args error<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9,!solaris

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/go-fsnotify/fsnotify"
)

func cat(p string) error {
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var fileInfo os.FileInfo
	fileInfo, err = f.Stat()
	if err != nil {
		return err
	}

	isdir := (fileInfo.Mode() & os.ModeDir) != 0
	if isdir {
		return fmt.Errorf("cat cannot open %s", p)
	}

	scanner := bufio.NewScanner(f)

	// Clear the screen first
	fmt.Printf("\033[2J\033[;H")

	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	return nil
}

func NewWatcher(p string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()
	done := make(chan bool)

	// Initial cat, at least once
	cat(p)

	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if (event.Op & fsnotify.Write) == fsnotify.Write {
					cat(event.Name)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			default:
				continue
			}
		}
	}()

	err = watcher.Add(p)
	if err != nil {
		log.Fatal(err)
	}
	<-done
}

func main() {
	flag.Parse()
	argv := flag.Args()
	argc := len(argv)
	if argc < 1 {
		fmt.Fprintf(os.Stderr, "expecting a file path at least\n")
		os.Exit(-1)
	}

	absPath, err := filepath.Abs(argv[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(-2)
	}
	NewWatcher(absPath)
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package kbcmf

import (
	"bytes"
	"crypto/hmac"
	"golang.org/x/crypto/nacl/secretbox"
	"io"
	"io/ioutil"
)

type decryptState int

const (
	stateHeader      decryptState = iota
	stateBody        decryptState = iota
	stateEndOfStream decryptState = iota
)

type decryptStream struct {
	ring       Keyring
	fmps       *framedMsgpackStream
	err        error
	state      decryptState
	keys       *receiverKeysPlaintext
	sessionKey SymmetricKey
	buf        []byte
}

func (ds *decryptStream) Read(b []byte) (n int, err error) {
	for n == 0 && err == nil {
		n, err = ds.read(b)
	}
	if err == io.EOF && ds.state != stateEndOfStream {
		err = io.ErrUnexpectedEOF
	}
	return n, err
}

func (ds *decryptStream) read(b []byte) (n int, err error) {

	// Handle the case of a previous error. Just return the error
	// again.
	if ds.err != nil {
		return 0, ds.err
	}

	// Handle the case first of a previous read that couldn't put all
	// of its data into the outgoing buffer.
	if len(ds.buf) > 0 {
		n = copy(b, ds.buf)
		ds.buf = ds.buf[n:]
		return n, nil
	}

	// We have three states we can be in, but we can definitely
	// fall through during one read, so be careful.

	if ds.state == stateHeader {
		if ds.err = ds.readHeader(); ds.err != nil {
			return 0, ds.err
		}
		ds.state = stateBody
	}

	if ds.state == stateBody {
		var last bool
		n, last, ds.err = ds.readBlock(b)
		if ds.err != nil {
			return 0, ds.err
		}

		if last {
			ds.state = stateEndOfStream
		}
	}

	if ds.state == stateEndOfStream {
		ds.err = ds.assertEndOfStream()
		if ds.err != nil {
			return 0, ds.err
		}
	}

	return n, nil
}

func (ds *decryptStream) readHeader() error {
	var hdr EncryptionHeader
	seqno, err := ds.fmps.Read(&hdr)
	if err != nil {
		return err
	}
	hdr.seqno = seqno
	return ds.processEncryptionHeader(&hdr)
}

func (ds *decryptStream) readBlock(b []byte) (n int, lastBlock bool, err error) {
	var eb EncryptionBlock
	var seqno PacketSeqno
	seqno, err = ds.fmps.Read(&eb)
	if err != nil {
		return 0, false, err
	}
	eb.seqno = seqno
	var plaintext []byte
	plaintext, err = ds.processEncryptionBlock(&eb)
	if err != nil {
		return 0, false, err
	}
	if plaintext == nil {
		return 0, true, err
	}

	// Copy as much as we can into the given outbuffer
	n = copy(b, plaintext)
	// Leave the remainder for a subsequent read
	ds.buf = plaintext[n:]

	return n, false, err
}

func (ds *decryptStream) assertEndOfStream() error {
	var i interface{}
	_, err := ds.fmps.Read(&i)
	if err == nil {
		err = ErrTrailingGarbage
	}
	return err
}

func (ds *decryptStream) tryVisibileReceivers(hdr *EncryptionHeader, nonce *Nonce, ephemeralKey BoxPublicKey) (BoxSecretKey, []byte, int, error) {
	var kids [][]byte
	for _, r := range hdr.Receivers {
		if r.KID != nil {
			kids = append(kids, r.KID)
		}
	}

	i, sk := ds.ring.LookupBoxSecretKey(kids)
	if i < 0 || sk == nil {
		return nil, nil, -1, nil
	}

	// Decrypt the sender's public key
	nonce.writeCounter32(uint32(2 * i))
	senderPublicRawKey, err := sk.Unbox(ephemeralKey, nonce, hdr.Receivers[i].Sender)
	if err != nil {
		return nil, nil, -1, err
	}
	return sk, senderPublicRawKey, i, err
}

func (ds *decryptStream) tryHiddenReceivers(hdr *EncryptionHeader, nonce *Nonce, ephemeralKey BoxPublicKey) (BoxSecretKey, []byte, int, error) {
	secretKeys := ds.ring.GetAllSecretKeys()
	var bpsk BoxPrecomputedSharedKey

	for _, secretKey := range secretKeys {
		bpsk = secretKey.Precompute(ephemeralKey)

		for i, r := range hdr.Receivers {
			// Decrypt the sender's public key
			nonce.writeCounter32(uint32(2 * i))
			senderPublicRawKey, err := bpsk.Unbox(nonce, r.Sender)
			if err == nil {
				return secretKey, senderPublicRawKey, i, nil
			}
		}
	}

	return nil, nil, -1, nil
}

func (ds *decryptStream) processEncryptionHeader(hdr *EncryptionHeader) error {
	if err := hdr.validate(); err != nil {
		return err
	}
	ephemeralKey := ds.ring.ImportEphemeralKey(hdr.Sender)
	if ephemeralKey == nil {
		return ErrBadEphemeralKey
	}

	var nonce Nonce
	copy(nonce[:], hdr.Nonce)

	secretKey, senderPublicRawKey, i, err := ds.tryVisibileReceivers(hdr, &nonce, ephemeralKey)
	if err != nil {
		return err
	}
	if secretKey == nil {
		secretKey, senderPublicRawKey, i, err = ds.tryHiddenReceivers(hdr, &nonce, ephemeralKey)
	}
	if err != nil {
		return err
	}
	if secretKey == nil || i < 0 {
		return ErrNoDecryptionKey
	}
	if err := verifyRawKey(senderPublicRawKey); err != nil {
		return err
	}

	// Lookup the sender's public key in our keyring, and import
	// it for use. However, if the sender key is the same as the ephemeral
	// key, then assume "anonymous mode", so use the already imported anonymous
	// key.
	pk := ephemeralKey
	if !hmac.Equal(hdr.Sender, senderPublicRawKey) {
		pk = ds.ring.LookupBoxPublicKey(senderPublicRawKey)
		if pk == nil {
			return ErrNoSenderKey
		}
	}

	nonce.writeCounter32(uint32(2*i + 1))

	keysPacked, err := secretKey.Unbox(pk, &nonce, hdr.Receivers[i].Keys)
	if err != nil {
		return err
	}

	var keys receiverKeysPlaintext
	if err = decodeFromBytes(&keys, keysPacked); err != nil {
		return err
	}
	ds.keys = &keys
	copy(ds.sessionKey[:], ds.keys.SessionKey)

	return nil
}

func (ds *decryptStream) checkMAC(bl *EncryptionBlock, b []byte) error {
	gid := (ds.keys.GroupID ^ groupIDMask)

	if len(bl.MACs) <= gid {
		return ErrBadGroupID(gid)
	}

	if len(ds.keys.MACKey) == 0 {
		return ErrNoGroupMACKey
	}

	mac := hmacSHA512(ds.keys.MACKey, b)
	if !hmac.Equal(mac, bl.MACs[gid]) {
		return ErrMACMismatch(bl.seqno)
	}
	return nil
}

func (ds *decryptStream) processEncryptionBlock(bl *EncryptionBlock) ([]byte, error) {
	if err := bl.validate(); err != nil {
		return nil, err
	}

	if bl.seqno <= 0 {
		return nil, errPacketUnderflow
	}

	blockNum := encryptionBlockNumber(bl.seqno - 1)

	if err := blockNum.check(); err != nil {
		return nil, err
	}

	nonce := blockNum.newCounterNonce()

	plaintext, ok := secretbox.Open([]byte{}, bl.Ciphertext, (*[24]byte)(nonce), (*[32]byte)(&ds.sessionKey))
	if !ok {
		return nil, ErrBadCiphertext(bl.seqno)
	}

	sum := hashNonceAndAuthTag(nonce, bl.Ciphertext)
	if err := ds.checkMAC(bl, sum[:]); err != nil {
		return nil, err
	}

	// The encoding of the empty buffer implies the EOF.  But otherwise, all mechanisms are the same.
	if len(plaintext) == 0 {
		return nil, nil
	}
	return plaintext, nil
}

// NewDecryptStream starts a streaming decryption. You should give it an io.Writer
// to write plaintext output to, and also a Keyring to lookup private and public keys
// as necessary. The stream will only ever write validated data.  It returns
// an io.Writer that you can write the ciphertext stream to, and an error if anything
// goes wrong in the construction process.
func NewDecryptStream(r io.Reader, keyring Keyring) (ds io.Reader, err error) {
	ds = &decryptStream{
		ring: keyring,
		fmps: newFramedMsgpackStream(r),
	}
	return ds, nil
}

// Open simply opens a ciphertext given the set of keys in the specified keyring.
// It return a plaintext on sucess, and an error on failure.
func Open(ciphertext []byte, keyring Keyring) (plaintext []byte, err error) {
	buf := bytes.NewBuffer(ciphertext)
	plaintextStream, err := NewDecryptStream(buf, keyring)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(plaintextStream)
}
<commit_msg>misspelling<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package kbcmf

import (
	"bytes"
	"crypto/hmac"
	"golang.org/x/crypto/nacl/secretbox"
	"io"
	"io/ioutil"
)

type decryptState int

const (
	stateHeader      decryptState = iota
	stateBody        decryptState = iota
	stateEndOfStream decryptState = iota
)

type decryptStream struct {
	ring       Keyring
	fmps       *framedMsgpackStream
	err        error
	state      decryptState
	keys       *receiverKeysPlaintext
	sessionKey SymmetricKey
	buf        []byte
}

func (ds *decryptStream) Read(b []byte) (n int, err error) {
	for n == 0 && err == nil {
		n, err = ds.read(b)
	}
	if err == io.EOF && ds.state != stateEndOfStream {
		err = io.ErrUnexpectedEOF
	}
	return n, err
}

func (ds *decryptStream) read(b []byte) (n int, err error) {

	// Handle the case of a previous error. Just return the error
	// again.
	if ds.err != nil {
		return 0, ds.err
	}

	// Handle the case first of a previous read that couldn't put all
	// of its data into the outgoing buffer.
	if len(ds.buf) > 0 {
		n = copy(b, ds.buf)
		ds.buf = ds.buf[n:]
		return n, nil
	}

	// We have three states we can be in, but we can definitely
	// fall through during one read, so be careful.

	if ds.state == stateHeader {
		if ds.err = ds.readHeader(); ds.err != nil {
			return 0, ds.err
		}
		ds.state = stateBody
	}

	if ds.state == stateBody {
		var last bool
		n, last, ds.err = ds.readBlock(b)
		if ds.err != nil {
			return 0, ds.err
		}

		if last {
			ds.state = stateEndOfStream
		}
	}

	if ds.state == stateEndOfStream {
		ds.err = ds.assertEndOfStream()
		if ds.err != nil {
			return 0, ds.err
		}
	}

	return n, nil
}

func (ds *decryptStream) readHeader() error {
	var hdr EncryptionHeader
	seqno, err := ds.fmps.Read(&hdr)
	if err != nil {
		return err
	}
	hdr.seqno = seqno
	return ds.processEncryptionHeader(&hdr)
}

func (ds *decryptStream) readBlock(b []byte) (n int, lastBlock bool, err error) {
	var eb EncryptionBlock
	var seqno PacketSeqno
	seqno, err = ds.fmps.Read(&eb)
	if err != nil {
		return 0, false, err
	}
	eb.seqno = seqno
	var plaintext []byte
	plaintext, err = ds.processEncryptionBlock(&eb)
	if err != nil {
		return 0, false, err
	}
	if plaintext == nil {
		return 0, true, err
	}

	// Copy as much as we can into the given outbuffer
	n = copy(b, plaintext)
	// Leave the remainder for a subsequent read
	ds.buf = plaintext[n:]

	return n, false, err
}

func (ds *decryptStream) assertEndOfStream() error {
	var i interface{}
	_, err := ds.fmps.Read(&i)
	if err == nil {
		err = ErrTrailingGarbage
	}
	return err
}

func (ds *decryptStream) tryVisibleReceivers(hdr *EncryptionHeader, nonce *Nonce, ephemeralKey BoxPublicKey) (BoxSecretKey, []byte, int, error) {
	var kids [][]byte
	for _, r := range hdr.Receivers {
		if r.KID != nil {
			kids = append(kids, r.KID)
		}
	}

	i, sk := ds.ring.LookupBoxSecretKey(kids)
	if i < 0 || sk == nil {
		return nil, nil, -1, nil
	}

	// Decrypt the sender's public key
	nonce.writeCounter32(uint32(2 * i))
	senderPublicRawKey, err := sk.Unbox(ephemeralKey, nonce, hdr.Receivers[i].Sender)
	if err != nil {
		return nil, nil, -1, err
	}
	return sk, senderPublicRawKey, i, err
}

func (ds *decryptStream) tryHiddenReceivers(hdr *EncryptionHeader, nonce *Nonce, ephemeralKey BoxPublicKey) (BoxSecretKey, []byte, int, error) {
	secretKeys := ds.ring.GetAllSecretKeys()
	var bpsk BoxPrecomputedSharedKey

	for _, secretKey := range secretKeys {
		bpsk = secretKey.Precompute(ephemeralKey)

		for i, r := range hdr.Receivers {
			// Decrypt the sender's public key
			nonce.writeCounter32(uint32(2 * i))
			senderPublicRawKey, err := bpsk.Unbox(nonce, r.Sender)
			if err == nil {
				return secretKey, senderPublicRawKey, i, nil
			}
		}
	}

	return nil, nil, -1, nil
}

func (ds *decryptStream) processEncryptionHeader(hdr *EncryptionHeader) error {
	if err := hdr.validate(); err != nil {
		return err
	}
	ephemeralKey := ds.ring.ImportEphemeralKey(hdr.Sender)
	if ephemeralKey == nil {
		return ErrBadEphemeralKey
	}

	var nonce Nonce
	copy(nonce[:], hdr.Nonce)

	secretKey, senderPublicRawKey, i, err := ds.tryVisibleReceivers(hdr, &nonce, ephemeralKey)
	if err != nil {
		return err
	}
	if secretKey == nil {
		secretKey, senderPublicRawKey, i, err = ds.tryHiddenReceivers(hdr, &nonce, ephemeralKey)
	}
	if err != nil {
		return err
	}
	if secretKey == nil || i < 0 {
		return ErrNoDecryptionKey
	}
	if err := verifyRawKey(senderPublicRawKey); err != nil {
		return err
	}

	// Lookup the sender's public key in our keyring, and import
	// it for use. However, if the sender key is the same as the ephemeral
	// key, then assume "anonymous mode", so use the already imported anonymous
	// key.
	pk := ephemeralKey
	if !hmac.Equal(hdr.Sender, senderPublicRawKey) {
		pk = ds.ring.LookupBoxPublicKey(senderPublicRawKey)
		if pk == nil {
			return ErrNoSenderKey
		}
	}

	nonce.writeCounter32(uint32(2*i + 1))

	keysPacked, err := secretKey.Unbox(pk, &nonce, hdr.Receivers[i].Keys)
	if err != nil {
		return err
	}

	var keys receiverKeysPlaintext
	if err = decodeFromBytes(&keys, keysPacked); err != nil {
		return err
	}
	ds.keys = &keys
	copy(ds.sessionKey[:], ds.keys.SessionKey)

	return nil
}

func (ds *decryptStream) checkMAC(bl *EncryptionBlock, b []byte) error {
	gid := (ds.keys.GroupID ^ groupIDMask)

	if len(bl.MACs) <= gid {
		return ErrBadGroupID(gid)
	}

	if len(ds.keys.MACKey) == 0 {
		return ErrNoGroupMACKey
	}

	mac := hmacSHA512(ds.keys.MACKey, b)
	if !hmac.Equal(mac, bl.MACs[gid]) {
		return ErrMACMismatch(bl.seqno)
	}
	return nil
}

func (ds *decryptStream) processEncryptionBlock(bl *EncryptionBlock) ([]byte, error) {
	if err := bl.validate(); err != nil {
		return nil, err
	}

	if bl.seqno <= 0 {
		return nil, errPacketUnderflow
	}

	blockNum := encryptionBlockNumber(bl.seqno - 1)

	if err := blockNum.check(); err != nil {
		return nil, err
	}

	nonce := blockNum.newCounterNonce()

	plaintext, ok := secretbox.Open([]byte{}, bl.Ciphertext, (*[24]byte)(nonce), (*[32]byte)(&ds.sessionKey))
	if !ok {
		return nil, ErrBadCiphertext(bl.seqno)
	}

	sum := hashNonceAndAuthTag(nonce, bl.Ciphertext)
	if err := ds.checkMAC(bl, sum[:]); err != nil {
		return nil, err
	}

	// The encoding of the empty buffer implies the EOF.  But otherwise, all mechanisms are the same.
	if len(plaintext) == 0 {
		return nil, nil
	}
	return plaintext, nil
}

// NewDecryptStream starts a streaming decryption. You should give it an io.Writer
// to write plaintext output to, and also a Keyring to lookup private and public keys
// as necessary. The stream will only ever write validated data.  It returns
// an io.Writer that you can write the ciphertext stream to, and an error if anything
// goes wrong in the construction process.
func NewDecryptStream(r io.Reader, keyring Keyring) (ds io.Reader, err error) {
	ds = &decryptStream{
		ring: keyring,
		fmps: newFramedMsgpackStream(r),
	}
	return ds, nil
}

// Open simply opens a ciphertext given the set of keys in the specified keyring.
// It return a plaintext on sucess, and an error on failure.
func Open(ciphertext []byte, keyring Keyring) (plaintext []byte, err error) {
	buf := bytes.NewBuffer(ciphertext)
	plaintextStream, err := NewDecryptStream(buf, keyring)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(plaintextStream)
}
<|endoftext|>


<commit_before>// Copyright 2017 ETH Zurich
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package snet implements interfaces net.Conn and net.PacketConn for SCION
// connections.
//
// The default (package-wide) SCION network must first be initialized by
// calling Init. All future package scoped DialSCION and ListenSCION calls will
// use this initial context to get the local ISD-AS, dispatcher or sciond.
//
// A connection can be created by calling DialSCION or ListenSCION; both
// functions register an address-port pair with the local dispatcher. For Dial,
// the remote address is fixed, meaning only Read and Write can be used.
// Attempting to ReadFrom or WriteTo a connection created by Dial is an invalid
// operation. For Listen, the remote address cannot be fixed. ReadFrom,
// ReadFromSCION can be used to read from the connection and find out the
// sender's address; WriteTo and WriteToSCION can be used to send a message to
// a chosen destination.
//
// For applications that need to run in multiple ASes, new networking contexts
// can be created using NewNetwork. Calling the DialSCION or ListenSCION
// methods on the networking context yields connections that run in that context.
//
// Multiple networking contexts can share the same SCIOND and/or dispatcher.
//
// Known issues:
//   - Apps do not have access to overlay ports. This means they cannot
// communicate with SCION hosts behind legacy NAT or with border routers
// directly
//   - Each path lookup registers a persistent entry with the path resolver
package snet

import (
	"net"
	"time"

	log "github.com/inconshreveable/log15"

	"github.com/netsec-ethz/scion/go/lib/addr"
	"github.com/netsec-ethz/scion/go/lib/common"
	"github.com/netsec-ethz/scion/go/lib/pathmgr"
	"github.com/netsec-ethz/scion/go/lib/sciond"
	"github.com/netsec-ethz/scion/go/lib/sock/reliable"
)

var (
	// Default SCION networking context for package-level Dial and Listen
	DefNetwork *Network
)

// Init initializes the default SCION networking context.
func Init(ia *addr.ISD_AS, sPath string, dPath string) error {
	if DefNetwork != nil {
		return common.NewCError("Cannot initialize global SCION network twice")
	}

	network, err := NewNetwork(ia, sPath, dPath)
	if err != nil {
		return err
	}
	DefNetwork = network
	return nil
}

// IA returns the default ISD-AS
func IA() *addr.ISD_AS {
	if DefNetwork == nil {
		return nil
	}
	return DefNetwork.localIA.Copy()
}

// SCION networking context, containing local ISD-AS, SCIOND, Dispatcher and
// Path resolver.
type Network struct {
	sciondPath     string
	dispatcherPath string
	pathResolver   *pathmgr.PR
	localIA        *addr.ISD_AS
}

// NewNetwork creates a new networking context, on which future Dial or Listen
// calls can be made. The new connections use the SCIOND server at sPath, the
// dispatcher at dPath, and ia for the local ISD-AS.
func NewNetwork(ia *addr.ISD_AS, sPath string, dPath string) (*Network, error) {
	network := &Network{
		sciondPath:     sPath,
		dispatcherPath: dPath,
		localIA:        ia,
	}
	sd := sciond.NewService(sPath)
	timers := pathmgr.Timers{
		NormalRefire: time.Minute,
		ErrorRefire:  3 * time.Second,
		MaxAge:       time.Hour,
	}
	pathResolver, err := pathmgr.New(sd, timers, log.Root())
	if err != nil {
		return nil, common.NewCError("Unable to initialize path resolver", "err", err)
	}
	network.pathResolver = pathResolver
	return network, nil
}

// DialSCION returns a SCION connection to raddr. Nil values for laddr are not
// supported yet.  Parameter network must be "udp4". The returned connection's
// Read and Write methods can be used to receive and send SCION packets.
func (n *Network) DialSCION(network string, laddr, raddr *Addr) (*Conn, error) {
	if raddr == nil {
		return nil, common.NewCError("Unable to dial to nil remote")
	}
	conn, err := n.ListenSCION(network, laddr)
	if err != nil {
		return nil, err
	}
	conn.raddr = raddr.Copy()
	conn.sp, err = n.pathResolver.Watch(conn.laddr.IA, conn.raddr.IA)
	if err != nil {
		return nil, common.NewCError("Unable to establish path", "err", err)
	}
	return conn, nil
}

// ListenSCION registers laddr with the dispatcher. Nil values for laddr are
// not supported yet. The returned connection's ReadFrom and WriteTo methods
// can be used to receive and send SCION packets with per-packet addressing.
// Parameter network must be "udp4".
func (n *Network) ListenSCION(network string, laddr *Addr) (*Conn, error) {
	if network != "udp4" {
		return nil, common.NewCError("Network not implemented", "net", network)
	}
	// FIXME(scrye): If no local address is specified, we want to
	// bind to the address of the outbound interface on a random
	// free port. However, the current dispatcher version cannot
	// expose that address. Additionally, the dispatcher does not follow
	// normal operating system semantics for binding on 0.0.0.0 (it
	// considers it to be a fixed address instead of a wildcard). To avoid
	// misuse, disallow binding to nil or 0.0.0.0 addresses for now.
	if laddr == nil {
		return nil, common.NewCError("Nil laddr not supported")
	}
	if laddr.Host.Type() != addr.HostTypeIPv4 {
		return nil, common.NewCError("Supplied local address does not match network",
			"expected", addr.HostTypeIPv4, "actual", laddr.Host.Type())
	}
	if laddr.Host.IP().Equal(net.IPv4zero) {
		return nil, common.NewCError("Binding to 0.0.0.0 not supported")
	}
	conn := &Conn{
		net:        network,
		scionNet:   n,
		recvBuffer: make(common.RawBytes, BufSize),
		sendBuffer: make(common.RawBytes, BufSize)}

	// Initialize local bind address
	var regAddr reliable.AppAddr
	// NOTE: keep nil address logic for now, even though we do not support
	// it yet
	if laddr != nil {
		conn.laddr = laddr.Copy()
		regAddr.Port = conn.laddr.L4Port
	} else {
		conn.laddr = &Addr{}
		conn.laddr.Host = addr.HostFromIP(net.IPv4zero)
		conn.laddr.IA = conn.scionNet.localIA
	}
	regAddr.Addr = conn.laddr.Host

	if conn.laddr.IA == nil {
		conn.laddr.IA = n.IA()
	}

	if !conn.laddr.IA.Eq(conn.scionNet.localIA) {
		return nil, common.NewCError("Unable to listen on non-local IA",
			"expected", conn.scionNet.localIA, "actual", conn.laddr.IA)
	}

	rconn, port, err := reliable.Register(conn.scionNet.dispatcherPath,
		conn.laddr.IA, regAddr)
	if err != nil {
		return nil, common.NewCError("Unable to register with dispatcher",
			"err", err)
	}
	log.Info("Registered with dispatcher", "ia", conn.scionNet.localIA, "host", regAddr.Addr,
		"port", port)
	conn.laddr.L4Port = port
	conn.conn = rconn
	return conn, nil
}

// PathResolver returns the pathmgr.PR that the network is using.
func (n *Network) PathResolver() *pathmgr.PR {
	return n.pathResolver
}

// IA returns a copy of the ISD-AS assigned to n
func (n *Network) IA() *addr.ISD_AS {
	return n.localIA.Copy()
}
<commit_msg>Clarify SCMP behavior of snet (#1323)<commit_after>// Copyright 2017 ETH Zurich
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package snet implements interfaces net.Conn and net.PacketConn for SCION
// connections.
//
// The default (package-wide) SCION network must first be initialized by
// calling Init. All future package scoped DialSCION and ListenSCION calls will
// use this initial context to get the local ISD-AS, dispatcher or sciond.
//
// A connection can be created by calling DialSCION or ListenSCION; both
// functions register an address-port pair with the local dispatcher. For Dial,
// the remote address is fixed, meaning only Read and Write can be used.
// Attempting to ReadFrom or WriteTo a connection created by Dial is an invalid
// operation. For Listen, the remote address cannot be fixed. ReadFrom,
// ReadFromSCION can be used to read from the connection and find out the
// sender's address; WriteTo and WriteToSCION can be used to send a message to
// a chosen destination.
//
// For applications that need to run in multiple ASes, new networking contexts
// can be created using NewNetwork. Calling the DialSCION or ListenSCION
// methods on the networking context yields connections that run in that context.
//
// Multiple networking contexts can share the same SCIOND and/or dispatcher.
//
// Write calls never return SCMP errors directly. If a write call caused an
// SCMP message to be received by the Conn, it can be inspected by calling
// Read. In this case, the error value is non-nil and can be type asserted to
// *OpError. Method SCMP() can be called on the error to extract the SCMP
// header.
//
// Important: not draining SCMP errors via Read calls can cause the dispatcher
// to block (see Issue #1278). To prevent this on a Conn object with only Write
// calls, run a separate goroutine that continuously calls Read on the Conn.
package snet

import (
	"net"
	"time"

	log "github.com/inconshreveable/log15"

	"github.com/netsec-ethz/scion/go/lib/addr"
	"github.com/netsec-ethz/scion/go/lib/common"
	"github.com/netsec-ethz/scion/go/lib/pathmgr"
	"github.com/netsec-ethz/scion/go/lib/sciond"
	"github.com/netsec-ethz/scion/go/lib/sock/reliable"
)

var (
	// Default SCION networking context for package-level Dial and Listen
	DefNetwork *Network
)

// Init initializes the default SCION networking context.
func Init(ia *addr.ISD_AS, sPath string, dPath string) error {
	if DefNetwork != nil {
		return common.NewCError("Cannot initialize global SCION network twice")
	}

	network, err := NewNetwork(ia, sPath, dPath)
	if err != nil {
		return err
	}
	DefNetwork = network
	return nil
}

// IA returns the default ISD-AS
func IA() *addr.ISD_AS {
	if DefNetwork == nil {
		return nil
	}
	return DefNetwork.localIA.Copy()
}

// SCION networking context, containing local ISD-AS, SCIOND, Dispatcher and
// Path resolver.
type Network struct {
	sciondPath     string
	dispatcherPath string
	pathResolver   *pathmgr.PR
	localIA        *addr.ISD_AS
}

// NewNetwork creates a new networking context, on which future Dial or Listen
// calls can be made. The new connections use the SCIOND server at sPath, the
// dispatcher at dPath, and ia for the local ISD-AS.
func NewNetwork(ia *addr.ISD_AS, sPath string, dPath string) (*Network, error) {
	network := &Network{
		sciondPath:     sPath,
		dispatcherPath: dPath,
		localIA:        ia,
	}
	sd := sciond.NewService(sPath)
	timers := pathmgr.Timers{
		NormalRefire: time.Minute,
		ErrorRefire:  3 * time.Second,
		MaxAge:       time.Hour,
	}
	pathResolver, err := pathmgr.New(sd, timers, log.Root())
	if err != nil {
		return nil, common.NewCError("Unable to initialize path resolver", "err", err)
	}
	network.pathResolver = pathResolver
	return network, nil
}

// DialSCION returns a SCION connection to raddr. Nil values for laddr are not
// supported yet.  Parameter network must be "udp4". The returned connection's
// Read and Write methods can be used to receive and send SCION packets.
func (n *Network) DialSCION(network string, laddr, raddr *Addr) (*Conn, error) {
	if raddr == nil {
		return nil, common.NewCError("Unable to dial to nil remote")
	}
	conn, err := n.ListenSCION(network, laddr)
	if err != nil {
		return nil, err
	}
	conn.raddr = raddr.Copy()
	conn.sp, err = n.pathResolver.Watch(conn.laddr.IA, conn.raddr.IA)
	if err != nil {
		return nil, common.NewCError("Unable to establish path", "err", err)
	}
	return conn, nil
}

// ListenSCION registers laddr with the dispatcher. Nil values for laddr are
// not supported yet. The returned connection's ReadFrom and WriteTo methods
// can be used to receive and send SCION packets with per-packet addressing.
// Parameter network must be "udp4".
func (n *Network) ListenSCION(network string, laddr *Addr) (*Conn, error) {
	if network != "udp4" {
		return nil, common.NewCError("Network not implemented", "net", network)
	}
	// FIXME(scrye): If no local address is specified, we want to
	// bind to the address of the outbound interface on a random
	// free port. However, the current dispatcher version cannot
	// expose that address. Additionally, the dispatcher does not follow
	// normal operating system semantics for binding on 0.0.0.0 (it
	// considers it to be a fixed address instead of a wildcard). To avoid
	// misuse, disallow binding to nil or 0.0.0.0 addresses for now.
	if laddr == nil {
		return nil, common.NewCError("Nil laddr not supported")
	}
	if laddr.Host.Type() != addr.HostTypeIPv4 {
		return nil, common.NewCError("Supplied local address does not match network",
			"expected", addr.HostTypeIPv4, "actual", laddr.Host.Type())
	}
	if laddr.Host.IP().Equal(net.IPv4zero) {
		return nil, common.NewCError("Binding to 0.0.0.0 not supported")
	}
	conn := &Conn{
		net:        network,
		scionNet:   n,
		recvBuffer: make(common.RawBytes, BufSize),
		sendBuffer: make(common.RawBytes, BufSize)}

	// Initialize local bind address
	var regAddr reliable.AppAddr
	// NOTE: keep nil address logic for now, even though we do not support
	// it yet
	if laddr != nil {
		conn.laddr = laddr.Copy()
		regAddr.Port = conn.laddr.L4Port
	} else {
		conn.laddr = &Addr{}
		conn.laddr.Host = addr.HostFromIP(net.IPv4zero)
		conn.laddr.IA = conn.scionNet.localIA
	}
	regAddr.Addr = conn.laddr.Host

	if conn.laddr.IA == nil {
		conn.laddr.IA = n.IA()
	}

	if !conn.laddr.IA.Eq(conn.scionNet.localIA) {
		return nil, common.NewCError("Unable to listen on non-local IA",
			"expected", conn.scionNet.localIA, "actual", conn.laddr.IA)
	}

	rconn, port, err := reliable.Register(conn.scionNet.dispatcherPath,
		conn.laddr.IA, regAddr)
	if err != nil {
		return nil, common.NewCError("Unable to register with dispatcher",
			"err", err)
	}
	log.Info("Registered with dispatcher", "ia", conn.scionNet.localIA, "host", regAddr.Addr,
		"port", port)
	conn.laddr.L4Port = port
	conn.conn = rconn
	return conn, nil
}

// PathResolver returns the pathmgr.PR that the network is using.
func (n *Network) PathResolver() *pathmgr.PR {
	return n.pathResolver
}

// IA returns a copy of the ISD-AS assigned to n
func (n *Network) IA() *addr.ISD_AS {
	return n.localIA.Copy()
}
<|endoftext|>