<commit_before>// Copyright 2013 Paul Hammond.
// This software is licensed under the MIT license, see LICENSE.txt for details.

package jp

import (
	"bufio"
	"errors"
	"io"
	"strings"
	"unicode"
)

type dict struct {
	indented string
	objEmpty string
	objOpen  string
	objClose string
	arrEmpty string
	arrOpen  string
	arrClose string
	colon    string
	comma    string
	strOpen  string
	strClose string
	otherSpace string
	otherOpen  string
	otherClose string
	end      string
}

var dicts = map[string]dict{
	"pretty":  {"\n  ", "{ }", "{\n", "\n}", "[ ]", "[\n", "\n]", ": ", ",\n", `"`, `"`, " ", "", "", "\n"},
	"compact": {"\n", "{}", "{", "}", "[]", "[", "]", ":", ",", `"`, `"`,  " ", "", "", ""},
}

func (d dict) indent() dict {
	return dict{
		d.indented,
		strings.Replace(d.objEmpty, "\n", d.indented, 1),
		strings.Replace(d.objOpen , "\n", d.indented, 1),
		strings.Replace(d.objClose, "\n", d.indented, 1),
		strings.Replace(d.arrEmpty, "\n", d.indented, 1),
		strings.Replace(d.arrOpen , "\n", d.indented, 1),
		strings.Replace(d.arrClose, "\n", d.indented, 1),
		strings.Replace(d.colon   , "\n", d.indented, 1),
		strings.Replace(d.comma   , "\n", d.indented, 1),
		strings.Replace(d.strOpen , "\n", d.indented, 1),
		strings.Replace(d.strClose, "\n", d.indented, 1),
		d.otherSpace,
		strings.Replace(d.otherOpen , "\n", d.indented, 1),
		strings.Replace(d.otherClose, "\n", d.indented, 1),
		d.end,
	}
}

type scanner struct {
	r      *bufio.Reader
	w      *bufio.Writer
	indentSize int
	indentDicts []dict
	dict   *dict
}

func (s scanner) writeRune(r rune) (e error) {
	_, e = s.w.WriteRune(r)
	return e
}

func (s scanner) writeString(str string) (e error) {
	_, e = s.w.WriteString(str)
	return e
}

func (s *scanner) indent(d int) {
	s.indentSize += d
	if len(s.indentDicts) <= s.indentSize {
		s.indentDicts = append(s.indentDicts, s.indentDicts[len(s.indentDicts)-1].indent())
	}
	s.dict = &s.indentDicts[s.indentSize]
}

func (s scanner) read() (r rune, e error) {
	for e == nil {
		r, _, e = s.r.ReadRune()
		if !unicode.IsSpace(r) {
			break
		}
	}
	return r, e
}

func (s scanner) copyString() (e error) {
	var r rune
	var last rune
	e = s.writeString(s.dict.strOpen)
	for e == nil {
		r, _, e = s.r.ReadRune()
		if e != nil {
			break
		}

		if r == '"' && last != '\\' {
			e = s.writeString(s.dict.strClose)
			break
		} else if last != '\\' {
			last = r
			e = s.writeRune(r)
		} else {
			last = 0
			e = s.writeRune(r)
		}
	}
	return e
}

func (s scanner) copyOther() (e error) {
	var r rune
	var space bool
	e = s.writeString(s.dict.otherOpen)
	for e == nil {
		r, _, e = s.r.ReadRune()
		if e != nil {
			break
		}
		switch r {
		case '{', '}', '[', ']', ',', ':', '"':
			e = s.r.UnreadRune()
			if e != nil {
				return e // this really shouldn't happen
			}
			e = s.writeString(s.dict.otherClose)
			return e
		default:
			if unicode.IsSpace(r) {
				space = true
				break
			}
			if space {
				space = false
				e = s.writeString(s.dict.otherSpace)
				if e != nil {
					return e
				}
			}
			e = s.writeRune(r)
		}
	}
	return e
}

func (s scanner) expand() (e error) {
	var r rune
	for e == nil {
		r, e = s.read()
		if e != nil {
			break
		}
		switch r {
		case '{':
			r, e = s.read()
			if e != nil {
				break
			}
			if r == '}' {
				e = s.writeString(s.dict.objEmpty)
			} else {
				e = s.r.UnreadRune()
				if e != nil {
					break // this really shouldn't happen
				}
				s.indent(1)
				e = s.writeString(s.dict.objOpen)
			}
		case '}':
			s.indent(-1)
			e = s.writeString(s.dict.objClose)
		case '[':
			r, e = s.read()
			if e != nil {
				break
			}
			if r == ']' {
				e = s.writeString(s.dict.arrEmpty)
			} else {
				e = s.r.UnreadRune()
				if e != nil {
					break // this really shouldn't happen
				}
				s.indent(1)
				e = s.writeString(s.dict.arrOpen)
			}
		case ']':
			s.indent(-1)
			e = s.writeString(s.dict.arrClose)
		case ',':
			e = s.writeString(s.dict.comma)
		case ':':
			e = s.writeString(s.dict.colon)
		case '"':
			e = s.copyString()
		// todo unicode.ReplacementChar
		default:
			e = s.r.UnreadRune()
			if e != nil {
				break // this really shouldn't happen
			}
			e = s.copyOther()
		}
	}
	s.writeString(s.dict.end)
	s.w.Flush()
	if e == io.EOF {
		return nil
	}
	return e
}

func Expand(reader io.Reader, writer io.Writer, format string) error {
	d, ok := dicts[format]
	if !ok {
		return errors.New("unknown format")
	}
	indentDicts := []dict{d}
	s := &scanner{
		r: bufio.NewReader(reader),
		w: bufio.NewWriter(writer),
		indentDicts: indentDicts,
		dict: &indentDicts[0],
	}
	return s.expand()
}
<commit_msg>Remove duplicated code<commit_after>// Copyright 2013 Paul Hammond.
// This software is licensed under the MIT license, see LICENSE.txt for details.

package jp

import (
	"bufio"
	"errors"
	"io"
	"strings"
	"unicode"
)

type dict struct {
	indented string
	objEmpty string
	objOpen  string
	objClose string
	arrEmpty string
	arrOpen  string
	arrClose string
	colon    string
	comma    string
	strOpen  string
	strClose string
	otherSpace string
	otherOpen  string
	otherClose string
	end      string
}

var dicts = map[string]dict{
	"pretty":  {"\n  ", "{ }", "{\n", "\n}", "[ ]", "[\n", "\n]", ": ", ",\n", `"`, `"`, " ", "", "", "\n"},
	"compact": {"\n", "{}", "{", "}", "[]", "[", "]", ":", ",", `"`, `"`,  " ", "", "", ""},
}

func (d dict) indent() dict {
	return dict{
		d.indented,
		strings.Replace(d.objEmpty, "\n", d.indented, 1),
		strings.Replace(d.objOpen , "\n", d.indented, 1),
		strings.Replace(d.objClose, "\n", d.indented, 1),
		strings.Replace(d.arrEmpty, "\n", d.indented, 1),
		strings.Replace(d.arrOpen , "\n", d.indented, 1),
		strings.Replace(d.arrClose, "\n", d.indented, 1),
		strings.Replace(d.colon   , "\n", d.indented, 1),
		strings.Replace(d.comma   , "\n", d.indented, 1),
		strings.Replace(d.strOpen , "\n", d.indented, 1),
		strings.Replace(d.strClose, "\n", d.indented, 1),
		d.otherSpace,
		strings.Replace(d.otherOpen , "\n", d.indented, 1),
		strings.Replace(d.otherClose, "\n", d.indented, 1),
		d.end,
	}
}

type scanner struct {
	r      *bufio.Reader
	w      *bufio.Writer
	indentSize int
	indentDicts []dict
	dict   *dict
}

func (s scanner) writeRune(r rune) (e error) {
	_, e = s.w.WriteRune(r)
	return e
}

func (s scanner) writeString(str string) (e error) {
	_, e = s.w.WriteString(str)
	return e
}

func (s *scanner) indent(d int) {
	s.indentSize += d
	if len(s.indentDicts) <= s.indentSize {
		s.indentDicts = append(s.indentDicts, s.indentDicts[len(s.indentDicts)-1].indent())
	}
	s.dict = &s.indentDicts[s.indentSize]
}

func (s scanner) unread() {
	e := s.r.UnreadRune()
	if e != nil {
		panic(e.Error()) // we only ever read runes, so this shouldn't happen
	}
}

func (s scanner) read() (r rune, e error) {
	for e == nil {
		r, _, e = s.r.ReadRune()
		if !unicode.IsSpace(r) {
			break
		}
	}
	return r, e
}

func (s scanner) copyString() (e error) {
	var r rune
	var last rune
	e = s.writeString(s.dict.strOpen)
	for e == nil {
		r, _, e = s.r.ReadRune()
		if e != nil {
			break
		}

		if r == '"' && last != '\\' {
			e = s.writeString(s.dict.strClose)
			break
		} else if last != '\\' {
			last = r
			e = s.writeRune(r)
		} else {
			last = 0
			e = s.writeRune(r)
		}
	}
	return e
}

func (s scanner) copyOther() (e error) {
	var r rune
	var space bool
	e = s.writeString(s.dict.otherOpen)
	for e == nil {
		r, _, e = s.r.ReadRune()
		if e != nil {
			break
		}
		switch r {
		case '{', '}', '[', ']', ',', ':', '"':
			s.unread()
			e = s.writeString(s.dict.otherClose)
			return e
		default:
			if unicode.IsSpace(r) {
				space = true
				break
			}
			if space {
				space = false
				e = s.writeString(s.dict.otherSpace)
				if e != nil {
					return e
				}
			}
			e = s.writeRune(r)
		}
	}
	return e
}

func (s scanner) expand() (e error) {
	var r rune
	for e == nil {
		r, e = s.read()
		if e != nil {
			break
		}
		switch r {
		case '{':
			r, e = s.read()
			if e != nil {
				break
			}
			if r == '}' {
				e = s.writeString(s.dict.objEmpty)
			} else {
				s.unread()
				s.indent(1)
				e = s.writeString(s.dict.objOpen)
			}
		case '}':
			s.indent(-1)
			e = s.writeString(s.dict.objClose)
		case '[':
			r, e = s.read()
			if e != nil {
				break
			}
			if r == ']' {
				e = s.writeString(s.dict.arrEmpty)
			} else {
				s.unread()
				s.indent(1)
				e = s.writeString(s.dict.arrOpen)
			}
		case ']':
			s.indent(-1)
			e = s.writeString(s.dict.arrClose)
		case ',':
			e = s.writeString(s.dict.comma)
		case ':':
			e = s.writeString(s.dict.colon)
		case '"':
			e = s.copyString()
		// todo unicode.ReplacementChar
		default:
			s.unread()
			e = s.copyOther()
		}
	}
	s.writeString(s.dict.end)
	s.w.Flush()
	if e == io.EOF {
		return nil
	}
	return e
}

func Expand(reader io.Reader, writer io.Writer, format string) error {
	d, ok := dicts[format]
	if !ok {
		return errors.New("unknown format")
	}
	indentDicts := []dict{d}
	s := &scanner{
		r: bufio.NewReader(reader),
		w: bufio.NewWriter(writer),
		indentDicts: indentDicts,
		dict: &indentDicts[0],
	}
	return s.expand()
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 AKUALAB INC., All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ju provides utilities for manipulating json objects.
package ju

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// ReadJSON unmarshals json data from an io.Reader.
// The param "o" must be a pointer to an object.
func ReadJSON(r io.Reader, o interface{}) error {
	dec := json.NewDecoder(r)
	err := dec.Decode(o)
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}

// ReadJSONFile unmarshals json data from a file.
func ReadJSONFile(fn string, o interface{}) error {

	f, err := os.Open(fn)
	if err != nil {
		return err
	}
	e := ReadJSON(f, o)
	if e != nil {
		return e
	}
	e = f.Close()
	return e
}

// WriteJSON writes an object to an io.Writer.
func WriteJSON(w io.Writer, o interface{}) error {

	enc := json.NewEncoder(w)
	err := enc.Encode(o)
	if err != nil {
		return err
	}
	return nil
}

// WriteJSONFile writes to a file.
func WriteJSONFile(fn string, o interface{}) error {

	e := os.MkdirAll(filepath.Dir(fn), 0755)
	if e != nil {
		return e
	}
	f, err := os.Create(fn)
	if err != nil {
		return err
	}
	ee := WriteJSON(f, o)
	if ee != nil {
		return ee
	}
	e = f.Close()
	return e
}

// JSONStreamer returns a reader that returns JSON objects read from a file or from multiple files. JSON objects are stored as follows:
//   {"example":11, "any":"json"}
//   {"example":12, "any":"json"}
//   ...
//   EOF
// Supported cases:
//   1 - path is a file. Read stream of JSON objects from file.
//   2 - path is a directory. Read stream from all the files in that directory that have extension ".json".
//   3 - path is a file with extension ".list" that contains a list of paths to json files. Read from all the files in the list.
// The return value is of type io.ReadCloser, the caller is responsible for closing the JSONStreamer to release resources
// using the Close() method.
func JSONStreamer(path string) (io.ReadCloser, error) {

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}
	switch {
	case fi.IsDir():
		return streamDir(path)
	case filepath.Ext(path) == ".json":
		return streamFile(path)
	case filepath.Ext(path) == ".list":
		return streamList(path)
	default:
		return nil, fmt.Errorf("can't parse path to stream json objects - must be a dir or have extensions \".json\" or \".list\"")
	}
}

func streamDir(path string) (io.ReadCloser, error) {

	files := []string{}
	filepath.Walk(path, func(fn string, info os.FileInfo, err error) error {

		if filepath.Ext(fn) != ".json" {
			return nil
		}
		files = append(files, fn)
		return nil
	})

	return &multi{files: files}, nil
}

type multi struct {
	files []string
	idx   int
	file  *os.File
}

func (m *multi) Read(p []byte) (int, error) {

	var err error
	if m.file == nil {
		m.file, err = os.Open(m.files[m.idx])
		if err != nil {
			return 0, err
		}
		m.idx++
	}
	n, e := m.file.Read(p)
	switch {

	case e == nil:
		// We are good.
		return n, nil

	case e == io.EOF && m.idx < len(m.files):
		// End of file but we have more files.
		err := m.file.Close()
		if err != nil {
			return n, err
		}
		m.file = nil
		return n, nil // we are not done yet!

	case e == io.EOF:
		// End of last file.
		err := m.file.Close()
		if err != nil {
			return 0, err
		}
		m.file = nil
		return n, io.EOF // we are done!

	default:
		// Some unknown error.
		err := m.file.Close()
		if err != nil {
			return 0, err
		}
		return n, e
	}
}

// Close closes the underlying resources.
func (m *multi) Close() error {
	m.idx = 0
	m.files = nil
	if m.file != nil {
		err := m.file.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func streamFile(path string) (io.ReadCloser, error) {
	f, e := os.Open(path)
	if e != nil {
		return nil, e
	}
	return f, nil
}

func streamList(path string) (io.ReadCloser, error) {

	f, e := os.Open(path)
	if e != nil {
		return nil, nil
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	files := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		if filepath.Ext(line) != ".json" {
			return nil, fmt.Errorf("in list [%s] found a line without a .json extension: %s", path, line)
		}
		files = append(files, line)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return &multi{files: files}, nil
}

// PanicIf panics if error is not nil.
func PanicIf(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Remove PaniIf.<commit_after>// Copyright (c) 2015 AKUALAB INC., All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ju provides utilities for manipulating json objects.
package ju

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// ReadJSON unmarshals json data from an io.Reader.
// The param "o" must be a pointer to an object.
func ReadJSON(r io.Reader, o interface{}) error {
	dec := json.NewDecoder(r)
	err := dec.Decode(o)
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}

// ReadJSONFile unmarshals json data from a file.
func ReadJSONFile(fn string, o interface{}) error {

	f, err := os.Open(fn)
	if err != nil {
		return err
	}
	e := ReadJSON(f, o)
	if e != nil {
		return e
	}
	e = f.Close()
	return e
}

// WriteJSON writes an object to an io.Writer.
func WriteJSON(w io.Writer, o interface{}) error {

	enc := json.NewEncoder(w)
	err := enc.Encode(o)
	if err != nil {
		return err
	}
	return nil
}

// WriteJSONFile writes to a file.
func WriteJSONFile(fn string, o interface{}) error {

	e := os.MkdirAll(filepath.Dir(fn), 0755)
	if e != nil {
		return e
	}
	f, err := os.Create(fn)
	if err != nil {
		return err
	}
	ee := WriteJSON(f, o)
	if ee != nil {
		return ee
	}
	e = f.Close()
	return e
}

// JSONStreamer returns a reader that returns JSON objects read from a file or from multiple files. JSON objects are stored as follows:
//   {"example":11, "any":"json"}
//   {"example":12, "any":"json"}
//   ...
//   EOF
// Supported cases:
//   1 - path is a file. Read stream of JSON objects from file.
//   2 - path is a directory. Read stream from all the files in that directory that have extension ".json".
//   3 - path is a file with extension ".list" that contains a list of paths to json files. Read from all the files in the list.
// The return value is of type io.ReadCloser, the caller is responsible for closing the JSONStreamer to release resources
// using the Close() method.
func JSONStreamer(path string) (io.ReadCloser, error) {

	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}
	switch {
	case fi.IsDir():
		return streamDir(path)
	case filepath.Ext(path) == ".json":
		return streamFile(path)
	case filepath.Ext(path) == ".list":
		return streamList(path)
	default:
		return nil, fmt.Errorf("can't parse path to stream json objects - must be a dir or have extensions \".json\" or \".list\"")
	}
}

func streamDir(path string) (io.ReadCloser, error) {

	files := []string{}
	filepath.Walk(path, func(fn string, info os.FileInfo, err error) error {

		if filepath.Ext(fn) != ".json" {
			return nil
		}
		files = append(files, fn)
		return nil
	})

	return &multi{files: files}, nil
}

type multi struct {
	files []string
	idx   int
	file  *os.File
}

func (m *multi) Read(p []byte) (int, error) {

	var err error
	if m.file == nil {
		m.file, err = os.Open(m.files[m.idx])
		if err != nil {
			return 0, err
		}
		m.idx++
	}
	n, e := m.file.Read(p)
	switch {

	case e == nil:
		// We are good.
		return n, nil

	case e == io.EOF && m.idx < len(m.files):
		// End of file but we have more files.
		err := m.file.Close()
		if err != nil {
			return n, err
		}
		m.file = nil
		return n, nil // we are not done yet!

	case e == io.EOF:
		// End of last file.
		err := m.file.Close()
		if err != nil {
			return 0, err
		}
		m.file = nil
		return n, io.EOF // we are done!

	default:
		// Some unknown error.
		err := m.file.Close()
		if err != nil {
			return 0, err
		}
		return n, e
	}
}

// Close closes the underlying resources.
func (m *multi) Close() error {
	m.idx = 0
	m.files = nil
	if m.file != nil {
		err := m.file.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func streamFile(path string) (io.ReadCloser, error) {
	f, e := os.Open(path)
	if e != nil {
		return nil, e
	}
	return f, nil
}

func streamList(path string) (io.ReadCloser, error) {

	f, e := os.Open(path)
	if e != nil {
		return nil, nil
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	files := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		if filepath.Ext(line) != ".json" {
			return nil, fmt.Errorf("in list [%s] found a line without a .json extension: %s", path, line)
		}
		files = append(files, line)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return &multi{files: files}, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2013  gokabinet authors.
// Use of this source code is governed by a GPLv3
// license that can be found in the LICENSE file.

package kt

/*
#cgo pkg-config: kyototycoon
#include <ktlangc.h>
#include "kt.h"
*/
import "C"

import (
	"bytes"
	"fmt"
	"unsafe"
	"encoding/gob"
)

const (
	_ = iota
	DEFAULT_TIMEOUT = -1.
)

// KTError is used for errors using the gokabinet library.  It implements the
// builtin error interface.
type KTError string

func (err KTError) Error() string {
	return string(err)
}

// DB is the basic type for the gokabinet library. Holds an unexported instance
// of the database, for interactions.
type RemoteDB struct {
	db   *C.KTRDB
}

// Open opens a connection to a remote database.
//
func Open(host string, port int, timeout float32) (*RemoteDB, error) {
	d := &RemoteDB{db: C.ktdbnew()}
	cHost := C.CString(host)
	defer C.free(unsafe.Pointer(cHost))
	cPort := C.int32_t(port)
	cTimeout := C.double(timeout)

	if C.ktdbopen(d.db, cHost, cPort, cTimeout) == 0 {
		errMsg := d.LastError()
		return nil, KTError(fmt.Sprintf("Error opening %s:%d -- %s", host, port, errMsg))
	}
	return d, nil
}

// LastError returns a KTError instance representing the last occurred error in
// the database.
func (d *RemoteDB) LastError() error {
	errMsg := C.GoString(C.ktecodename(C.ktdbecode(d.db)))
	return KTError(errMsg)
}

// Close the connection to the db
func (d *RemoteDB) Close() {
	if (d != nil) {
		C.ktdbclose(d.db)
		C.ktdbdel(d.db)
	}
}

func (d *RemoteDB) Set(key, value string) error {

	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	cValue := C.CString(value)
	defer C.free(unsafe.Pointer(cValue))
	lKey := C.size_t(len(key))
	lValue := C.size_t(len(value))
	if C.ktdbset(d.db, cKey, lKey, cValue, lValue) == 0 {
		errMsg := d.LastError()
		return KTError(fmt.Sprintf("Failed to add a record with the value %s and the key %s: %s", value, key, errMsg))
	}
	return nil
}

// Get byte[] directly
func (d *RemoteDB) GetBytes(key string) ([]byte, error) {
	var resultLen C.size_t
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.ktdbget(d.db, cKey, lKey, &resultLen)
	defer C.free(unsafe.Pointer(cValue))
	if cValue == nil {
		errMsg := d.LastError()
		return []byte{}, KTError(fmt.Sprintf("Failed to get the record with the key %s: %s", key, errMsg))
	}
	return C.GoBytes(unsafe.Pointer(cValue), C.int(resultLen)), nil
}

func (d *RemoteDB) Get(key string) (string, error) {
	var resultLen C.size_t
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.ktdbget(d.db, cKey, lKey, &resultLen)
	defer C.free(unsafe.Pointer(cValue))
	if cValue == nil {
		errMsg := d.LastError()
		return "", KTError(fmt.Sprintf("Failed to get the record with the key %s: %s", key, errMsg))
	}
	return C.GoStringN(cValue, C.int(resultLen)), nil
}

// GetGob gets a record in the database by its key, decoding from gob format.
//
// Returns nil in case of success. In case of errors, it returns a KCError
// instance explaining what happened.
func (d *RemoteDB) GetGob(key string, e interface{}) error {
	data, err := d.Get(key)
	if err != nil {
		return err
	}
	buffer := bytes.NewBufferString(data)
	decoder := gob.NewDecoder(buffer)
	if err := decoder.Decode(e); err != nil {
		return KTError(fmt.Sprintf("Failed to decode the record with the key %s: %s", key, err))
	}
	return nil
}

// SetGob adds a record to the database, stored in gob format.
//
// Returns a KCError instance in case of errors, otherwise, returns nil.
func (d *RemoteDB) SetGob(key string, e interface{}) error {
	var buffer bytes.Buffer
	encoder := gob.NewEncoder(&buffer)
	err := encoder.Encode(e)
	if err != nil {
		return KTError(fmt.Sprintf("Failed to add a record with the value %s and the key %s: %s", e, key, err))
	}
	err = d.Set(key, buffer.String())
	return err
}

// Removes the key from the DB.
func (d *RemoteDB) Remove(key string) error {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	status := C.ktdbremove(d.db, cKey, lKey)
	if status == 0 {
		errMsg := d.LastError()
		return KTError(fmt.Sprintf("Failed to remove the record with the key %s: %s", key, errMsg))
	}
	return nil
}

// Clear removes all records from the database.
//
// Returns a KTError in case of failure.
func (d *RemoteDB) Clear() error {
	if C.ktdbclear(d.db) == 0 {
		msg := d.LastError()
		return KTError(fmt.Sprintf("Failed to clear the database: %s.", msg))
	}
	return nil
}

// Increment increments the value of a numeric record by a given number, and
// return the incremented value.
//
// In case of errors, returns 0 and a KCError instance with detailed error
// message.
func (d *RemoteDB) Increment(key string, number int) (int, error) {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.int64_t(number)
	v := C.ktdbincrint(d.db, cKey, lKey, cValue, 0)
	if v == C.INT64_MIN {
		return 0, KTError("It's not possible to increment a non-numeric record")
	}
	return int(v), nil
}

// Returns the number of elements
func (d *RemoteDB) Count() (int, error) {
	var err error
	v := int(C.ktdbcount(d.db))
	if v == -1 {
		err = d.LastError()
	}
	return v, err
}

// MatchPrefix returns a list of keys that matches a prefix or an error in case
// of failure.
func (d *RemoteDB) MatchPrefix(prefix string, max int64) ([]string, error) {
	cprefix := C.CString(prefix)
	defer C.free(unsafe.Pointer(cprefix))
	strary := C.match_prefix(d.db, cprefix, C.size_t(max))
	if strary.v == nil {
		return nil, d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil, nil
	}
	result := make([]string, n)
	for i := int64(0); i < n; i++ {
		result[i] = C.GoString(C.strary_item(&strary, C.int64_t(i)))
	}
	return result, nil
}

// GetBulk returns all values for the passed in array of keys. if a key does not exist, the value for this key is set to empty string
// If the key does exist, the value in the passed in map is set accordingly.
func (d *RemoteDB) GetBulk(keysAndVals map[string]string) (error) {

	keyList := make([]string, len(keysAndVals))
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	next := 0;
	for s, _ := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		keyList[next] = s
		next++
	}

	strary := C.get_bulk_binary(d.db, cKeys, C.size_t(len(keysAndVals)))
	if strary.v == nil {
		return d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil
	}

	for i := int64(0); i < n; i++ {
		if C.bool(C.strary_present(&strary, C.int64_t(i))) {
			keysAndVals[keyList[i]] = C.GoString(C.strary_item(&strary, C.int64_t(i)))
		} else {
			delete(keysAndVals, keyList[i])
		}
	}
	return nil
}

// GetBulk returns all values for the passed in array of keys. if a key does not exist, the value for this key is set to empty string
// If the key does exist, the value in the passed in map is set accordingly.
func (d *RemoteDB) GetBulkBytes(keysAndVals map[string][]byte) (error) {

	keyList := make([]string, len(keysAndVals))
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	next := 0;
	for s, _ := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		keyList[next] = s
		next++
	}

	strary := C.get_bulk_binary(d.db, cKeys, C.size_t(len(keysAndVals)))
	if strary.v == nil {
		return d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil
	}

	for i := int64(0); i < n; i++ {
		if C.bool(C.strary_present(&strary, C.int64_t(i))) {
			keysAndVals[keyList[i]] = C.GoBytes(unsafe.Pointer(C.strary_item(&strary, C.int64_t(i))), C.int(C.strary_size(&strary, C.int64_t(i))))
		} else {
			delete(keysAndVals, keyList[i])
		}
	}
	return nil
}

// RemoveBulk removes all of the keys passed in at once.
func (d *RemoteDB) RemoveBulk(keys []string) (int64, error) {

	var err error
	cKeys := C.make_char_array(C.int(len(keys)))
	defer C.free_char_array(cKeys, C.int(len(keys)))
	next := 0;
	for _, s := range (keys) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		next++
	}

	n := int64(C.ktdbremovebulkbinary(d.db, cKeys, C.size_t(len(keys))))
	if n == -1 {
		err = d.LastError()
	}
	return n, err
}

// SetBulk sets all of the keys passed in at once.
func (d *RemoteDB) SetBulk(keysAndVals map[string]string) (int64, error) {

	var err error
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	cVals := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	defer C.free_char_array(cVals, C.int(len(keysAndVals)))
	next := 0;
	for k, v := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(k), C.int(next))
        C.set_array_string(cVals, C.CString(v), C.int(next))
		next++
	}

	n := int64(C.ktdbsetbulkbinary(d.db, cKeys, C.size_t(len(keysAndVals)), cVals, C.size_t(len(keysAndVals))))
	if n == -1 {
		err = d.LastError()
	}
	return n, err
}

// Plays the passed in lua script, returning the result as a key->value map.
func (d *RemoteDB) PlayScript(script string, params map[string]string) (map[string]string, error) {

	result := map[string]string{}
	cScript := C.CString(script)
	defer C.free(unsafe.Pointer(cScript))

	paramSize := len(params) * 2
	cParams := C.make_char_array(C.int(paramSize))
	defer C.free_char_array(cParams, C.int(paramSize))
	next := 0;
	for k, v := range (params) {
        C.set_array_string(cParams, C.CString(k), C.int(next))
		next++
        C.set_array_string(cParams, C.CString(v), C.int(next))
		next++
	}

	strary := C.play_script(d.db, cScript, cParams, C.size_t(paramSize))
	if strary.v == nil {
		return nil, d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return result, nil
	}

	for i := int64(0); i < n; i++ {
		result[C.GoString(C.strary_item(&strary, C.int64_t(i)))] = C.GoString(C.strary_item(&strary, C.int64_t(i+1)))
		i++
	}
	return result, nil
}<commit_msg>fixing incorrect comment blocks<commit_after>// Copyright (C) 2013  gokabinet authors.
// Use of this source code is governed by a GPLv3
// license that can be found in the LICENSE file.

package kt

/*
#cgo pkg-config: kyototycoon
#include <ktlangc.h>
#include "kt.h"
*/
import "C"

import (
	"bytes"
	"fmt"
	"unsafe"
	"encoding/gob"
)

const (
	_ = iota
	DEFAULT_TIMEOUT = -1.
)

// KTError is used for errors using the gokabinet library.  It implements the
// builtin error interface.
type KTError string

func (err KTError) Error() string {
	return string(err)
}

// DB is the basic type for the gokabinet library. Holds an unexported instance
// of the database, for interactions.
type RemoteDB struct {
	db   *C.KTRDB
}

// Open opens a connection to a remote database.
//
func Open(host string, port int, timeout float32) (*RemoteDB, error) {
	d := &RemoteDB{db: C.ktdbnew()}
	cHost := C.CString(host)
	defer C.free(unsafe.Pointer(cHost))
	cPort := C.int32_t(port)
	cTimeout := C.double(timeout)

	if C.ktdbopen(d.db, cHost, cPort, cTimeout) == 0 {
		errMsg := d.LastError()
		return nil, KTError(fmt.Sprintf("Error opening %s:%d -- %s", host, port, errMsg))
	}
	return d, nil
}

// LastError returns a KTError instance representing the last occurred error in
// the database.
func (d *RemoteDB) LastError() error {
	errMsg := C.GoString(C.ktecodename(C.ktdbecode(d.db)))
	return KTError(errMsg)
}

// Close the connection to the db
func (d *RemoteDB) Close() {
	if (d != nil) {
		C.ktdbclose(d.db)
		C.ktdbdel(d.db)
	}
}

func (d *RemoteDB) Set(key, value string) error {

	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	cValue := C.CString(value)
	defer C.free(unsafe.Pointer(cValue))
	lKey := C.size_t(len(key))
	lValue := C.size_t(len(value))
	if C.ktdbset(d.db, cKey, lKey, cValue, lValue) == 0 {
		errMsg := d.LastError()
		return KTError(fmt.Sprintf("Failed to add a record with the value %s and the key %s: %s", value, key, errMsg))
	}
	return nil
}

// Get byte[] directly
func (d *RemoteDB) GetBytes(key string) ([]byte, error) {
	var resultLen C.size_t
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.ktdbget(d.db, cKey, lKey, &resultLen)
	defer C.free(unsafe.Pointer(cValue))
	if cValue == nil {
		errMsg := d.LastError()
		return []byte{}, KTError(fmt.Sprintf("Failed to get the record with the key %s: %s", key, errMsg))
	}
	return C.GoBytes(unsafe.Pointer(cValue), C.int(resultLen)), nil
}

func (d *RemoteDB) Get(key string) (string, error) {
	var resultLen C.size_t
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.ktdbget(d.db, cKey, lKey, &resultLen)
	defer C.free(unsafe.Pointer(cValue))
	if cValue == nil {
		errMsg := d.LastError()
		return "", KTError(fmt.Sprintf("Failed to get the record with the key %s: %s", key, errMsg))
	}
	return C.GoStringN(cValue, C.int(resultLen)), nil
}

// GetGob gets a record in the database by its key, decoding from gob format.
//
// Returns nil in case of success. In case of errors, it returns a KCError
// instance explaining what happened.
func (d *RemoteDB) GetGob(key string, e interface{}) error {
	data, err := d.Get(key)
	if err != nil {
		return err
	}
	buffer := bytes.NewBufferString(data)
	decoder := gob.NewDecoder(buffer)
	if err := decoder.Decode(e); err != nil {
		return KTError(fmt.Sprintf("Failed to decode the record with the key %s: %s", key, err))
	}
	return nil
}

// SetGob adds a record to the database, stored in gob format.
//
// Returns a KCError instance in case of errors, otherwise, returns nil.
func (d *RemoteDB) SetGob(key string, e interface{}) error {
	var buffer bytes.Buffer
	encoder := gob.NewEncoder(&buffer)
	err := encoder.Encode(e)
	if err != nil {
		return KTError(fmt.Sprintf("Failed to add a record with the value %s and the key %s: %s", e, key, err))
	}
	err = d.Set(key, buffer.String())
	return err
}

// Removes the key from the DB.
func (d *RemoteDB) Remove(key string) error {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	status := C.ktdbremove(d.db, cKey, lKey)
	if status == 0 {
		errMsg := d.LastError()
		return KTError(fmt.Sprintf("Failed to remove the record with the key %s: %s", key, errMsg))
	}
	return nil
}

// Clear removes all records from the database.
//
// Returns a KTError in case of failure.
func (d *RemoteDB) Clear() error {
	if C.ktdbclear(d.db) == 0 {
		msg := d.LastError()
		return KTError(fmt.Sprintf("Failed to clear the database: %s.", msg))
	}
	return nil
}

// Increment increments the value of a numeric record by a given number, and
// return the incremented value.
//
// In case of errors, returns 0 and a KCError instance with detailed error
// message.
func (d *RemoteDB) Increment(key string, number int) (int, error) {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	lKey := C.size_t(len(key))
	cValue := C.int64_t(number)
	v := C.ktdbincrint(d.db, cKey, lKey, cValue, 0)
	if v == C.INT64_MIN {
		return 0, KTError("It's not possible to increment a non-numeric record")
	}
	return int(v), nil
}

// Returns the number of elements
func (d *RemoteDB) Count() (int, error) {
	var err error
	v := int(C.ktdbcount(d.db))
	if v == -1 {
		err = d.LastError()
	}
	return v, err
}

// MatchPrefix returns a list of keys that matches a prefix or an error in case
// of failure.
func (d *RemoteDB) MatchPrefix(prefix string, max int64) ([]string, error) {
	cprefix := C.CString(prefix)
	defer C.free(unsafe.Pointer(cprefix))
	strary := C.match_prefix(d.db, cprefix, C.size_t(max))
	if strary.v == nil {
		return nil, d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil, nil
	}
	result := make([]string, n)
	for i := int64(0); i < n; i++ {
		result[i] = C.GoString(C.strary_item(&strary, C.int64_t(i)))
	}
	return result, nil
}

// GetBulk fetches all keys in the given map. If a key does not exist, it is deleted from the map before being returned.
// If the key does exist, the value in the map is set accordingly.
func (d *RemoteDB) GetBulk(keysAndVals map[string]string) (error) {

	keyList := make([]string, len(keysAndVals))
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	next := 0;
	for s, _ := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		keyList[next] = s
		next++
	}

	strary := C.get_bulk_binary(d.db, cKeys, C.size_t(len(keysAndVals)))
	if strary.v == nil {
		return d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil
	}

	for i := int64(0); i < n; i++ {
		if C.bool(C.strary_present(&strary, C.int64_t(i))) {
			keysAndVals[keyList[i]] = C.GoString(C.strary_item(&strary, C.int64_t(i)))
		} else {
			delete(keysAndVals, keyList[i])
		}
	}
	return nil
}

// GetBulkBytes fetches all keys in the given map. If a key does not exist, it is deleted from the map before being returned.
// If the key does exist, the value in the map is set accordingly.
func (d *RemoteDB) GetBulkBytes(keysAndVals map[string][]byte) (error) {

	keyList := make([]string, len(keysAndVals))
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	next := 0;
	for s, _ := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		keyList[next] = s
		next++
	}

	strary := C.get_bulk_binary(d.db, cKeys, C.size_t(len(keysAndVals)))
	if strary.v == nil {
		return d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return nil
	}

	for i := int64(0); i < n; i++ {
		if C.bool(C.strary_present(&strary, C.int64_t(i))) {
			keysAndVals[keyList[i]] = C.GoBytes(unsafe.Pointer(C.strary_item(&strary, C.int64_t(i))), C.int(C.strary_size(&strary, C.int64_t(i))))
		} else {
			delete(keysAndVals, keyList[i])
		}
	}
	return nil
}

// RemoveBulk removes all of the keys passed in at once.
func (d *RemoteDB) RemoveBulk(keys []string) (int64, error) {

	var err error
	cKeys := C.make_char_array(C.int(len(keys)))
	defer C.free_char_array(cKeys, C.int(len(keys)))
	next := 0;
	for _, s := range (keys) {
        C.set_array_string(cKeys, C.CString(s), C.int(next))
		next++
	}

	n := int64(C.ktdbremovebulkbinary(d.db, cKeys, C.size_t(len(keys))))
	if n == -1 {
		err = d.LastError()
	}
	return n, err
}

// SetBulk sets all of the keys passed in at once.
func (d *RemoteDB) SetBulk(keysAndVals map[string]string) (int64, error) {

	var err error
	cKeys := C.make_char_array(C.int(len(keysAndVals)))
	cVals := C.make_char_array(C.int(len(keysAndVals)))
	defer C.free_char_array(cKeys, C.int(len(keysAndVals)))
	defer C.free_char_array(cVals, C.int(len(keysAndVals)))
	next := 0;
	for k, v := range (keysAndVals) {
        C.set_array_string(cKeys, C.CString(k), C.int(next))
        C.set_array_string(cVals, C.CString(v), C.int(next))
		next++
	}

	n := int64(C.ktdbsetbulkbinary(d.db, cKeys, C.size_t(len(keysAndVals)), cVals, C.size_t(len(keysAndVals))))
	if n == -1 {
		err = d.LastError()
	}
	return n, err
}

// Plays the passed in lua script, returning the result as a key->value map.
func (d *RemoteDB) PlayScript(script string, params map[string]string) (map[string]string, error) {

	result := map[string]string{}
	cScript := C.CString(script)
	defer C.free(unsafe.Pointer(cScript))

	paramSize := len(params) * 2
	cParams := C.make_char_array(C.int(paramSize))
	defer C.free_char_array(cParams, C.int(paramSize))
	next := 0;
	for k, v := range (params) {
        C.set_array_string(cParams, C.CString(k), C.int(next))
		next++
        C.set_array_string(cParams, C.CString(v), C.int(next))
		next++
	}

	strary := C.play_script(d.db, cScript, cParams, C.size_t(paramSize))
	if strary.v == nil {
		return nil, d.LastError()
	}
	defer C.free_strary(&strary)
	n := int64(strary.n)
	if n == 0 {
		return result, nil
	}

	for i := int64(0); i < n; i++ {
		result[C.GoString(C.strary_item(&strary, C.int64_t(i)))] = C.GoString(C.strary_item(&strary, C.int64_t(i+1)))
		i++
	}
	return result, nil
}<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
)

const VERSION = `0.1`

// Flags
var (
	version = flag.Bool("V", false, "Display version information and exit")
	useTLS = flag.Bool("tls", false, "Use TLS")
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("netcat-sec %s\n", VERSION)
	}

	args := flag.Args()
	if len(args) != 2 {
		fmt.Fprintln(os.Stderr, "Need host and port")
		return
	}
	host, port := args[0], args[1]

	var (
		conn net.Conn
		err  error
	)

	if *useTLS {
		conn, err = tls.Dial("tcp", host+":"+port, nil)
	} else {
		conn, err = net.Dial("tcp", host+":"+port)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	go func() {
		r := bufio.NewReader(os.Stdin)
		for {
			l, _, err := r.ReadLine()
			if err != nil {
				os.Exit(0)
			}
			_, err = conn.Write([]byte(string(l)+"\n"))
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		}
	}()

	go func() {
		r := bufio.NewReader(conn)
		for {
			l, _, err := r.ReadLine()
			if err != nil {
				if err == io.EOF {
					os.Exit(0)
				}
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			fmt.Println(string(l))
		}
	}()

	<-make(chan interface{})
}
<commit_msg>Setting default<commit_after>package main

import (
	"bufio"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
)

const VERSION = `0.1`

// Flags
var (
	version = flag.Bool("V", false, "Display version information and exit")
	useTLS = flag.Bool("tls", true, "Use TLS")
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("netcat-sec %s\n", VERSION)
	}

	args := flag.Args()
	if len(args) != 2 {
		fmt.Fprintln(os.Stderr, "Need host and port")
		return
	}
	host, port := args[0], args[1]

	var (
		conn net.Conn
		err  error
	)

	if *useTLS {
		conn, err = tls.Dial("tcp", host+":"+port, nil)
	} else {
		conn, err = net.Dial("tcp", host+":"+port)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	go func() {
		r := bufio.NewReader(os.Stdin)
		for {
			l, _, err := r.ReadLine()
			if err != nil {
				os.Exit(0)
			}
			_, err = conn.Write([]byte(string(l)+"\n"))
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		}
	}()

	go func() {
		r := bufio.NewReader(conn)
		for {
			l, _, err := r.ReadLine()
			if err != nil {
				if err == io.EOF {
					os.Exit(0)
				}
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			fmt.Println(string(l))
		}
	}()

	<-make(chan interface{})
}
<|endoftext|>


<commit_before>package nn

import (
	"fmt"
	"github.com/jcla1/matrix"
	"math"
)

var (
	_ = fmt.Println
)

type TrainingExample struct {
	Input, ExpectedOutput *matrix.Matrix
}

type Parameters []*matrix.Matrix
type Deltas []*matrix.Matrix

func CostFunction(data []TrainingExample, thetas Parameters, lambda float64) float64 {
	cost := float64(0)
	var estimation []float64
	var expected_output []float64

	// Cost

	for _, datum := range data {
		estimation = Hypothesis(thetas, datum).Values()
		expected_output = datum.ExpectedOutput.Values()

		for k, y := range expected_output {
			// heart of the cost function
			cost += y*math.Log(estimation[k]) + (1-y)*math.Log(1-estimation[k])
		}
	}

	// Regularization
	regularizationCost := float64(0)

	for _, theta := range thetas {
		for i, param := range theta.Values() {

			// ignore theta0
			if i%theta.Columns() == 0 {
				continue
			}

			regularizationCost += param * param
		}
	}

	return cost/float64(len(data)) + (lambda/(2*float64(len(data))))*regularizationCost
}

func Hypothesis(thetas Parameters, trainingEx TrainingExample) *matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	for _, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(curValues.InsertRows(biasValueMatrix, 0)).Apply(sigmoidMatrix)
	}

	return curValues
}

func HypothesisHistory(thetas Parameters, trainingEx TrainingExample) []*matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	history := make([]*matrix.Matrix, 0, len(thetas)+1)
	history = append(history, curValues.InsertRows(biasValueMatrix, 0))

	for i, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(history[len(history)-1]).Apply(sigmoidMatrix)

		if i != len(thetas)-1 {
			history = append(history, curValues.InsertRows(biasValueMatrix, 0))
		} else {
			history = append(history, curValues)
		}

	}

	return history
}

func DeltaTerms(thetas Parameters, trainingEx TrainingExample) Deltas {
	deltas := make(Deltas, len(thetas))

	biasValueMatrix := matrix.Ones(1, 1)

	deltas[len(deltas)-1], _ = Hypothesis(thetas, trainingEx).Sub(trainingEx.ExpectedOutput)

	for i := len(deltas) - 2; i >= 0; i-- {
		workingTheta := thetas[i+1]

		levelPrediction := Hypothesis(thetas[:i+1], trainingEx).InsertRows(biasValueMatrix, 0)
		tmp, _ := matrix.Ones(levelPrediction.Rows(), 1).Sub(levelPrediction)
		levelGradient := levelPrediction.Dot(tmp)

		deltas[i] = workingTheta.Transpose().Mul(deltas[i+1]).Dot(levelGradient).RemoveRow(1)
	}

	return deltas
}

func BackProp(thetas Parameters, trainingSet []TrainingExample, lambda float64) []*matrix.Matrix {
	// Make new BigDelta matrix
	bigDeltas := make(Deltas, len(thetas))

	for i, _ := range bigDeltas {
		bigDeltas[i] = matrix.Zeros(thetas[i].Rows(), thetas[i].Columns())
	}

	// Make new gradient matrix
	gradients := make([]*matrix.Matrix, len(thetas))

	var activations []*matrix.Matrix
	var deltaTerms Deltas

	for _, trainingEx := range trainingSet {
		activations = HypothesisHistory(thetas, trainingEx)
		deltaTerms = DeltaTerms(thetas, trainingEx)

		for i := 0; i < len(deltaTerms); i++ {
			bigDeltas[i], _ = bigDeltas[i].Add(deltaTerms[i].Mul(activations[i].Transpose()))
		}
	}

	for i, _ := range gradients {
		biasSanitizer := matrix.Eye(thetas[i].Columns())
		biasSanitizer.Set(1, 1, 0)

		sanitizedTheta := thetas[i].Mul(biasSanitizer).Scale(lambda)

		summedGradients, _ := bigDeltas[i].Add(sanitizedTheta)

		gradients[i] = summedGradients.Scale(1 / float64(len(trainingSet)))
	}

	return gradients
}

// Helper functions

func sigmoidMatrix(index int, value float64) float64 {
	return sigmoid(value)
}

func sigmoid(z float64) float64 {
	return 1 / (1 + math.Pow(math.E, -z))
}
<commit_msg>in matrix lib Dot was actually element-wise product<commit_after>package nn

import (
	"fmt"
	"github.com/jcla1/matrix"
	"math"
)

var (
	_ = fmt.Println
)

type TrainingExample struct {
	Input, ExpectedOutput *matrix.Matrix
}

type Parameters []*matrix.Matrix
type Deltas []*matrix.Matrix

func CostFunction(data []TrainingExample, thetas Parameters, lambda float64) float64 {
	cost := float64(0)
	var estimation []float64
	var expected_output []float64

	// Cost

	for _, datum := range data {
		estimation = Hypothesis(thetas, datum).Values()
		expected_output = datum.ExpectedOutput.Values()

		for k, y := range expected_output {
			// heart of the cost function
			cost += y*math.Log(estimation[k]) + (1-y)*math.Log(1-estimation[k])
		}
	}

	// Regularization
	regularizationCost := float64(0)

	for _, theta := range thetas {
		for i, param := range theta.Values() {

			// ignore theta0
			if i%theta.Columns() == 0 {
				continue
			}

			regularizationCost += param * param
		}
	}

	return cost/float64(len(data)) + (lambda/(2*float64(len(data))))*regularizationCost
}

func Hypothesis(thetas Parameters, trainingEx TrainingExample) *matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	for _, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(curValues.InsertRows(biasValueMatrix, 0)).Apply(sigmoidMatrix)
	}

	return curValues
}

func HypothesisHistory(thetas Parameters, trainingEx TrainingExample) []*matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	history := make([]*matrix.Matrix, 0, len(thetas)+1)
	history = append(history, curValues.InsertRows(biasValueMatrix, 0))

	for i, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(history[len(history)-1]).Apply(sigmoidMatrix)

		if i != len(thetas)-1 {
			history = append(history, curValues.InsertRows(biasValueMatrix, 0))
		} else {
			history = append(history, curValues)
		}

	}

	return history
}

func DeltaTerms(thetas Parameters, trainingEx TrainingExample) Deltas {
	deltas := make(Deltas, len(thetas))

	biasValueMatrix := matrix.Ones(1, 1)

	deltas[len(deltas)-1], _ = Hypothesis(thetas, trainingEx).Sub(trainingEx.ExpectedOutput)

	for i := len(deltas) - 2; i >= 0; i-- {
		workingTheta := thetas[i+1]

		levelPrediction := Hypothesis(thetas[:i+1], trainingEx).InsertRows(biasValueMatrix, 0)
		tmp, _ := matrix.Ones(levelPrediction.Rows(), 1).Sub(levelPrediction)
		levelGradient := levelPrediction.EWProd(tmp)

		deltas[i] = workingTheta.Transpose().Mul(deltas[i+1]).EWProd(levelGradient).RemoveRow(1)
	}

	return deltas
}

func BackProp(thetas Parameters, trainingSet []TrainingExample, lambda float64) []*matrix.Matrix {
	// Make new BigDelta matrix
	bigDeltas := make(Deltas, len(thetas))

	for i, _ := range bigDeltas {
		bigDeltas[i] = matrix.Zeros(thetas[i].Rows(), thetas[i].Columns())
	}

	// Make new gradient matrix
	gradients := make([]*matrix.Matrix, len(thetas))

	var activations []*matrix.Matrix
	var deltaTerms Deltas

	for _, trainingEx := range trainingSet {
		activations = HypothesisHistory(thetas, trainingEx)
		deltaTerms = DeltaTerms(thetas, trainingEx)

		for i := 0; i < len(deltaTerms); i++ {
			bigDeltas[i], _ = bigDeltas[i].Add(deltaTerms[i].Mul(activations[i].Transpose()))
		}
	}

	for i, _ := range gradients {
		biasSanitizer := matrix.Eye(thetas[i].Columns())
		biasSanitizer.Set(1, 1, 0)

		sanitizedTheta := thetas[i].Mul(biasSanitizer).Scale(lambda)

		summedGradients, _ := bigDeltas[i].Add(sanitizedTheta)

		gradients[i] = summedGradients.Scale(1 / float64(len(trainingSet)))
	}

	return gradients
}

// Helper functions

func sigmoidMatrix(index int, value float64) float64 {
	return sigmoid(value)
}

func sigmoid(z float64) float64 {
	return 1 / (1 + math.Pow(math.E, -z))
}
<|endoftext|>


<commit_before>// +build !windows

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
)

var (
	gDefaultShell      = "/bin/sh"
	gDefaultSocketProt = "unix"
	gDefaultSocketPath string
)

var (
	gUser       *user.User
	gConfigPath string
)

func init() {
	var err error

	gUser, err = user.Current()
	if err != nil {
		log.Printf("user: %s", err)
	}

	config := os.Getenv("XDG_CONFIG_HOME")
	if config == "" {
		config = filepath.Join(gUser.HomeDir, ".config")
	}

	gConfigPath = filepath.Join(config, "lf", "lfrc")

	gDefaultSocketPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.sock", gUser.Username))
}

func pauseCommand() *exec.Cmd {
	c := `echo
	      echo -n 'Press any key to continue'
	      old=$(stty -g)
	      stty raw -echo
	      eval "ignore=\$(dd bs=1 count=1 2> /dev/null)"
	      stty $old
	      echo`

	return exec.Command(gOpts.shell, "-c", c)
}

func shellCommand(s string, args []string) *exec.Cmd {
	if len(gOpts.ifs) != 0 {
		s = fmt.Sprintf("IFS='%s'; %s", gOpts.ifs, s)
	}

	args = append([]string{"-c", s, "--"}, args...)

	return exec.Command(gOpts.shell, args...)
}

func putCommand(list []string, dir *dir, copy bool) *exec.Cmd {
	var sh string
	var args []string

	if copy {
		sh = "cp"
		args = append(args, "-R")
	} else {
		sh = "mv"
	}
	args = append(args, list...)
	args = append(args, dir.path)

	return exec.Command(sh, args...)
}

func moveCursor(y, x int) {
	fmt.Printf("\033[%d;%dH", y, x)
}
<commit_msg>add interactive flag to cp/mv to avoid overwrite<commit_after>// +build !windows

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
)

var (
	gDefaultShell      = "/bin/sh"
	gDefaultSocketProt = "unix"
	gDefaultSocketPath string
)

var (
	gUser       *user.User
	gConfigPath string
)

func init() {
	var err error

	gUser, err = user.Current()
	if err != nil {
		log.Printf("user: %s", err)
	}

	config := os.Getenv("XDG_CONFIG_HOME")
	if config == "" {
		config = filepath.Join(gUser.HomeDir, ".config")
	}

	gConfigPath = filepath.Join(config, "lf", "lfrc")

	gDefaultSocketPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.sock", gUser.Username))
}

func pauseCommand() *exec.Cmd {
	c := `echo
	      echo -n 'Press any key to continue'
	      old=$(stty -g)
	      stty raw -echo
	      eval "ignore=\$(dd bs=1 count=1 2> /dev/null)"
	      stty $old
	      echo`

	return exec.Command(gOpts.shell, "-c", c)
}

func shellCommand(s string, args []string) *exec.Cmd {
	if len(gOpts.ifs) != 0 {
		s = fmt.Sprintf("IFS='%s'; %s", gOpts.ifs, s)
	}

	args = append([]string{"-c", s, "--"}, args...)

	return exec.Command(gOpts.shell, args...)
}

func putCommand(list []string, dir *dir, copy bool) *exec.Cmd {
	var sh string
	var args []string

	if copy {
		sh = "cp"
		args = append(args, "-R")
	} else {
		sh = "mv"
	}

	// XXX: POSIX standard states that -i flag shall do nothing when the
	// response is not affirmative. Since this command is run with a nil stdin,
	// it should not give an affirmative answer and in return this command
	// should not overwrite existing files. Our intention here is to use the
	// standard -i flag in place of non-standard -n flag to avoid overwrites.
	args = append(args, "-i")

	args = append(args, list...)
	args = append(args, dir.path)

	return exec.Command(sh, args...)
}

func moveCursor(y, x int) {
	fmt.Printf("\033[%d;%dH", y, x)
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "net/http"
  "github.com/codegangsta/martini"
  "github.com/codegangsta/martini-contrib/render"
  "github.com/codegangsta/martini-contrib/sessions"
  "os"
  "encoding/json"
  "github.com/sirsean/go-mailgun/mailgun"
  "math/rand"
  "log"
)

type Configuration struct
{
  Domain string;
  Port string;
  MG_API_KEY string;
  MG_DOMAIN string;
  CookieSecret string;
}

func loadConfiguration() *Configuration {
  file, _ := os.Open("config.json")
  decoder := json.NewDecoder(file)
  configuration := &Configuration{}
  decoder.Decode(&configuration)
  fmt.Println(configuration)
  return configuration;
}

func main() {
  var config *Configuration;
  config = loadConfiguration();
  fmt.Printf("%s is domain", config.Port);
  fmt.Printf("Domain: %s", config.Domain);
  m := martini.Classic()

  //m.Use(render.Renderer())
  m.Use(render.Renderer(render.Options{
    Directory: "templates", // Specify what path to load the templates from.
    Layout: "layout", // Specify a layout template. Layouts can call {{ yield }} to render the current template.
    Extensions: []string{".tmpl", ".html"}, // Specify extensions to load for templates.
    //Funcs: []template.FuncMap{AppHelpers}, // Specify helper function maps for templates to access.
    //Delims: render.Delims{"{[{", "}]}"}, // Sets delimiters to the specified strings.
    Charset: "UTF-8", // Sets encoding for json and html content-types.
  }))

  store := sessions.NewCookieStore([]byte(config.CookieSecret))
  m.Use(sessions.Sessions("my_session", store))

  m.Get("/", func(r render.Render, session sessions.Session) {
    type Site struct {
      Domain string
      VisitCount  uint
    }
    site := Site{config.Domain, 12}
    rand := rand.New(rand.NewSource(99))
    csrf := fmt.Sprintf("%f", rand.Float64()*1000000)
    session.Set("CSRF", csrf)
    var msg string;
    if nil != session.Get("FlashMessage") {
      msg = session.Get("FlashMessage").(string)
      session.Delete("FlashMessage")
    } else {
      msg = ""
    }

    r.HTML(200, "hello", struct{A,B, CSRF, FlashMessage string; Site Site}{ "Guesis", "bar", csrf, msg, site })
    //return "
Hello, world!
"
  })

  m.Get("/download", func (r render.Render, session sessions.Session) {

  })

  m.Post("/support", func (res http.ResponseWriter, req *http.Request, params martini.Params, session sessions.Session) {
    var config *Configuration
    config = loadConfiguration()
    mg_client := mailgun.NewClient(config.MG_API_KEY, config.MG_DOMAIN)

    req.ParseForm()
    p := req.Form
    log.Println(req.Form)
    if session.Get("CSRF") == p["csrf"][0] {
      message1 := mailgun.Message{
        FromName: "QSlider",
        FromAddress: "qslide@mg.axcoto.com",
        ToAddress: "kureikain@gmail.com",
        Subject: "Ticket on qslide.axcoto.com",
        Body: "Contact from: " + p["youremail"][0] + " with content: \n
" + p["content"][0],
      }

      fmt.Println("Attempting to send to ", mg_client.Endpoint(message1))

      body, err := mg_client.Send(message1)
      if err != nil {
        fmt.Println("Got an error:", err)
      } else {
        fmt.Println(body)
        session.Set("FlashMessage", "Message sent")
        http.Redirect(res, req, "http://slide.dev", 302)
        //http.Redirect(res, req, "http://slide.dev", http.StatusTemporaryRedirect)
      }
    }
  })

  m.Get("/tutorial", func (r render.Render) {
  })

  //m.Run()
  http.ListenAndServe(fmt.Sprintf(":%s", config.Port), m)
  //http.ListenAndServe(":10005", m)
}
<commit_msg>Get redirect domain in config file<commit_after>package main

import (
  "fmt"
  "net/http"
  "github.com/codegangsta/martini"
  "github.com/codegangsta/martini-contrib/render"
  "github.com/codegangsta/martini-contrib/sessions"
  "os"
  "encoding/json"
  "github.com/sirsean/go-mailgun/mailgun"
  "math/rand"
  "log"
)

type Configuration struct
{
  Domain string;
  Port string;
  MG_API_KEY string;
  MG_DOMAIN string;
  CookieSecret string;
}

func loadConfiguration() *Configuration {
  file, _ := os.Open("config.json")
  decoder := json.NewDecoder(file)
  configuration := &Configuration{}
  decoder.Decode(&configuration)
  fmt.Println(configuration)
  return configuration;
}

func main() {
  var config *Configuration;
  config = loadConfiguration();
  fmt.Printf("%s is domain", config.Port);
  fmt.Printf("Domain: %s", config.Domain);
  m := martini.Classic()

  //m.Use(render.Renderer())
  m.Use(render.Renderer(render.Options{
    Directory: "templates", // Specify what path to load the templates from.
    Layout: "layout", // Specify a layout template. Layouts can call {{ yield }} to render the current template.
    Extensions: []string{".tmpl", ".html"}, // Specify extensions to load for templates.
    //Funcs: []template.FuncMap{AppHelpers}, // Specify helper function maps for templates to access.
    //Delims: render.Delims{"{[{", "}]}"}, // Sets delimiters to the specified strings.
    Charset: "UTF-8", // Sets encoding for json and html content-types.
  }))

  store := sessions.NewCookieStore([]byte(config.CookieSecret))
  m.Use(sessions.Sessions("my_session", store))

  m.Get("/", func(r render.Render, session sessions.Session) {
    type Site struct {
      Domain string
      VisitCount  uint
    }
    site := Site{config.Domain, 12}
    rand := rand.New(rand.NewSource(99))
    csrf := fmt.Sprintf("%f", rand.Float64()*1000000)
    session.Set("CSRF", csrf)
    var msg string;
    if nil != session.Get("FlashMessage") {
      msg = session.Get("FlashMessage").(string)
      session.Delete("FlashMessage")
    } else {
      msg = ""
    }

    r.HTML(200, "hello", struct{A,B, CSRF, FlashMessage string; Site Site}{ "Guesis", "bar", csrf, msg, site })
    //return "
Hello, world!
"
  })

  m.Get("/download", func (r render.Render, session sessions.Session) {

  })

  m.Post("/support", func (res http.ResponseWriter, req *http.Request, params martini.Params, session sessions.Session) {
    var config *Configuration
    config = loadConfiguration()
    mg_client := mailgun.NewClient(config.MG_API_KEY, config.MG_DOMAIN)

    req.ParseForm()
    p := req.Form
    log.Println(req.Form)
    if session.Get("CSRF") == p["csrf"][0] {
      message1 := mailgun.Message{
        FromName: "QSlider",
        FromAddress: "qslide@mg.axcoto.com",
        ToAddress: "kureikain@gmail.com",
        Subject: "Ticket on qslide.axcoto.com",
        Body: "Contact from: " + p["youremail"][0] + " with content: \n
" + p["content"][0],
      }

      fmt.Println("Attempting to send to ", mg_client.Endpoint(message1))

      body, err := mg_client.Send(message1)
      if err != nil {
        fmt.Println("Got an error:", err)
      } else {
        fmt.Println(body)
        session.Set("FlashMessage", "Message sent")
        http.Redirect(res, req, config.Domain, 302)
        //http.Redirect(res, req, "http://slide.dev", http.StatusTemporaryRedirect)
      }
    }
  })

  m.Get("/tutorial", func (r render.Render) {
  })

  //m.Run()
  http.ListenAndServe(fmt.Sprintf(":%s", config.Port), m)
  //http.ListenAndServe(":10005", m)
}
<|endoftext|>


<commit_before>package logging

// Re returns a new logger that annotates log messages with the
// appropriate context (as defined by the Annotater)
func (l *Logger) Re(a Annotater) *Logger {
	lcopy := *l
	lcopy.annotater = a
	return &lcopy
}

type Annotater interface {
	Annotate(*Record)
}
<commit_msg>Allow log contextualization Re() to stack<commit_after>package logging

type Annotater interface {
	Annotate(*Record)
}

// an annotator stack level
type stacked struct {
	lower, higher Annotater
}

func (s stacked) Annotate(rec *Record) {
	s.lower.Annotate(rec)
	s.higher.Annotate(rec)
}

// Re returns a new logger that annotates log messages with the
// appropriate context (as defined by the Annotater)
func (l *Logger) Re(a Annotater) *Logger {

	// if there's an existing annotator, stack them together
	if l.annotater != nil {
		a = stacked{l.annotater, a}
	}

	lcopy := *l
	lcopy.annotater = a
	return &lcopy
}
<|endoftext|>


<commit_before>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

// https://github.com/goamz/goamz/blob/master/aws/aws.go
// https://github.com/goamz/goamz/blob/master/s3/s3.go

import (
	"bufio"
	"fmt"
	"github.com/goamz/goamz/aws"
	aws_s3 "github.com/goamz/goamz/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Sync struct {
	ACL           aws_s3.ACL
	Bucket        aws_s3.Bucket
	Prefix        string
	WorkPool      tunny.WorkPool
	Logger        *log.WOFLogger
	Debug         bool
	Success       int64
	Error         int64
	Skipped       int64
	Scheduled     int64
	Completed     int64
	Retried       int64
	TimeToProcess *time.Duration
	Retries       *pool.LIFOPool
	MaxRetries    float64 // max percentage of errors over scheduled
}

func NewSync(auth aws.Auth, region aws.Region, acl aws_s3.ACL, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	logger.Info("creating a new Sync thing-y with %d processes", procs)

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()

	retries := pool.NewLIFOPool()

	s := aws_s3.New(auth, region)
	b := s.Bucket(bucket)

	ttp := new(time.Duration)

	return &Sync{
		ACL:           acl,
		Bucket:        *b,
		Prefix:        prefix,
		WorkPool:      *workpool,
		Debug:         debug,
		Logger:        logger,
		Scheduled:     0,
		Completed:     0,
		Skipped:       0,
		Error:         0,
		Success:       0,
		Retried:       0,
		TimeToProcess: ttp,
		Retries:       retries,
		MaxRetries:    25.0, // maybe allow this to be user-defined ?
	}
}

func WOFSync(auth aws.Auth, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	return NewSync(auth, aws.USEast, aws_s3.PublicRead, bucket, prefix, procs, debug, logger)
}

func (sink *Sync) SyncDirectory(root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	callback := func(source string, info os.FileInfo) error {

		if info.IsDir() {
			return nil
		}

		err := sink.SyncFile(source, root)

		if err != nil {
			sink.Logger.Error("failed to sync %s, because '%s'", source, err)
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFiles(files []string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	for _, path := range files {

		wg.Add(1)

		go func() {
			defer wg.Done()
			sink.SyncFile(path, root)
		}()

	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFileList(path string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()

	wg := new(sync.WaitGroup)

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {

		path := scanner.Text()

		wg.Add(1)

		go func() {
			defer wg.Done()
			sink.SyncFile(path, root)
		}()

	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFile(source string, root string) error {

	atomic.AddInt64(&sink.Scheduled, 1)

	dest := source

	dest = strings.Replace(dest, root, "", -1)

	if sink.Prefix != "" {
		dest = path.Join(sink.Prefix, dest)
	}

	// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
	// which is a potential waste of time and resource. Or maybe we just
	// don't care? (20150930/thisisaaronland)

	sink.Logger.Debug("Looking for changes %s (%s)", dest, sink.Prefix)

	change, ch_err := sink.HasChanged(source, dest)

	if ch_err != nil {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Error, 1)
		sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)

		sink.Retries.Push(&pool.PoolString{String: source})
		return nil
	}

	if sink.Debug == true {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Skipped, 1)
		sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
		return nil
	}

	if !change {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Skipped, 1)
		sink.Logger.Debug("%s has not changed, skipping", source)
		return nil
	}

	err := sink.DoSyncFile(source, dest)

	if err != nil {
		sink.Retries.Push(&pool.PoolString{String: source})
		atomic.AddInt64(&sink.Error, 1)
	} else {
		atomic.AddInt64(&sink.Success, 1)
	}

	atomic.AddInt64(&sink.Completed, 1)
	return err
}

func (sink *Sync) DoSyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	_, err = sink.WorkPool.SendWork(func() {

		sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

		o := aws_s3.Options{}

		err := sink.Bucket.Put(dest, body, "text/plain", sink.ACL, o)

		if err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		}

	})

	if err != nil {
		sink.Logger.Error("failed to schedule %s for processing, because '%s'", source, err)
		return err
	}

	sink.Logger.Debug("scheduled %s for processing", source)
	return nil
}

func (sink *Sync) HasChanged(source string, dest string) (ch bool, err error) {

	headers := make(http.Header)
	rsp, err := sink.Bucket.Head(dest, headers)

	if err != nil {
		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		return false, nil
	}

	// Okay so we think that things have changed but let's just check
	// modification times to be extra sure (20151112/thisisaaronland)

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()

	last_mod := rsp.Header.Get("Last-Modified")
	mtime_remote, err := time.Parse(time.RFC1123, last_mod)

	if err != nil {
		sink.Logger.Error("failed to parse timestamp %s because %s", last_mod, err)
		return false, err
	}

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	sink.Logger.Debug("local %s %s", mtime_local, source)
	sink.Logger.Debug("remote %s %s", mtime_remote, dest)

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	return true, nil
}

func (sink *Sync) ProcessRetries(root string) bool {

	to_retry := sink.Retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(sink.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > sink.MaxRetries {
			sink.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		sink.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for sink.Retries.Length() > 0 {

			r, ok := sink.Retries.Pop()

			if !ok {
				sink.Logger.Error("failed to pop retries because... computers?")
				break
			}

			source := r.StringValue()

			wg.Add(1)

			go func() {

				defer wg.Done()

				atomic.AddInt64(&sink.Scheduled, 1)

				sink.WorkPool.SendWork(func() {

					atomic.AddInt64(&sink.Retried, 1)

					sink.Logger.Info("retry syncing %s", source)

					sink.SyncFile(source, root)

					atomic.AddInt64(&sink.Completed, 1)
				})

			}()
		}

		wg.Wait()
	}

	return true
}

func (sink *Sync) MonitorStatus() {

	go func() {

		t0 := time.Now()

		for {

			rpt := sink.StatusReport()
			ttp := time.Since(t0)

			sink.Logger.Info("%s Time %v", rpt, ttp)

			time.Sleep(10 * time.Second)

			if sink.Scheduled == sink.Completed {
				break
			}
		}

		sink.Logger.Info(sink.StatusReport())
		sink.Logger.Info("monitoring complete")
	}()
}

func (sink *Sync) StatusReport() string {
	return fmt.Sprintf("Scheduled %d Completed %d Success %d Error %d Skipped %d Retried %d Goroutines %d",
		sink.Scheduled, sink.Completed, sink.Success, sink.Error, sink.Skipped, sink.Retried, runtime.NumGoroutine())
}
<commit_msg>remove E_EXCESSIVE_FILEHANDLES code<commit_after>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

// https://github.com/goamz/goamz/blob/master/aws/aws.go
// https://github.com/goamz/goamz/blob/master/s3/s3.go

import (
	"bufio"
	"fmt"
	"github.com/goamz/goamz/aws"
	aws_s3 "github.com/goamz/goamz/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Sync struct {
	ACL           aws_s3.ACL
	Bucket        aws_s3.Bucket
	Prefix        string
	WorkPool      tunny.WorkPool
	Logger        *log.WOFLogger
	Debug         bool
	Success       int64
	Error         int64
	Skipped       int64
	Scheduled     int64
	Completed     int64
	Retried       int64
	TimeToProcess *time.Duration
	Retries       *pool.LIFOPool
	MaxRetries    float64 // max percentage of errors over scheduled
}

func NewSync(auth aws.Auth, region aws.Region, acl aws_s3.ACL, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	logger.Info("creating a new Sync thing-y with %d processes", procs)

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()

	retries := pool.NewLIFOPool()

	s := aws_s3.New(auth, region)
	b := s.Bucket(bucket)

	ttp := new(time.Duration)

	return &Sync{
		ACL:           acl,
		Bucket:        *b,
		Prefix:        prefix,
		WorkPool:      *workpool,
		Debug:         debug,
		Logger:        logger,
		Scheduled:     0,
		Completed:     0,
		Skipped:       0,
		Error:         0,
		Success:       0,
		Retried:       0,
		TimeToProcess: ttp,
		Retries:       retries,
		MaxRetries:    25.0, // maybe allow this to be user-defined ?
	}
}

func WOFSync(auth aws.Auth, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	return NewSync(auth, aws.USEast, aws_s3.PublicRead, bucket, prefix, procs, debug, logger)
}

func (sink *Sync) SyncDirectory(root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	callback := func(source string, info os.FileInfo) error {

		if info.IsDir() {
			return nil
		}

		err := sink.SyncFile(source, root)

		if err != nil {
			sink.Logger.Error("failed to sync %s, because '%s'", source, err)
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFiles(files []string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	for _, path := range files {

		wg.Add(1)

		go func() {
			defer wg.Done()
			sink.SyncFile(path, root)
		}()

	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFileList(path string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {

		path := scanner.Text()

		_, err = sink.WorkPool.SendWork(func() {
			sink.SyncFile(path, root)
		})

		if err != nil {
			sink.Logger.Error("failed to schedule %s for processing, because '%s'", path, err)
		}

	}

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFile(source string, root string) error {

	atomic.AddInt64(&sink.Scheduled, 1)

	dest := source

	dest = strings.Replace(dest, root, "", -1)

	if sink.Prefix != "" {
		dest = path.Join(sink.Prefix, dest)
	}

	// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
	// which is a potential waste of time and resource. Or maybe we just
	// don't care? (20150930/thisisaaronland)

	sink.Logger.Debug("Looking for changes %s (%s)", dest, sink.Prefix)

	change, ch_err := sink.HasChanged(source, dest)

	if ch_err != nil {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Error, 1)
		sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)

		sink.Retries.Push(&pool.PoolString{String: source})
		return nil
	}

	if sink.Debug == true {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Skipped, 1)
		sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
		return nil
	}

	if !change {
		atomic.AddInt64(&sink.Completed, 1)
		atomic.AddInt64(&sink.Skipped, 1)
		sink.Logger.Debug("%s has not changed, skipping", source)
		return nil
	}

	err := sink.DoSyncFile(source, dest)

	if err != nil {
		sink.Retries.Push(&pool.PoolString{String: source})
		atomic.AddInt64(&sink.Error, 1)
	} else {
		atomic.AddInt64(&sink.Success, 1)
	}

	atomic.AddInt64(&sink.Completed, 1)
	return err
}

func (sink *Sync) DoSyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	_, err = sink.WorkPool.SendWork(func() {

		sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

		o := aws_s3.Options{}

		err := sink.Bucket.Put(dest, body, "text/plain", sink.ACL, o)

		if err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		}

	})

	if err != nil {
		sink.Logger.Error("failed to schedule %s for processing, because '%s'", source, err)
		return err
	}

	sink.Logger.Debug("scheduled %s for processing", source)
	return nil
}

func (sink *Sync) HasChanged(source string, dest string) (ch bool, err error) {

	headers := make(http.Header)
	rsp, err := sink.Bucket.Head(dest, headers)

	if err != nil {
		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		return false, nil
	}

	// Okay so we think that things have changed but let's just check
	// modification times to be extra sure (20151112/thisisaaronland)

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()

	last_mod := rsp.Header.Get("Last-Modified")
	mtime_remote, err := time.Parse(time.RFC1123, last_mod)

	if err != nil {
		sink.Logger.Error("failed to parse timestamp %s because %s", last_mod, err)
		return false, err
	}

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	sink.Logger.Debug("local %s %s", mtime_local, source)
	sink.Logger.Debug("remote %s %s", mtime_remote, dest)

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	return true, nil
}

func (sink *Sync) ProcessRetries(root string) bool {

	to_retry := sink.Retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(sink.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > sink.MaxRetries {
			sink.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		sink.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for sink.Retries.Length() > 0 {

			r, ok := sink.Retries.Pop()

			if !ok {
				sink.Logger.Error("failed to pop retries because... computers?")
				break
			}

			source := r.StringValue()

			wg.Add(1)

			go func() {

				defer wg.Done()

				atomic.AddInt64(&sink.Scheduled, 1)

				sink.WorkPool.SendWork(func() {

					atomic.AddInt64(&sink.Retried, 1)

					sink.Logger.Info("retry syncing %s", source)

					sink.SyncFile(source, root)

					atomic.AddInt64(&sink.Completed, 1)
				})

			}()
		}

		wg.Wait()
	}

	return true
}

func (sink *Sync) MonitorStatus() {

	go func() {

		t0 := time.Now()

		for {

			rpt := sink.StatusReport()
			ttp := time.Since(t0)

			sink.Logger.Info("%s Time %v", rpt, ttp)

			time.Sleep(10 * time.Second)

			if sink.Scheduled == sink.Completed {
				break
			}
		}

		sink.Logger.Info(sink.StatusReport())
		sink.Logger.Info("monitoring complete")
	}()
}

func (sink *Sync) StatusReport() string {
	return fmt.Sprintf("Scheduled %d Completed %d Success %d Error %d Skipped %d Retried %d Goroutines %d",
		sink.Scheduled, sink.Completed, sink.Success, sink.Error, sink.Skipped, sink.Retried, runtime.NumGoroutine())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"reflect"
	"sync"
)

var (
	name = flag.String("name", "worker", "a service name")
)

// T is an alias for interface{} to make code shorter.
type T interface{}

type Task struct {
	Inputs   []*Dataset
	Outputs  []*Dataset
	Function interface{}
}

func NewTask(inputs []*Dataset, outputs []*Dataset, function interface{}) *Task {
	return &Task{Inputs: inputs, Outputs: outputs, Function: function}
}

func (t *Task) Run() {
	// function
	// ft := reflect.TypeOf(t.Function)
	fn := reflect.ValueOf(t.Function)
	// input, type is same as parent Dataset's type
	// output chan, element type is same as current Dataset's type
	ch := t.Outputs[0].WriteChan
	// println("task run 1")
	for input := range t.Inputs[0].ReadChan {
		// println("task run 2", input.String())
		fn.Call([]reflect.Value{input, ch})
	}
}

type FlowContext struct {
	Tasks []*Task
}

type Dataset struct {
	Type      reflect.Type
	context   *FlowContext
	ReadChan  chan reflect.Value
	WriteChan reflect.Value
	ErrorChan chan error
	Parents   []*Dataset

	Generator func()
}

func NewDataset(context *FlowContext, t reflect.Type) *Dataset {
	ctype := reflect.ChanOf(reflect.BothDir, t)
	return &Dataset{
		context:   context,
		Type:      t,
		ReadChan:  make(chan reflect.Value, 0),
		WriteChan: reflect.MakeChan(ctype, 0),
		ErrorChan: make(chan error, 0),
	}
}

func (d *Dataset) DependsOn(parent *Dataset) {
	for _, p := range d.Parents {
		if p == parent {
			return
		}
	}
	d.Parents = append(d.Parents, parent)
}

func (d *Dataset) Map(f interface{}) (ret *Dataset) {
	t := reflect.TypeOf(f)
	if d.Type != t.In(0) {
		panic(fmt.Sprintf("The input %v does not match function input %v", d.Type, t.In(0)))
	}

	ret = NewDataset(d.context, t.In(1).Elem())
	ret.DependsOn(d)

	d.context.Tasks = append(d.context.Tasks, NewTask([]*Dataset{d}, []*Dataset{ret}, f))
	return
}

func (d *Dataset) RunSelf() {
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for ok := true; ok; {
			if t, ok := d.WriteChan.Recv(); ok {
				// fmt.Printf("%s -> r\n", t)
				d.ReadChan <- t
			}
		}
	}()
	if d.Generator != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			d.Generator()
		}()
	}
	wg.Wait()
	return
}

func NewFlowContext() *FlowContext {
	return &FlowContext{}
}

func main() {
	flag.Parse()

	NewFlowContext().TextFile("/etc/passwd").Map(func(line string, ch chan string) {
		println(line)
	}).Run()
}

func (fc *FlowContext) TextFile(fname string) (ret *Dataset) {
	ret = NewDataset(fc, reflect.TypeOf(""))
	ret.Generator = func() {
		// println("generate", fname)
		file, err := os.Open(fname)
		if err != nil {
			ret.ErrorChan <- err
			return
		}
		defer file.Close()

		defer ret.WriteChan.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			// println("w <-", scanner.Text())
			ret.WriteChan.Send(reflect.ValueOf(scanner.Text()))
		}

		if err := scanner.Err(); err != nil {
			ret.ErrorChan <- err
			return
		}
	}
	return
}

func (d *Dataset) Run() {
	var wg sync.WaitGroup

	// collect all dataset nodes
	initialDatasets := make(map[*Dataset]bool)
	for _, t := range d.context.Tasks {
		for _, d := range t.Inputs {
			initialDatasets[d] = true
		}
		for _, d := range t.Outputs {
			initialDatasets[d] = true
		}
	}

	// start all dataset nodes
	for k, _ := range initialDatasets {
		wg.Add(1)
		go func(k *Dataset) {
			defer wg.Done()
			k.RunSelf()
		}(k)
	}

	// start all task edges
	for _, t := range d.context.Tasks {
		wg.Add(1)
		go func(t *Task) {
			defer wg.Done()
			t.Run()
		}(t)
	}
	wg.Wait()
}
<commit_msg>working function flow<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"reflect"
	"strings"
	"sync"
)

var ()

type Task struct {
	Inputs   []*Dataset
	Outputs  []*Dataset
	Function func()
}

func NewTask(inputs []*Dataset, outputs []*Dataset) *Task {
	return &Task{Inputs: inputs, Outputs: outputs}
}

type FlowContext struct {
	Tasks []*Task
}

type Dataset struct {
	Type      reflect.Type
	context   *FlowContext
	ReadChan  chan reflect.Value
	WriteChan reflect.Value
	ErrorChan chan error
	Parents   []*Dataset

	Generator func()
}

func NewDataset(context *FlowContext, t reflect.Type) *Dataset {
	ctype := reflect.ChanOf(reflect.BothDir, t)
	return &Dataset{
		context:   context,
		Type:      t,
		ReadChan:  make(chan reflect.Value, 0),
		WriteChan: reflect.MakeChan(ctype, 0),
		ErrorChan: make(chan error, 0),
	}
}

func (d *Dataset) DependsOn(parent *Dataset) {
	for _, p := range d.Parents {
		if p == parent {
			return
		}
	}
	d.Parents = append(d.Parents, parent)
}

// f(A, chan B)
func (d *Dataset) Map(f interface{}) (ret *Dataset) {
	t := reflect.TypeOf(f)
	if d.Type != t.In(0) {
		panic(fmt.Sprintf("The input %v does not match function input %v", d.Type, t.In(0)))
	}

	ret = NewDataset(d.context, t.In(1).Elem())
	ret.DependsOn(d)

	task := NewTask([]*Dataset{d}, []*Dataset{ret})
	task.Function = func() {
		fn := reflect.ValueOf(f)
		// input, type is same as parent Dataset's type
		// output chan, element type is same as current Dataset's type
		ch := task.Outputs[0].WriteChan
		// println("map run 1")
		for input := range task.Inputs[0].ReadChan {
			println("map run 2", input.String())
			fn.Call([]reflect.Value{input, ch})
		}
	}

	d.context.Tasks = append(d.context.Tasks, task)
	return
}

// f(A)bool
func (d *Dataset) Filter(f interface{}) (ret *Dataset) {
	t := reflect.TypeOf(f)
	if d.Type != t.In(0) {
		panic(fmt.Sprintf("The input %v does not match function input %v", d.Type, t.In(0)))
	}

	ret = NewDataset(d.context, t.In(0))
	ret.DependsOn(d)

	task := NewTask([]*Dataset{d}, []*Dataset{ret})
	task.Function = func() {
		fn := reflect.ValueOf(f)
		// println("filter run 1")
		outChan := task.Outputs[0].WriteChan
		for input := range task.Inputs[0].ReadChan {
			println("filter run 2", input.String())
			outs := fn.Call([]reflect.Value{input})
			if outs[0].Bool() {
				outChan.Send(input)
			}
		}
	}

	d.context.Tasks = append(d.context.Tasks, task)
	return
}

func (t *Task) Run() {
	t.Function()
}

func (d *Dataset) RunSelf() {
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		var t reflect.Value
		for ok := true; ok; {
			if t, ok = d.WriteChan.Recv(); ok {
				// fmt.Printf("%s -> r\n", t)
				d.ReadChan <- t
			}
		}
		println("closing generator")
		close(d.ReadChan)
	}()
	if d.Generator != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			d.Generator()
			println("end of the file!")
		}()
	}
	wg.Wait()
	return
}

func NewFlowContext() *FlowContext {
	return &FlowContext{}
}

func main() {
	flag.Parse()

	TextFile("/etc/passwd").Map(func(line string, ch chan string) {
		ch <- line
	}).Filter(func(line string) bool {
		return !strings.HasPrefix(line, "#")
	}).Map(func(line string, ch chan string) {
		println(line)
	}).Run()

}

func TextFile(fname string) (ret *Dataset) {
	ret = NewDataset(NewFlowContext(), reflect.TypeOf(""))
	ret.Generator = func() {
		// println("generate", fname)
		file, err := os.Open(fname)
		if err != nil {
			ret.ErrorChan <- err
			return
		}
		defer file.Close()

		defer ret.WriteChan.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			// println("w <-", scanner.Text())
			ret.WriteChan.Send(reflect.ValueOf(scanner.Text()))
		}

		if err := scanner.Err(); err != nil {
			ret.ErrorChan <- err
			return
		}
	}
	return
}

func (d *Dataset) Run() {
	var wg sync.WaitGroup

	// collect all dataset nodes
	initialDatasets := make(map[*Dataset]bool)
	for _, t := range d.context.Tasks {
		for _, d := range t.Inputs {
			initialDatasets[d] = true
		}
		for _, d := range t.Outputs {
			initialDatasets[d] = true
		}
	}

	// start all dataset nodes
	for k, _ := range initialDatasets {
		wg.Add(1)
		go func(k *Dataset) {
			defer wg.Done()
			k.RunSelf()
			println("init dataset stopped!")
		}(k)
	}

	// start all task edges
	for _, t := range d.context.Tasks {
		wg.Add(1)
		go func(t *Task) {
			defer wg.Done()
			t.Run()
		}(t)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package gcache

import (
	"fmt"
	"net/http"

	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	rootFolderID string

	folderParams = &drive.File{
		Name:     rootFolderName,
		MimeType: mimeGSuiteFolder,
	}

	folderPermission *drive.Permission
	rootFolderName   string
)

// SetRootFolder sets name and permission to a top folder on Google Drive.
func SetRootFolder(
	name string,
	permission *drive.Permission,
) {
	folderPermission = permission
	rootFolderName = name
}

func getRootFolderID(
	r *http.Request,
) (
	string,
	error,
) {
	if rootFolderID != "" {
		return rootFolderID, nil
	}
	fileList, err := getFolder(r, fmt.Sprintf("name='%s'", rootFolderName), "")
	if err != nil {
		return "", err
	}

	if len(fileList.Files) == 1 {
		return fileList.Files[0].Id, nil
	}

	return createRootFolder(r)
}

func getFolder(
	r *http.Request,
	q string,
	field googleapi.Field,
) (
	*drive.FileList,
	error,
) {
	if q != "" {
		q = fmt.Sprintf("mimeType='%s' and (%s)", mimeGSuiteFolder, q)
	}
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, err
	}

retry:
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(q).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	return fileList, nil
}

// GetGDiveFolderIDs returns a map of folder IDs on Google Drive.
func GetGDiveFolderIDs(
	r *http.Request,
	q string,
) (
	*map[string]string,
	error,
) {

	const field googleapi.Field = "files(" +
		"name," +
		"id)"

	fileList, err := getFolder(r, q, field)
	if err != nil {
		return nil, err
	}

	result := make(map[string]string, len(fileList.Files))
	for _, v := range fileList.Files {
		result[v.Name] = v.Id
	}
	return &result, nil
}

func createRootFolder(
	r *http.Request,
) (
	string,
	error,
) {

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return "", err
	}

retryFiles:
	file, err := service.Files.Create(folderParams).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return "", err
		}
		if refreshToken {
			goto refresh
		}
		goto retryFiles
	}

retryPermissions:
	_, err = service.Permissions.Create(file.Id, folderPermission).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return "", err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return "", err
			}
		}
		goto retryPermissions
	}

	return file.Id, nil
}
<commit_msg>add CreateFolder<commit_after>package gcache

import (
	"errors"
	"fmt"
	"net/http"

	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	rootFolderID string

	folderPermission *drive.Permission
	rootFolderName   string
)

// SetRootFolder sets name and permission to a top folder on Google Drive.
func SetRootFolder(
	name string,
	permission *drive.Permission,
) {
	folderPermission = permission
	rootFolderName = name
}

func getRootFolderID(
	r *http.Request,
) (
	string,
	error,
) {
	if rootFolderID != "" {
		return rootFolderID, nil
	}
	fileList, err := getFolder(r, fmt.Sprintf("name='%s'", rootFolderName), "")
	if err != nil {
		return "", err
	}

	if len(fileList.Files) == 1 {
		return fileList.Files[0].Id, nil
	}

	return createRootFolder(r)
}

func getFolder(
	r *http.Request,
	q string,
	field googleapi.Field,
) (
	*drive.FileList,
	error,
) {
	if q != "" {
		q = fmt.Sprintf("mimeType='%s' and (%s)", mimeGSuiteFolder, q)
	}
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, err
	}

retry:
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(q).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	return fileList, nil
}

// GetGDiveFolderIDs returns a map of folder IDs on Google Drive.
func GetGDiveFolderIDs(
	r *http.Request,
	q string,
) (
	*map[string]string,
	error,
) {

	const field googleapi.Field = "files(" +
		"name," +
		"id)"

	fileList, err := getFolder(r, q, field)
	if err != nil {
		return nil, err
	}

	result := make(map[string]string, len(fileList.Files))
	for _, v := range fileList.Files {
		result[v.Name] = v.Id
	}
	return &result, nil
}

// CreateFolder returns a ID of new Google Drive Folder.
func CreateFolder(
	r *http.Request,
	name string,
) (
	string,
	error,
) {

	if name == "" {
		return "", errors.New("`name` must be enough")
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return "", err
	}

retryFiles:
	file, err := service.Files.Create(
		&drive.File{
			Name:     name,
			MimeType: mimeGSuiteFolder,
		},
	).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return "", err
		}
		if refreshToken {
			goto refresh
		}
		goto retryFiles
	}

retryPermissions:
	_, err = service.Permissions.Create(file.Id, folderPermission).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return "", err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return "", err
			}
		}
		goto retryPermissions
	}

	return file.Id, nil
}

func createRootFolder(
	r *http.Request,
) (
	string,
	error,
) {
	return CreateFolder(r, rootFolderName)
}
<|endoftext|>