<commit_before>// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package pinentry

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/keybase/client/go/logger"
)

//
// some borrowed from here:
//
//  https://github.com/bradfitz/camlistore/blob/master/pkg/misc/pinentry/pinentry.go
//
// Under the Apache 2.0 license
//

func canExec(s string) error {
	fi, err := os.Stat(s)
	if err != nil {
		return err
	}
	mode := fi.Mode()

	//
	// Only consider non-directories that have at least one +x
	//  bit set.
	//
	// TODO: Recheck this on windows!
	//   See here for lookpath: http://golang.org/src/pkg/os/exec/lp_windows.go
	//
	// Similar to check from exec.LookPath below
	//   See here: http://golang.org/src/pkg/os/exec/lp_unix.go
	//
	if mode.IsDir() {
		return fmt.Errorf("Program '%s' is a directory", s)
	} else if int(mode)&0111 == 0 {
		return fmt.Errorf("Program '%s' isn't executable", s)
	} else {
		return nil
	}
}

func FindPinentry(log logger.Logger) (string, error) {
	if HasWindows() {
		return "", fmt.Errorf("Can't spawn gui window, not using pinentry")
	}
	bins := []string{
		// If you install MacTools you'll wind up with this pinentry
		"/usr/local/MacGPG2/libexec/pinentry-mac.app/Contents/MacOS/pinentry-mac",
	}

	extraPaths := []string{}

	log.Debug("+ FindPinentry()")

	cmds := []string{
		"pinentry-gtk-2",
		"pinentry-qt4",
		"pinentry",
	}

	checkFull := func(s string) bool {
		log.Debug("| Check fullpath %s", s)
		found := (canExec(s) == nil)
		if found {
			log.Debug("- Found: %s", s)
		}
		return found
	}

	for _, b := range bins {
		if checkFull(b) {
			return b, nil
		}
	}

	path := os.Getenv("PATH")
	for _, c := range cmds {
		log.Debug("| Looking for %s in standard PATH %s", c, path)
		fullc, err := exec.LookPath(c)
		if err == nil {
			log.Debug("- Found %s", fullc)
			return fullc, nil
		}
	}

	for _, ep := range extraPaths {
		for _, c := range cmds {
			full := ep + "/" + c
			if checkFull(full) {
				return full, nil
			}
		}
	}

	log.Debug("- FindPinentry: none found")
	return "", fmt.Errorf("No pinentry found, checked a bunch of different places")
}
<commit_msg>Fix for #833<commit_after>// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package pinentry

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/keybase/client/go/logger"
)

//
// some borrowed from here:
//
//  https://github.com/bradfitz/camlistore/blob/master/pkg/misc/pinentry/pinentry.go
//
// Under the Apache 2.0 license
//

func canExec(s string) error {
	fi, err := os.Stat(s)
	if err != nil {
		return err
	}
	mode := fi.Mode()

	//
	// Only consider non-directories that have at least one +x
	//  bit set.
	//
	// TODO: Recheck this on windows!
	//   See here for lookpath: http://golang.org/src/pkg/os/exec/lp_windows.go
	//
	// Similar to check from exec.LookPath below
	//   See here: http://golang.org/src/pkg/os/exec/lp_unix.go
	//
	if mode.IsDir() {
		return fmt.Errorf("Program '%s' is a directory", s)
	} else if int(mode)&0111 == 0 {
		return fmt.Errorf("Program '%s' isn't executable", s)
	} else {
		return nil
	}
}

func FindPinentry(log logger.Logger) (string, error) {
	if !HasWindows() {
		return "", fmt.Errorf("Can't spawn gui window, not using pinentry")
	}
	bins := []string{
		// If you install MacTools you'll wind up with this pinentry
		"/usr/local/MacGPG2/libexec/pinentry-mac.app/Contents/MacOS/pinentry-mac",
	}

	extraPaths := []string{}

	log.Debug("+ FindPinentry()")

	cmds := []string{
		"pinentry-gtk-2",
		"pinentry-qt4",
		"pinentry",
	}

	checkFull := func(s string) bool {
		log.Debug("| Check fullpath %s", s)
		found := (canExec(s) == nil)
		if found {
			log.Debug("- Found: %s", s)
		}
		return found
	}

	for _, b := range bins {
		if checkFull(b) {
			return b, nil
		}
	}

	path := os.Getenv("PATH")
	for _, c := range cmds {
		log.Debug("| Looking for %s in standard PATH %s", c, path)
		fullc, err := exec.LookPath(c)
		if err == nil {
			log.Debug("- Found %s", fullc)
			return fullc, nil
		}
	}

	for _, ep := range extraPaths {
		for _, c := range cmds {
			full := ep + "/" + c
			if checkFull(full) {
				return full, nil
			}
		}
	}

	log.Debug("- FindPinentry: none found")
	return "", fmt.Errorf("No pinentry found, checked a bunch of different places")
}
<|endoftext|>


<commit_before>package zeusclient

import (
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	"github.com/burke/ttyutils"
	"github.com/burke/zeus/go/config"
	"github.com/burke/zeus/go/messages"
	slog "github.com/burke/zeus/go/shinylog"
	"github.com/burke/zeus/go/unixsocket"
	"github.com/burke/zeus/go/zerror"
	"github.com/kr/pty"
)

const (
	sigInt  = 3
	sigQuit = 28
	sigTstp = 26
)

func Run() {
	os.Exit(doRun())
}

// man signal | grep 'terminate process' | awk '{print $2}' | xargs -I '{}' echo -n "syscall.{}, "
var terminatingSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGKILL, syscall.SIGPIPE, syscall.SIGALRM, syscall.SIGTERM, syscall.SIGXCPU, syscall.SIGXFSZ, syscall.SIGVTALRM, syscall.SIGPROF, syscall.SIGUSR1, syscall.SIGUSR2}

func doRun() int {
	if os.Getenv("RAILS_ENV") != "" {
		println("Warning: Specifying a Rails environment via RAILS_ENV has no effect for commands run with zeus.")
		println("As a safety precaution to protect you from nuking your development database,")
		println("Zeus will now cowardly refuse to proceed. Please unset RAILS_ENV and try again.")
		return 1
	}

	isTerminal := ttyutils.IsTerminal(os.Stdout.Fd())

	var master, slave *os.File
	var err error
	if isTerminal {
		master, slave, err = pty.Open()
	} else {
		master, slave, err = unixsocket.Socketpair(syscall.SOCK_STREAM)
	}
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	defer master.Close()
	var oldState *ttyutils.Termios
	if isTerminal {
		oldState, err = ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	// should this happen if we're running over a pipe? I think maybe not?
	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", unixsocket.ZeusSockName())
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		zerror.ErrorCantConnectToMaster()
		return 1
	}
	usock := unixsocket.New(conn)

	msg := messages.CreateCommandAndArgumentsMessage(config.Args, os.Getpid())
	usock.WriteMessage(msg)
	err = sendCommandLineArguments(usock, config.Args)
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	usock.WriteFD(int(slave.Fd()))
	slave.Close()

	msg, err = usock.ReadMessage()
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	parts := strings.Split(msg, "\000")
	commandPid, err := strconv.Atoi(parts[0])
	defer func() {
		if commandPid > 0 {
			// Just in case.
			syscall.Kill(commandPid, 9)
		}
	}()

	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	if isTerminal {
		c := make(chan os.Signal, 1)
		handledSignals := append(append(terminatingSignals, syscall.SIGWINCH), syscall.SIGCONT)
		signal.Notify(c, handledSignals...)
		go func() {
			for sig := range c {
				if sig == syscall.SIGCONT {
					syscall.Kill(commandPid, syscall.SIGCONT)
				} else if sig == syscall.SIGWINCH {
					ttyutils.MirrorWinsize(os.Stdout, master)
					syscall.Kill(commandPid, syscall.SIGWINCH)
				} else { // member of terminatingSignals
					ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
					print("\r")
					syscall.Kill(commandPid, sig.(syscall.Signal))
					os.Exit(1)
				}
			}
		}()
	}

	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte, 1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		buf := make([]byte, 8192)
		for {
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			if isTerminal {
				for i := 0; i < n; i++ {
					switch buf[i] {
					case sigInt:
						syscall.Kill(commandPid, syscall.SIGINT)
					case sigQuit:
						syscall.Kill(commandPid, syscall.SIGQUIT)
					case sigTstp:
						syscall.Kill(commandPid, syscall.SIGTSTP)
						syscall.Kill(os.Getpid(), syscall.SIGTSTP)
					}
				}
			}
			master.Write(buf[:n])
		}
	}()

	<-eof

	if exitStatus == -1 {
		msg, err = usock.ReadMessage()
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
		parts := strings.Split(msg, "\000")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
	}

	return exitStatus
}

func sendCommandLineArguments(usock *unixsocket.Usock, args []string) error {
	master, slave, err := unixsocket.Socketpair(syscall.SOCK_STREAM)
	if err != nil {
		return err
	}
	usock.WriteFD(int(slave.Fd()))
	if err != nil {
		return err
	}
	slave.Close()

	go func() {
		defer master.Close()
		argAsBytes := []byte{}
		for _, arg := range args[1:] {
			argAsBytes = append(argAsBytes, []byte(arg)...)
			argAsBytes = append(argAsBytes, byte(0))
		}
		_, err = master.Write(argAsBytes)
		if err != nil {
			slog.ErrorString("Could not send arguments across: " +
				err.Error() + "\r")
			return
		}
	}()

	return nil
}
<commit_msg>Remove warning message used by RubyMine<commit_after>package zeusclient

import (
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"

	"github.com/burke/ttyutils"
	"github.com/burke/zeus/go/config"
	"github.com/burke/zeus/go/messages"
	slog "github.com/burke/zeus/go/shinylog"
	"github.com/burke/zeus/go/unixsocket"
	"github.com/burke/zeus/go/zerror"
	"github.com/kr/pty"
)

const (
	sigInt  = 3
	sigQuit = 28
	sigTstp = 26
)

func Run() {
	os.Exit(doRun())
}

// man signal | grep 'terminate process' | awk '{print $2}' | xargs -I '{}' echo -n "syscall.{}, "
var terminatingSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGKILL, syscall.SIGPIPE, syscall.SIGALRM, syscall.SIGTERM, syscall.SIGXCPU, syscall.SIGXFSZ, syscall.SIGVTALRM, syscall.SIGPROF, syscall.SIGUSR1, syscall.SIGUSR2}

func doRun() int {
	isTerminal := ttyutils.IsTerminal(os.Stdout.Fd())

	var master, slave *os.File
	var err error
	if isTerminal {
		master, slave, err = pty.Open()
	} else {
		master, slave, err = unixsocket.Socketpair(syscall.SOCK_STREAM)
	}
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	defer master.Close()
	var oldState *ttyutils.Termios
	if isTerminal {
		oldState, err = ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	// should this happen if we're running over a pipe? I think maybe not?
	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", unixsocket.ZeusSockName())
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		zerror.ErrorCantConnectToMaster()
		return 1
	}
	usock := unixsocket.New(conn)

	msg := messages.CreateCommandAndArgumentsMessage(config.Args, os.Getpid())
	usock.WriteMessage(msg)
	err = sendCommandLineArguments(usock, config.Args)
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	usock.WriteFD(int(slave.Fd()))
	slave.Close()

	msg, err = usock.ReadMessage()
	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	parts := strings.Split(msg, "\000")
	commandPid, err := strconv.Atoi(parts[0])
	defer func() {
		if commandPid > 0 {
			// Just in case.
			syscall.Kill(commandPid, 9)
		}
	}()

	if err != nil {
		slog.ErrorString(err.Error() + "\r")
		return 1
	}

	if isTerminal {
		c := make(chan os.Signal, 1)
		handledSignals := append(append(terminatingSignals, syscall.SIGWINCH), syscall.SIGCONT)
		signal.Notify(c, handledSignals...)
		go func() {
			for sig := range c {
				if sig == syscall.SIGCONT {
					syscall.Kill(commandPid, syscall.SIGCONT)
				} else if sig == syscall.SIGWINCH {
					ttyutils.MirrorWinsize(os.Stdout, master)
					syscall.Kill(commandPid, syscall.SIGWINCH)
				} else { // member of terminatingSignals
					ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
					print("\r")
					syscall.Kill(commandPid, sig.(syscall.Signal))
					os.Exit(1)
				}
			}
		}()
	}

	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte, 1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		buf := make([]byte, 8192)
		for {
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			if isTerminal {
				for i := 0; i < n; i++ {
					switch buf[i] {
					case sigInt:
						syscall.Kill(commandPid, syscall.SIGINT)
					case sigQuit:
						syscall.Kill(commandPid, syscall.SIGQUIT)
					case sigTstp:
						syscall.Kill(commandPid, syscall.SIGTSTP)
						syscall.Kill(os.Getpid(), syscall.SIGTSTP)
					}
				}
			}
			master.Write(buf[:n])
		}
	}()

	<-eof

	if exitStatus == -1 {
		msg, err = usock.ReadMessage()
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
		parts := strings.Split(msg, "\000")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			slog.ErrorString(err.Error() + "\r")
			return 1
		}
	}

	return exitStatus
}

func sendCommandLineArguments(usock *unixsocket.Usock, args []string) error {
	master, slave, err := unixsocket.Socketpair(syscall.SOCK_STREAM)
	if err != nil {
		return err
	}
	usock.WriteFD(int(slave.Fd()))
	if err != nil {
		return err
	}
	slave.Close()

	go func() {
		defer master.Close()
		argAsBytes := []byte{}
		for _, arg := range args[1:] {
			argAsBytes = append(argAsBytes, []byte(arg)...)
			argAsBytes = append(argAsBytes, byte(0))
		}
		_, err = master.Write(argAsBytes)
		if err != nil {
			slog.ErrorString("Could not send arguments across: " +
				err.Error() + "\r")
			return
		}
	}()

	return nil
}
<|endoftext|>


<commit_before>package cgroup

/*
#include <libcgroup.h>
#cgo LDFLAGS: -lcgroup

// work around for the 'type' special word
enum cgroup_file_type type_from_file_info(struct cgroup_file_info fi) {
  return fi.type;
}
*/
import "C"
import (
	"errors"
	"runtime"
	"unsafe"
)

/*
Structure describing one or more control groups. The structure is opaque to
applications.
*/
type Cgroup struct {
	g *C.struct_group
}

func NewCgroup(name string) Cgroup {
	cg := Cgroup{
		C.cgroup_new_cgroup(C.CString(name)),
	}
	runtime.SetFinalizer(cg, freeCgroupThings)
	return cg
}

func (cg *Cgroup) AddController(name string) Controller {
	return Controller{
		C.cgroup_add_controller(cg.g, C.CString(name)),
	}
}
func (cg Cgroup) GetController(name string) Controller {
	return Controller{
		C.cgroup_get_controller(cg.g, C.CString(name)),
	}
}

func freeCgroupThings(cg Cgroup) {
  freeCgroup(cg)
  freeControllers(cg)
}

func freeCgroup(cg Cgroup) {
	C.cgroup_free(&cg.g)
}

func freeControllers(cg Cgroup) {
	C.cgroup_free_controllers(cg.g)
}

/*
Physically create a control group in kernel. The group is created in all
hierarchies, which cover controllers added by Cgroup.AddController().

TODO correct docs for golang implementation

All parameters set by cgroup_add_value_* functions are written.
The created groups has owner which was set by cgroup_set_uid_gid() and
permissions set by cgroup_set_permissions.
*/
func CreateGroup(cg Cgroup, ignore_ownership bool) error {
  var i int = 0
  if ignore_ownership == true {
    i = 1
  }
  return _err(C.cgroup_create_cgroup(cg.g, C.int(i)))
}

/*
Structure describing a controller attached to one struct @c cgroup, including
parameters of the group and their values. The structure is opaque to
applications.
*/
type Controller struct {
	c *C.struct_cgroup_controller
}

/*
Initialize libcgroup. Information about mounted hierarchies are examined
and cached internally (just what's mounted where, not the groups themselves).
*/
func Init() error {
	return _err(C.cgroup_init())
}

/*
Load configuration file and mount and create control groups described there.
See cgconfig.conf man page for format of the file.
*/
func LoadConfig(filename string) error {
	return _err(C.cgroup_config_load_config(C.CString(filename)))
}

/*
Delete all control groups and unmount all hierarchies.
*/
func Unload() error {
	return _err(C.cgroup_unload_cgroups())
}

type DeleteFlag int

const (
	// Ignore errors caused by migration of tasks to parent group.
	DeleteIgnoreMigration = DeleteFlag(C.CGFLAG_DELETE_IGNORE_MIGRATION)

	// Recursively delete all child groups.
	DeleteRecursive = DeleteFlag(C.CGFLAG_DELETE_RECURSIVE)

	/*
		Delete the cgroup only if it is empty, i.e. it has no subgroups and
		no processes inside. This flag cannot be used with
		DeleteRecursive
	*/
	DeleteEmptyOnly = DeleteFlag(C.CGFLAG_DELETE_EMPTY_ONLY)
)

/*
Delete all cgroups and unmount all mount points defined in specified config
file.

The groups are either removed recursively or only the empty ones, based
on given flags. Mount point are always umounted only if they are empty,
regardless of any flags.

The groups are sorted before they are removed, so the removal of empty ones
actually works (i.e. subgroups are removed first).
*/
func UnloadFromConfig(filename string, flags DeleteFlag) error {
	return _err(C.cgroup_config_unload_config(C.CString(filename), C.int(flags)))
}

/*
Sets default permissions of groups created by subsequent
cgroup_config_load_config() calls. If a config file contains a 'default {}'
section, the default permissions from the config file is then used.

Use cgroup_new_cgroup() to create a dummy group and cgroup_set_uid_gid() and
cgroup_set_permissions() to set its permissions. Use NO_UID_GID instead of
GID/UID and NO_PERMS instead of file/directory permissions to let kernel
decide the default permissions where you don't want specific user and/or
permissions. Kernel then uses current user/group and permissions from umask
then.

New default permissions from this group are copied to libcgroup internal
structures.
*/
func SetDefault(cg Cgroup) error {
  return _err(C.cgroup_config_set_default(cg.g))
}

type FileInfo struct {
	Type     FileType
	Path     string
	Parent   string
	FullPath string
	Depth    int8
}

func fromCFileInfo(cData C.struct_cgroup_file_info) FileInfo {
	return FileInfo{
		Type:     FileType(C.type_from_file_info(cData)),
		Path:     C.GoString(cData.path),
		Parent:   C.GoString(cData.parent),
		FullPath: C.GoString(cData.full_path),
		Depth:    int8(cData.depth),
	}
}

type FileType int

const (
	FileTypeFile  = FileType(C.CGROUP_FILE_TYPE_FILE)
	FileTypeDir   = FileType(C.CGROUP_FILE_TYPE_DIR)
	FileTypeOther = FileType(C.CGROUP_FILE_TYPE_OTHER)
)

/*
int cgroup_walk_tree_begin(const char *controller, const char *base_path, int depth,
				void **handle, struct cgroup_file_info *info,
				int *base_level);
*/

/*
Information model for Controllers available
*/
type ControllerData struct {
	Name       string
	Hierarchy  int
	NumCgroups int
	Enabled    int
}

func fromCControllerData(cData C.struct_controller_data) ControllerData {
	return ControllerData{
		Name:       C.GoString(&cData.name[0]),
		Hierarchy:  int(cData.hierarchy),
		NumCgroups: int(cData.num_cgroups),
		Enabled:    int(cData.enabled),
	}
}

func GetAllControllers() (controllers []ControllerData, err error) {
	var (
		cd     C.struct_controller_data
		handle unsafe.Pointer
	)
	err = _err(C.cgroup_get_all_controller_begin(&handle, &cd))
	if err != nil {
		return controllers, err
	}
	defer C.cgroup_get_all_controller_end(&handle)

	controllers = append(controllers, fromCControllerData(cd))
	for {
		err = _err(C.cgroup_get_all_controller_next(&handle, &cd))
		if err != nil && err != ECGEOF {
			return controllers, err
		}
		controllers = append(controllers, fromCControllerData(cd))
		if err == ECGEOF {
			break
		}
	}
	return controllers, nil
}

func GetSubSysMountPoint(controller string) (string, error) {
	var mp *C.char
	err := _err(C.cgroup_get_subsys_mount_point(C.CString(controller), &mp))
	if err != nil {
		return "", err
	}
	return C.GoString(mp), nil
}

var (
	// End-of-file for iterators
	ECGEOF   = errors.New(C.GoString(C.cgroup_strerror(C.ECGEOF)))
	ECGOTHER = errors.New(C.GoString(C.cgroup_strerror(C.ECGOTHER)))
)

/*
Return last errno, which caused ECGOTHER error.
*/
func LastError() error {
	return _err(C.cgroup_get_last_errno())
}

func _err(num C.int) error {
	switch num {
	case 0:
		return nil
	case C.ECGEOF:
		return ECGEOF
	case C.ECGOTHER:
		return ECGOTHER
	}
	// There's a lot. We'll create them as they come
	return errors.New(C.GoString(C.cgroup_strerror(num)))
}
<commit_msg>more progress<commit_after>package cgroup

/*
#include <libcgroup.h>
#cgo LDFLAGS: -lcgroup

// work around for the 'type' special word
enum cgroup_file_type type_from_file_info(struct cgroup_file_info fi) {
  return fi.type;
}
*/
import "C"
import (
	"errors"
	"runtime"
	"unsafe"
)

/*
Structure describing one or more control groups. The structure is opaque to
applications.
*/
type Cgroup struct {
	g *C.struct_group
}

func NewCgroup(name string) Cgroup {
	cg := Cgroup{
		C.cgroup_new_cgroup(C.CString(name)),
	}
	runtime.SetFinalizer(&cg, freeCgroupThings)
	return cg
}

func (cg *Cgroup) AddController(name string) Controller {
	return Controller{
		C.cgroup_add_controller(cg.g, C.CString(name)),
	}
}
func (cg Cgroup) GetController(name string) Controller {
	return Controller{
		C.cgroup_get_controller(cg.g, C.CString(name)),
	}
}

func freeCgroupThings(cg *Cgroup) {

	freeCgroup(*cg)
	freeControllers(*cg)
}

func freeCgroup(cg Cgroup) {
	C.cgroup_free(&cg.g)
}

func freeControllers(cg Cgroup) {
	C.cgroup_free_controllers(cg.g)
}

/*
Physically create a control group in kernel. The group is created in all
hierarchies, which cover controllers added by Cgroup.AddController().

TODO correct docs for golang implementation

All parameters set by cgroup_add_value_* functions are written.
The created groups has owner which was set by cgroup_set_uid_gid() and
permissions set by cgroup_set_permissions.

  foo = cgroup.NewCgroup("foo)
  foo.Create()

*/
func (cg Cgroup) Create() error {
	return _err(C.cgroup_create_cgroup(cg.g, C.int(0)))
}

/*
Same as Create(), but all errors are ignored when setting
owner of the group and/or its tasks file.
*/
func (cg Cgroup) CreateIgnoreOwnership() error {
	return _err(C.cgroup_create_cgroup(cg.g, C.int(1)))
}

/*
Physically create new control group in kernel, with all parameters and values
copied from its parent group. The group is created in all hierarchies, where
the parent group exists. I.e. following code creates subgroup in all
hierarchies, because all of them have root (=parent) group.

  foo = cgroup.NewCgroup("foo)
  foo.CreateFromParent()

*/
func (cg Cgroup) CreateFromParent() error {
	return _err(C.cgroup_create_cgroup_from_parent(cg.g, C.int(0)))
}

/*
Same as CreateFromParent(), but all errors are ignored when setting
owner of the group and/or its tasks file.
*/
func (cg Cgroup) CreateFromParentIgnoreOwnership() error {
	return _err(C.cgroup_create_cgroup_from_parent(cg.g, C.int(1)))
}

/*
Physically modify a control group in kernel. All parameters added by
cgroup_add_value_ or cgroup_set_value_ are written.
Currently it's not possible to change and owner of a group.

TODO correct docs for golang implementation
*/
func (cg Cgroup) Modify() error {
	return _err(C.cgroup_modify_cgroup(cg.g))
}

/*
Physically remove a control group from kernel. The group is removed from
all hierarchies,  which cover controllers added by Cgroup.AddController()
or GetCgroup(). All tasks inside the group are automatically moved
to parent group.

The group being removed must be empty, i.e. without subgroups. Use
cgroup_delete_cgroup_ext() for recursive delete.

TODO correct docs for golang implementation
*/
func (cg Cgroup) Delete() error {
	return _err(C.cgroup_delete_cgroup(cg.g, C.int(0)))
}

/*
Same as Delete(), but ignores errors when migrating.
*/
func (cg Cgroup) DeleteIgnoreMigration() error {
	return _err(C.cgroup_delete_cgroup(cg.g, C.int(1)))
}

/*
Physically remove a control group from kernel.
All tasks are automatically moved to parent group.
If DeleteIgnoreMigration flag is used, the errors that occurred
during the task movement are ignored.
DeleteRecursive flag specifies that all subgroups should be removed
too. If root group is being removed with this flag specified, all subgroups
are removed but the root group itself is left undeleted.
*/
func (cg Cgroup) DeleteExt(flags DeleteFlag) error {
	return _err(C.cgroup_delete_cgroup_ext(cg.g, C.int(flags)))
}

/*
Read all information regarding the group from kernel.
Based on name of the group, list of controllers and all parameters and their
values are read from all hierarchies, where a group with given name exists.
All existing controllers are replaced. I.e. following code will fill root with
controllers from all hierarchies, because the root group is available in all of
them.

  root := cgroup.NewCgroup("/")
  err := root.Get()
  ...

*/
func (cg Cgroup) Get() error {
	return _err(C.cgroup_get_cgroup(cg.g))
}

type UID C.uid_t
type GID C.gid_t

/*
Set owner of the group control files and the @c tasks file. This function
modifies only libcgroup internal cgroup structure, use
Cgroup.Create() afterwards to create the group with given owners.

@param cgroup
@param tasks_uid UID of the owner of group's @c tasks file.
@param tasks_gid GID of the owner of group's @c tasks file.
@param control_uid UID of the owner of group's control files (i.e.
parameters).
@param control_gid GID of the owner of group's control files (i.e.
parameters).
*/
func (cg Cgroup) SetUidGid(tasks_uid UID, tasks_gid GID,
	control_uid UID, control_gid GID) error {
	return _err(C.cgroup_set_uid_gid(cg.g,
		C.uid_t(tasks_uid), C.gid_t(tasks_gid),
		C.uid_t(control_uid), C.gid_t(control_gid)))

}

/*
Return owners of the group's @c tasks file and control files.
The data is read from libcgroup internal cgroup structure, use
Cgroup.SetUidGid() or Cgroup.Get() to fill it.
*/
func (cg Cgroup) GetUidGid() (tasks_uid UID, tasks_gid GID, control_uid UID, control_gid GID, err error) {
	var (
		c_t_u C.uid_t
		c_t_g C.gid_t
		c_c_u C.uid_t
		c_c_g C.gid_t
	)
	err = _err(C.cgroup_set_uid_gid(cg.g,
		c_t_u,
		c_t_g,
		c_c_u,
		c_c_g))
	return UID(c_t_u), GID(c_t_g), UID(c_c_u), GID(c_c_g), err

}

const (
	// Uninitialized file/directory permissions used for task/control files.
	NO_PERMS = C.NO_PERMS

	// Uninitialized UID/GID used for task/control files.
	NO_UID_GID = C.NO_UID_GID
)

type Mode C.mode_t

/*
Stores given file permissions of the group's control and tasks files
into the cgroup data structure. Use NO_PERMS if permissions shouldn't
be changed or a value which applicable to chmod(2). Please note that
the given permissions are masked with the file owner's permissions.
For example if a control file has permissions 640 and control_fperm is
471 the result will be 460.

control_dperm Directory permission for the group.
control_fperm File permission for the control files.
task_fperm File permissions for task file.

  g := cgroup.NewCgroup("foo")
  g.SetPermissions(cgroup.Mode(0777), cgroup.Mode(0777), cgroup.Mode(0777))
*/
func (cg Cgroup) SetPermissions(control_dperm, control_fperm, task_fperm Mode) {
	C.cgroup_set_permissions(cg.g, C.mode_t(control_dperm),
		C.mode_t(control_fperm), C.mode_t(task_fperm))
}

/*
Copy all controllers, parameters and their values. All existing controllers
in the source group are discarded.
*/
func CopyCgroup(src, dest Cgroup) error {
	return _err(C.cgroup_copy_cgroup(src.g, dest.g))
}

/*
Compare names, owners, controllers, parameters and values of two groups.

Return value of:
 * nil - a and b are equal
 * ECGROUPNOTEQUAL - groups are not equal
 * ECGCONTROLLERNOTEQUAL - controllers are not equal

*/
func CompareCgroup(a, b Cgroup) error {
	return _err(C.cgroup_compare_cgroup(a.g, b.g))
}

/*
Structure describing a controller attached to one struct @c cgroup, including
parameters of the group and their values. The structure is opaque to
applications.
*/
type Controller struct {
	c *C.struct_cgroup_controller
}

/*
Add parameter and its value to internal libcgroup structures.
Use Cgroup.Modify() or Cgroup.Create() to write it to kernel.

Name of the parameter and its value
*/
func (c Controller) AddValueString(name, value string) error {
	return _err(C.cgroup_add_value_string(c.c, C.CString(name), C.CString(value)))
}

/*
Compare names, parameters and values of two controllers.

Return value of:
 * nil - a and b are equal
 * ECGCONTROLLERNOTEQUAL - controllers are not equal
*/
func CompareControllers(a, b Controller) error {
	return _err(C.cgroup_compare_controllers(a.c, b.c))
}

/*
Initialize libcgroup. Information about mounted hierarchies are examined
and cached internally (just what's mounted where, not the groups themselves).
*/
func Init() error {
	return _err(C.cgroup_init())
}

/*
Load configuration file and mount and create control groups described there.
See cgconfig.conf man page for format of the file.
*/
func LoadConfig(filename string) error {
	return _err(C.cgroup_config_load_config(C.CString(filename)))
}

/*
Delete all control groups and unmount all hierarchies.
*/
func Unload() error {
	return _err(C.cgroup_unload_cgroups())
}

type DeleteFlag int

const (
	// Ignore errors caused by migration of tasks to parent group.
	DeleteIgnoreMigration = DeleteFlag(C.CGFLAG_DELETE_IGNORE_MIGRATION)

	// Recursively delete all child groups.
	DeleteRecursive = DeleteFlag(C.CGFLAG_DELETE_RECURSIVE)

	/*
		Delete the cgroup only if it is empty, i.e. it has no subgroups and
		no processes inside. This flag cannot be used with
		DeleteRecursive
	*/
	DeleteEmptyOnly = DeleteFlag(C.CGFLAG_DELETE_EMPTY_ONLY)
)

/*
Delete all cgroups and unmount all mount points defined in specified config
file.

The groups are either removed recursively or only the empty ones, based
on given flags. Mount point are always umounted only if they are empty,
regardless of any flags.

The groups are sorted before they are removed, so the removal of empty ones
actually works (i.e. subgroups are removed first).
*/
func UnloadFromConfig(filename string, flags DeleteFlag) error {
	return _err(C.cgroup_config_unload_config(C.CString(filename), C.int(flags)))
}

/*
Sets default permissions of groups created by subsequent
cgroup_config_load_config() calls. If a config file contains a 'default {}'
section, the default permissions from the config file is then used.

Use cgroup_new_cgroup() to create a dummy group and cgroup_set_uid_gid() and
cgroup_set_permissions() to set its permissions. Use NO_UID_GID instead of
GID/UID and NO_PERMS instead of file/directory permissions to let kernel
decide the default permissions where you don't want specific user and/or
permissions. Kernel then uses current user/group and permissions from umask
then.

New default permissions from this group are copied to libcgroup internal
structures.
*/
func SetDefault(cg Cgroup) error {
	return _err(C.cgroup_config_set_default(cg.g))
}

type FileInfo struct {
	Type     FileType
	Path     string
	Parent   string
	FullPath string
	Depth    int8
}

func fromCFileInfo(cData C.struct_cgroup_file_info) FileInfo {
	return FileInfo{
		Type:     FileType(C.type_from_file_info(cData)),
		Path:     C.GoString(cData.path),
		Parent:   C.GoString(cData.parent),
		FullPath: C.GoString(cData.full_path),
		Depth:    int8(cData.depth),
	}
}

type FileType int

const (
	FileTypeFile  = FileType(C.CGROUP_FILE_TYPE_FILE)
	FileTypeDir   = FileType(C.CGROUP_FILE_TYPE_DIR)
	FileTypeOther = FileType(C.CGROUP_FILE_TYPE_OTHER)
)

/*
int cgroup_walk_tree_begin(const char *controller, const char *base_path, int depth,
				void **handle, struct cgroup_file_info *info,
				int *base_level);
*/

/*
Information model for Controllers available
*/
type ControllerData struct {
	Name       string
	Hierarchy  int
	NumCgroups int
	Enabled    int
}

func fromCControllerData(cData C.struct_controller_data) ControllerData {
	return ControllerData{
		Name:       C.GoString(&cData.name[0]),
		Hierarchy:  int(cData.hierarchy),
		NumCgroups: int(cData.num_cgroups),
		Enabled:    int(cData.enabled),
	}
}

func GetAllControllers() (controllers []ControllerData, err error) {
	var (
		cd     C.struct_controller_data
		handle unsafe.Pointer
	)
	err = _err(C.cgroup_get_all_controller_begin(&handle, &cd))
	if err != nil {
		return controllers, err
	}
	defer C.cgroup_get_all_controller_end(&handle)

	controllers = append(controllers, fromCControllerData(cd))
	for {
		err = _err(C.cgroup_get_all_controller_next(&handle, &cd))
		if err != nil && err != ECGEOF {
			return controllers, err
		}
		controllers = append(controllers, fromCControllerData(cd))
		if err == ECGEOF {
			break
		}
	}
	return controllers, nil
}

func GetSubSysMountPoint(controller string) (string, error) {
	var mp *C.char
	err := _err(C.cgroup_get_subsys_mount_point(C.CString(controller), &mp))
	if err != nil {
		return "", err
	}
	return C.GoString(mp), nil
}

var (
	// End-of-file for iterators
	ECGEOF                = errors.New(C.GoString(C.cgroup_strerror(C.ECGEOF)))
	ECGOTHER              = errors.New(C.GoString(C.cgroup_strerror(C.ECGOTHER)))
	ECGROUPNOTEQUAL       = errors.New(C.GoString(C.cgroup_strerror(C.ECGROUPNOTEQUAL)))
	ECGCONTROLLERNOTEQUAL = errors.New(C.GoString(C.cgroup_strerror(C.ECGCONTROLLERNOTEQUAL)))
)

/*
Return last errno, which caused ECGOTHER error.
*/
func LastError() error {
	return _err(C.cgroup_get_last_errno())
}

func _err(num C.int) error {
	switch num {
	case 0:
		return nil
	case C.ECGEOF:
		return ECGEOF
	case C.ECGOTHER:
		return ECGOTHER
	case C.ECGROUPNOTEQUAL:
		return ECGROUPNOTEQUAL
	case C.ECGCONTROLLERNOTEQUAL:
		return ECGCONTROLLERNOTEQUAL
	}
	// There's a lot. We'll create them as they come
	return errors.New(C.GoString(C.cgroup_strerror(num)))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
)

// Service is a combination of a ServiceHandlerFunc and options, used by the rootServiceHandler
type Service struct {
	fn                ServiceHandlerFunc
	omitClientSession bool
}

// ServiceHandlerFunc defines the layout of a service handler func.. d'oh.
type ServiceHandlerFunc func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (outData interface{}, err error)

var services = map[string]Service{
	// list of services that don not require a clientSession.
	// please keep this list sorted
	"/service/sessionInit":    Service{newSessionInitHandlerFunc(), true},
	"/service/sessionCheck":   Service{newSessionCheckHandlerFunc(), true},
	"/service/sessionDestroy": Service{newSessionDestroyHandlerFunc(), false},
}

// initHTTPServer sets up the http.FileServer and other http services.
func initHTTPServer() {
	// add handlers to http.DefaultServeMux
	// try to serve static files on root uri
	http.Handle("/", http.FileServer(http.Dir("./http-files/")))
	http.HandleFunc("/service/", rootServiceHandler)

	// run http server in goroutine
	go func() {
		//++ TODO: make configurable
		port := "8000"

		// inform user of startup
		log.Printf("starting http server on port %s\n", port)

		// listen and serve on given port
		// error is fatal
		err := http.ListenAndServe(":"+port, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	if flags.UnixSocket {
		go func() {
			// socketClosign is used to omit error on socket read when closing down.
			var socketClosing bool

			//++ TODO: make configurable
			socketFilename := "./nulpunt.socket"

			// inform user of startup
			log.Printf("Starting http server on unix socket %s\n", socketFilename)

			// create and listen on this unix socket
			socket, err := net.ListenUnix("unix", &net.UnixAddr{socketFilename, "unix"})
			if err != nil {
				log.Fatal(err)
			}

			// append a function on graceful shutdown to close the unix socket
			processEndFuncs = append(processEndFuncs, func() {
				socketClosing = true
				socket.Close()
			})

			// serve on the opened unix socket
			// an error (when not closing down) is fatal
			err = http.Serve(socket, nil)
			if !socketClosing && err != nil {
				log.Fatal(err)
			}
		}()
	}
}

func rootServiceHandler(w http.ResponseWriter, r *http.Request) {
	var err error

	// lookup service
	s, exists := services[r.RequestURI]
	if !exists {
		log.Printf("invalid request (404) for service on: %s\n", r.RequestURI)
		http.NotFound(w, r)
		return
	}

	//++ TODO: check origin
	fmt.Println("check origin")

	//++ TODO: check referer
	fmt.Println("check referer")

	// find ClientSession
	var cs *ClientSession
	if !s.omitClientSession {
		cs, err = getClientSession(r.Header.Get("X-Nulpunt-SessionKey"))
		if err != nil {
			http.Error(w, "forbidden without valid sessionKey", http.StatusForbidden)
			return
		}
		defer cs.done()
	}

	// call actual handler
	outData, err := s.fn(w, r, cs)
	if err != nil {
		log.Printf("error from service %s: %s\n", r.RequestURI, err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}

	// we're responding with json
	w.Header().Add("Content-Type", "application/json")

	// encode response data to json for client
	err = json.NewEncoder(w).Encode(outData)
	if err != nil {
		log.Printf("error encoding service outData to client for service %s: %s\n", r.RequestURI, err)
	}
}

func newSessionInitHandlerFunc() ServiceHandlerFunc {
	type outDataType struct {
		SessionKey string `json:"sessionKey"`
	}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		// create a new cs
		newCS := newClientSession()

		// return cs key
		out := &outDataType{
			SessionKey: newCS.key,
		}

		return out, nil
	}
}
func newSessionCheckHandlerFunc() ServiceHandlerFunc {
	type inDataType struct {
		SessionKey string `json:"sessionKey"`
	}

	type outDataType struct {
		Result bool `json:"result"`
	}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		// decode input data
		inData := &inDataType{}
		err := json.NewDecoder(r.Body).Decode(inData)
		if err != nil {
			return nil, err
		}

		// new outData
		outData := &outDataType{}

		// get ClientSession
		s, err := getClientSession(inData.SessionKey)
		if err != nil {
			log.Printf("Could not find CS for key %s\n", inData.SessionKey)
			return outData, nil
		}

		// already done
		s.done()

		// session is valid
		outData.Result = true

		// return data
		return outData, nil
	}
}

func newSessionDestroyHandlerFunc() ServiceHandlerFunc {
	type outDataType struct{}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		cs.ping <- false //++ TODO: race when this is called more then once
		return nil, nil
	}
}
<commit_msg>http.go comments<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
)

// Service is a combination of a ServiceHandlerFunc and options, used by the rootServiceHandler
type Service struct {
	fn                ServiceHandlerFunc
	omitClientSession bool
}

// ServiceHandlerFunc defines the layout of a service handler func.. d'oh.
type ServiceHandlerFunc func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (outData interface{}, err error)

// list of services that don not require a clientSession
var services = map[string]Service{
	// NOTE: please keep this list sorted
	"/service/sessionInit":    Service{newSessionInitHandlerFunc(), true},
	"/service/sessionCheck":   Service{newSessionCheckHandlerFunc(), true},
	"/service/sessionDestroy": Service{newSessionDestroyHandlerFunc(), false},
}

// initHTTPServer sets up the http.FileServer and other http services.
func initHTTPServer() {
	// add handlers to http.DefaultServeMux
	// try to serve static files on root uri
	http.Handle("/", http.FileServer(http.Dir("./http-files/")))
	http.HandleFunc("/service/", rootServiceHandler)

	// run http server in goroutine
	go func() {
		//++ TODO: make configurable
		port := "8000"

		// inform user of startup
		log.Printf("starting http server on port %s\n", port)

		// listen and serve on given port
		// error is fatal
		err := http.ListenAndServe(":"+port, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	if flags.UnixSocket {
		go func() {
			// socketClosign is used to omit error on socket read when closing down.
			var socketClosing bool

			//++ TODO: make configurable
			socketFilename := "./nulpunt.socket"

			// inform user of startup
			log.Printf("Starting http server on unix socket %s\n", socketFilename)

			// create and listen on this unix socket
			socket, err := net.ListenUnix("unix", &net.UnixAddr{socketFilename, "unix"})
			if err != nil {
				log.Fatal(err)
			}

			// append a function on graceful shutdown to close the unix socket
			processEndFuncs = append(processEndFuncs, func() {
				socketClosing = true
				socket.Close()
			})

			// serve on the opened unix socket
			// an error (when not closing down) is fatal
			err = http.Serve(socket, nil)
			if !socketClosing && err != nil {
				log.Fatal(err)
			}
		}()
	}
}

func rootServiceHandler(w http.ResponseWriter, r *http.Request) {
	var err error

	// lookup service
	s, exists := services[r.RequestURI]
	if !exists {
		log.Printf("invalid request (404) for service on: %s\n", r.RequestURI)
		http.NotFound(w, r)
		return
	}

	//++ TODO: check origin
	fmt.Println("check origin")

	//++ TODO: check referer
	fmt.Println("check referer")

	// find ClientSession
	var cs *ClientSession
	if !s.omitClientSession {
		cs, err = getClientSession(r.Header.Get("X-Nulpunt-SessionKey"))
		if err != nil {
			http.Error(w, "forbidden without valid sessionKey", http.StatusForbidden)
			return
		}
		defer cs.done()
	}

	// call actual handler
	outData, err := s.fn(w, r, cs)
	if err != nil {
		log.Printf("error from service %s: %s\n", r.RequestURI, err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}

	// we're responding with json
	w.Header().Add("Content-Type", "application/json")

	// encode response data to json for client
	err = json.NewEncoder(w).Encode(outData)
	if err != nil {
		log.Printf("error encoding service outData to client for service %s: %s\n", r.RequestURI, err)
	}
}

// create a handler for new session initialization
func newSessionInitHandlerFunc() ServiceHandlerFunc {
	type outDataType struct {
		SessionKey string `json:"sessionKey"`
	}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		// create a new cs
		newCS := newClientSession()

		// return cs key
		out := &outDataType{
			SessionKey: newCS.key,
		}

		return out, nil
	}
}

// create a handler for session checks
func newSessionCheckHandlerFunc() ServiceHandlerFunc {
	type inDataType struct {
		SessionKey string `json:"sessionKey"`
	}

	type outDataType struct {
		Result bool `json:"result"`
	}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		// decode input data
		inData := &inDataType{}
		err := json.NewDecoder(r.Body).Decode(inData)
		if err != nil {
			return nil, err
		}

		// new outData
		outData := &outDataType{}

		// get ClientSession
		s, err := getClientSession(inData.SessionKey)
		if err != nil {
			log.Printf("Could not find CS for key %s\n", inData.SessionKey)
			return outData, nil
		}

		// already done
		s.done()

		// session is valid
		outData.Result = true

		// return data
		return outData, nil
	}
}

// create a handler for session destroy
func newSessionDestroyHandlerFunc() ServiceHandlerFunc {
	type outDataType struct{}

	return func(w http.ResponseWriter, r *http.Request, cs *ClientSession) (interface{}, error) {
		cs.ping <- false //++ TODO: race when this is called more then once
		return nil, nil
	}
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2016 Ivan Dejanovic

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package analyze

import (
	"errors"
	"fmt"
	"mlpl/types"
)

type procNode func(buf *buffer, node *types.TreeNode)

type buffer struct {
	location  int
	bucketMap map[string]types.Bucket
}

func (buf *buffer) st_insert(name string, lineno int) {
	bucket, ok := buf.bucketMap[name]

	if ok {
		line := bucket.Lines
		for line != nil {
			line = line.Next
		}
		line = &types.LineList{lineno, nil}
	} else {
		line := types.LineList{lineno, nil}
		bucket = types.Bucket{name, &line, buf.location}
		buf.location = buf.location + 1
		buf.bucketMap[name] = bucket
	}
}

func (buf *buffer) st_lookup(name string) int {
	bucketList, ok := buf.bucketMap[name]

	if ok {
		return bucketList.MemLoc
	}

	return -1
}

func typeError(lineno int, message string) {
	errorMessage := fmt.Sprintf("Type error at line %d: %s\n", lineno, message)
	err := errors.New(errorMessage)
	panic(err)
}

func insertNode(buf *buffer, node *types.TreeNode) {
	switch node.Node {
	case types.StmtK:
		if node.Stmt == types.AssignK || node.Stmt == types.ReadK {
			if buf.st_lookup(node.Name) == -1 {
				buf.st_insert(node.Name, node.Lineno)
			} else {
				buf.st_insert(node.Name, 0)
			}
		}
	case types.ExpK:
		if node.Exp == types.IdK {
			if buf.st_lookup(node.Name) == -1 {
				buf.st_insert(node.Name, node.Lineno)
			} else {
				buf.st_insert(node.Name, 0)
			}
		}
	}
}

func nullProc(buf *buffer, node *types.TreeNode) {
	return
}

func checkNode(buf *buffer, node *types.TreeNode) {
	switch node.Node {
	case types.ExpK:
		if node.Exp == types.OpK {
			if node.Children[0].Type != types.Integer || node.Children[1].Type != types.Integer {
				typeError(node.Lineno, "Op applied to non-integer")
			}
			if node.Op == types.EQ || node.Op == types.LT {
				node.Type = types.Boolean
			} else {
				node.Type = types.Integer
			}
		} else if node.Exp == types.ConstK || node.Exp == types.IdK {
			node.Type = types.Integer
		}
	case types.StmtK:
		switch node.Stmt {
		case types.IfK:
			if node.Children[0].Type == types.Integer {
				typeError(node.Lineno, "if test is not Boolean")
			}
		case types.AssignK:
			if node.Children[0].Type != types.Integer {
				typeError(node.Lineno, "assignment of non-integer value")
			}
		case types.WriteK:
			if node.Children[0].Type != types.Integer {
				typeError(node.Lineno, "write of non-integer value")
			}
		case types.RepeatK:
			if node.Children[0].Type == types.Integer {
				typeError(node.Lineno, "repeat test is not Boolean")
			}
		}
	}
}

func transverse(buf *buffer, node *types.TreeNode, preProc procNode, postProc procNode) {
	preProc(buf, node)
	for index := 0; index < len(node.Children); index++ {
		transverse(buf, node.Children[index], preProc, postProc)
	}
	postProc(buf, node)
	if node.Sibling != nil {
		transverse(buf, node.Sibling, preProc, postProc)
	}

}

func BuildSymtab(node *types.TreeNode) map[string]types.Bucket {
	buf := buffer{0, make(map[string]types.Bucket)}
	buf.location = 0
	transverse(&buf, node, insertNode, nullProc)
	return buf.bucketMap
}

func TypeCheck(node *types.TreeNode) {
	transverse(nil, node, nullProc, checkNode)
}
<commit_msg>Fixed typos.<commit_after>/*
The MIT License (MIT)

Copyright (c) 2016 Ivan Dejanovic

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package analyze

import (
	"errors"
	"fmt"
	"mlpl/types"
)

type procNode func(buf *buffer, node *types.TreeNode)

type buffer struct {
	location  int
	bucketMap map[string]types.Bucket
}

func (buf *buffer) st_insert(name string, lineno int) {
	bucket, ok := buf.bucketMap[name]

	if ok {
		line := bucket.Lines
		for line != nil {
			line = line.Next
		}
		line = &types.LineList{lineno, nil}
	} else {
		line := types.LineList{lineno, nil}
		bucket = types.Bucket{name, &line, buf.location}
		buf.location = buf.location + 1
		buf.bucketMap[name] = bucket
	}
}

func (buf *buffer) st_lookup(name string) int {
	bucket, ok := buf.bucketMap[name]

	if ok {
		return bucket.MemLoc
	}

	return -1
}

func typeError(lineno int, message string) {
	errorMessage := fmt.Sprintf("Type error at line %d: %s\n", lineno, message)
	err := errors.New(errorMessage)
	panic(err)
}

func insertNode(buf *buffer, node *types.TreeNode) {
	switch node.Node {
	case types.StmtK:
		if node.Stmt == types.AssignK || node.Stmt == types.ReadK {
			if buf.st_lookup(node.Name) == -1 {
				buf.st_insert(node.Name, node.Lineno)
			} else {
				buf.st_insert(node.Name, 0)
			}
		}
	case types.ExpK:
		if node.Exp == types.IdK {
			if buf.st_lookup(node.Name) == -1 {
				buf.st_insert(node.Name, node.Lineno)
			} else {
				buf.st_insert(node.Name, 0)
			}
		}
	}
}

func nullProc(buf *buffer, node *types.TreeNode) {
	return
}

func checkNode(buf *buffer, node *types.TreeNode) {
	switch node.Node {
	case types.ExpK:
		if node.Exp == types.OpK {
			if node.Children[0].Type != types.Integer || node.Children[1].Type != types.Integer {
				typeError(node.Lineno, "Op applied to non-integer")
			}
			if node.Op == types.EQ || node.Op == types.LT {
				node.Type = types.Boolean
			} else {
				node.Type = types.Integer
			}
		} else if node.Exp == types.ConstK || node.Exp == types.IdK {
			node.Type = types.Integer
		}
	case types.StmtK:
		switch node.Stmt {
		case types.IfK:
			if node.Children[0].Type == types.Integer {
				typeError(node.Lineno, "if test is not Boolean")
			}
		case types.AssignK:
			if node.Children[0].Type != types.Integer {
				typeError(node.Lineno, "assignment of non-integer value")
			}
		case types.WriteK:
			if node.Children[0].Type != types.Integer {
				typeError(node.Lineno, "write of non-integer value")
			}
		case types.RepeatK:
			if node.Children[0].Type == types.Integer {
				typeError(node.Lineno, "repeat test is not Boolean")
			}
		}
	}
}

func transverse(buf *buffer, node *types.TreeNode, preProc procNode, postProc procNode) {
	preProc(buf, node)
	for index := 0; index < len(node.Children); index++ {
		transverse(buf, node.Children[index], preProc, postProc)
	}
	postProc(buf, node)
	if node.Sibling != nil {
		transverse(buf, node.Sibling, preProc, postProc)
	}

}

func BuildSymtab(node *types.TreeNode) map[string]types.Bucket {
	buf := buffer{0, make(map[string]types.Bucket)}
	buf.location = 0
	transverse(&buf, node, insertNode, nullProc)
	return buf.bucketMap
}

func TypeCheck(node *types.TreeNode) {
	transverse(nil, node, nullProc, checkNode)
}
<|endoftext|>


<commit_before>package pg

import (
	"context"
	"fmt"
	"time"

	"github.com/go-pg/pg/internal/pool"
	"github.com/go-pg/pg/orm"
)

// Connect connects to a database using provided options.
//
// The returned DB is safe for concurrent use by multiple goroutines
// and maintains its own connection pool.
func Connect(opt *Options) *DB {
	opt.init()
	db := &DB{
		baseDB: &baseDB{
			opt:  opt,
			pool: newConnPool(opt),
		},
	}
	db.baseDB.db = db
	return db
}

// DB is a database handle representing a pool of zero or more
// underlying connections. It's safe for concurrent use by multiple
// goroutines.
type DB struct {
	*baseDB
	ctx context.Context
}

var _ orm.DB = (*DB)(nil)

func (db *DB) String() string {
	return fmt.Sprintf("DB<Addr=%q%s>", db.opt.Addr, db.fmter)
}

// Options returns read-only Options that were used to connect to the DB.
func (db *DB) Options() *Options {
	return db.opt
}

// Context returns DB context.
func (db *DB) Context() context.Context {
	if db.ctx != nil {
		return db.ctx
	}
	return context.Background()
}

// WithContext returns a copy of the DB that uses the ctx.
func (db *DB) WithContext(ctx context.Context) *DB {
	return &DB{
		baseDB: db.baseDB,
		ctx:    ctx,
	}
}

// WithTimeout returns a copy of the DB that uses d as the read/write timeout.
func (db *DB) WithTimeout(d time.Duration) *DB {
	return &DB{
		baseDB: db.baseDB.WithTimeout(d),
		ctx:    db.ctx,
	}
}

// WithParam returns a copy of the DB that replaces the param with the value
// in queries.
func (db *DB) WithParam(param string, value interface{}) *DB {
	return &DB{
		baseDB: db.baseDB.WithParam(param, value),
		ctx:    db.ctx,
	}
}

// Listen listens for notifications sent with NOTIFY command.
func (db *DB) Listen(channels ...string) *Listener {
	ln := &Listener{
		db: db,
	}
	ln.init()
	_ = ln.Listen(channels...)
	return ln
}

// Conn represents a single database connection rather than a pool of database
// connections. Prefer running queries from DB unless there is a specific
// need for a continuous single database connection.
//
// A Conn must call Close to return the connection to the database pool
// and may do so concurrently with a running query.
//
// After a call to Close, all operations on the connection fail.
type Conn struct {
	*baseDB
	ctx context.Context
}

var _ orm.DB = (*Conn)(nil)

// Conn returns a single connection by either opening a new connection
// or returning an existing connection from the connection pool. Conn will
// block until either a connection is returned or ctx is canceled.
// Queries run on the same Conn will be run in the same database session.
//
// Every Conn must be returned to the database pool after use by
// calling Conn.Close.
func (db *DB) Conn() *Conn {
	conn := &Conn{
		baseDB: db.baseDB.withPool(pool.NewSingleConnPool(db.pool)),
		ctx:    db.ctx,
	}
	conn.baseDB.db = conn
	return conn
}

// Context returns DB context.
func (db *Conn) Context() context.Context {
	if db.ctx != nil {
		return db.ctx
	}
	return context.Background()
}

// WithContext returns a copy of the DB that uses the ctx.
func (db *Conn) WithContext(ctx context.Context) *Conn {
	return &Conn{
		baseDB: db.baseDB,
		ctx:    ctx,
	}
}

// WithTimeout returns a copy of the DB that uses d as the read/write timeout.
func (db *Conn) WithTimeout(d time.Duration) *Conn {
	return &Conn{
		baseDB: db.baseDB.WithTimeout(d),
		ctx:    db.ctx,
	}
}

// WithParam returns a copy of the DB that replaces the param with the value
// in queries.
func (db *Conn) WithParam(param string, value interface{}) *Conn {
	return &Conn{
		baseDB: db.baseDB.WithParam(param, value),
		ctx:    db.ctx,
	}
}
<commit_msg>Cleanup<commit_after>package pg

import (
	"context"
	"fmt"
	"time"

	"github.com/go-pg/pg/internal/pool"
	"github.com/go-pg/pg/orm"
)

// Connect connects to a database using provided options.
//
// The returned DB is safe for concurrent use by multiple goroutines
// and maintains its own connection pool.
func Connect(opt *Options) *DB {
	opt.init()
	return newDB(&baseDB{
		opt:  opt,
		pool: newConnPool(opt),
	}, nil)
}

func newDB(baseDB *baseDB, ctx context.Context) *DB {
	db := &DB{
		baseDB: baseDB,
		ctx:    ctx,
	}
	db.baseDB.db = db
	return db
}

// DB is a database handle representing a pool of zero or more
// underlying connections. It's safe for concurrent use by multiple
// goroutines.
type DB struct {
	*baseDB
	ctx context.Context
}

var _ orm.DB = (*DB)(nil)

func (db *DB) String() string {
	return fmt.Sprintf("DB<Addr=%q%s>", db.opt.Addr, db.fmter)
}

// Options returns read-only Options that were used to connect to the DB.
func (db *DB) Options() *Options {
	return db.opt
}

// Context returns DB context.
func (db *DB) Context() context.Context {
	if db.ctx != nil {
		return db.ctx
	}
	return context.Background()
}

// WithContext returns a copy of the DB that uses the ctx.
func (db *DB) WithContext(ctx context.Context) *DB {
	return newDB(db.baseDB, ctx)
}

// WithTimeout returns a copy of the DB that uses d as the read/write timeout.
func (db *DB) WithTimeout(d time.Duration) *DB {
	return newDB(db.baseDB.WithTimeout(d), db.ctx)
}

// WithParam returns a copy of the DB that replaces the param with the value
// in queries.
func (db *DB) WithParam(param string, value interface{}) *DB {
	return newDB(db.baseDB.WithParam(param, value), db.ctx)
}

// Listen listens for notifications sent with NOTIFY command.
func (db *DB) Listen(channels ...string) *Listener {
	ln := &Listener{
		db: db,
	}
	ln.init()
	_ = ln.Listen(channels...)
	return ln
}

// Conn represents a single database connection rather than a pool of database
// connections. Prefer running queries from DB unless there is a specific
// need for a continuous single database connection.
//
// A Conn must call Close to return the connection to the database pool
// and may do so concurrently with a running query.
//
// After a call to Close, all operations on the connection fail.
type Conn struct {
	*baseDB
	ctx context.Context
}

var _ orm.DB = (*Conn)(nil)

// Conn returns a single connection by either opening a new connection
// or returning an existing connection from the connection pool. Conn will
// block until either a connection is returned or ctx is canceled.
// Queries run on the same Conn will be run in the same database session.
//
// Every Conn must be returned to the database pool after use by
// calling Conn.Close.
func (db *DB) Conn() *Conn {
	return newConn(db.baseDB.withPool(pool.NewSingleConnPool(db.pool)), db.ctx)
}

func newConn(baseDB *baseDB, ctx context.Context) *Conn {
	conn := &Conn{
		baseDB: baseDB,
		ctx:    ctx,
	}
	conn.baseDB.db = conn
	return conn
}

// Context returns DB context.
func (db *Conn) Context() context.Context {
	if db.ctx != nil {
		return db.ctx
	}
	return context.Background()
}

// WithContext returns a copy of the DB that uses the ctx.
func (db *Conn) WithContext(ctx context.Context) *Conn {
	return newConn(db.baseDB, ctx)
}

// WithTimeout returns a copy of the DB that uses d as the read/write timeout.
func (db *Conn) WithTimeout(d time.Duration) *Conn {
	return newConn(db.baseDB.WithTimeout(d), db.ctx)
}

// WithParam returns a copy of the DB that replaces the param with the value
// in queries.
func (db *Conn) WithParam(param string, value interface{}) *Conn {
	return newConn(db.baseDB.WithParam(param, value), db.ctx)
}
<|endoftext|>


<commit_before>package webgo

import (
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// This file contains a wrapper to use over Mgo driver.
// Only basic operations are implemented as of now.
// Need to add Delete function

/*
 DB Session creation is done inside a struct
 Developed based on the answer in
 http://stackoverflow.com/questions/26574594/best-practice-to-maintain-a-mgo-session
*/
type DataStore struct {
	DbName  string
	Session *mgo.Session
}

// ===

// Clone the master session and return
func (ds *DataStore) getSession() *mgo.Session {
	return ds.Session.Clone()
}

// ===

// Get appropriate MongoDB collection
func (ds *DataStore) GetSessionCollection(collection string) (*mgo.Session, *mgo.Collection) {
	s := ds.getSession()
	c := s.DB(ds.DbName).C(collection)

	return s, c
}

// ===

// Do a MongoDB Get
func (ds *DataStore) Get(collection string, conditions interface{}, resultStruct interface{}) ([]bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(conditions).All(resultStruct)
		if err != nil {
			return nil, err
		}
		return nil, nil
	}

	var data []bson.M
	err := c.Find(conditions).All(&data)
	if err != nil {
		if err == mgo.ErrNotFound {
			return nil, nil
		}
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB GetAll
func (ds *DataStore) GetAll(collection string, resultStruct interface{}) ([]bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(nil).All(resultStruct)
		if err != nil {
			return nil, err
		}
		return nil, nil
	}

	var data []bson.M
	err := c.Find(nil).All(&data)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB GetOne
func (ds *DataStore) GetOne(collection string, conditions interface{}, resultStruct interface{}) (bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(conditions).One(resultStruct)
		if err != nil {
			return nil, err
		}
		return nil, nil
	}

	var data bson.M
	err := c.Find(conditions).One(&data)
	if err != nil {
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB Save
func (ds *DataStore) Save(collection string, data interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.Insert(data)
	if err != nil {
		Err.Log("db.go", "Save()", err)
		return err
	}
	return nil
}

// ===

// Do a MongoDB Update - multiple records
func (ds *DataStore) Update(collection string, condition, updateData interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.Update(condition, updateData)
	if err != nil {
		return err
	}

	return nil
}

// ===

// Do a MongoDB update - single record, by MongoID
func (ds *DataStore) UpdateId(collection string, _id, data interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.UpdateId(_id, data)
	if err != nil {
		return err
	}

	return nil
}

// ===

// Create a new data store
func newDataStore(user, pass, host, port, name string) (*DataStore, error) {
	session, err := mgo.Dial("mongodb://" + user + ":" + pass + "@" + host + ":" + port + "/" + name)
	if err != nil {
		return nil, err
	}
	session.SetSafe(&mgo.Safe{})

	return &DataStore{DbName: name, Session: session}, nil
}

// ===

// Initializing Mongo DB
func InitDB(user, pass, host, port, name string) *DataStore {
	dStore, err := newDataStore(user, pass, host, port, name)
	if err != nil {
		Err.Fatal("db.go", "InitDB()", err)
		return nil
	}
	return dStore
}

// ===
<commit_msg>db functions return nil, nil on empty data<commit_after>package webgo

import (
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// This file contains a wrapper to use over Mgo driver.
// Only basic operations are implemented as of now.
// Need to add Delete function

/*
 DB Session creation is done inside a struct
 Developed based on the answer in
 http://stackoverflow.com/questions/26574594/best-practice-to-maintain-a-mgo-session
*/
type DataStore struct {
	DbName  string
	Session *mgo.Session
}

// ===

// Clone the master session and return
func (ds *DataStore) getSession() *mgo.Session {
	return ds.Session.Clone()
}

// ===

// Get appropriate MongoDB collection
func (ds *DataStore) GetSessionCollection(collection string) (*mgo.Session, *mgo.Collection) {
	s := ds.getSession()
	c := s.DB(ds.DbName).C(collection)

	return s, c
}

// ===

// Do a MongoDB Get
func (ds *DataStore) Get(collection string, conditions interface{}, resultStruct interface{}) ([]bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(conditions).All(resultStruct)
		if err != nil {
			if err == mgo.ErrNotFound {
				return nil, nil
			}
			return nil, err
		}
		return nil, nil
	}

	var data []bson.M
	err := c.Find(conditions).All(&data)
	if err != nil {
		if err == mgo.ErrNotFound {
			return nil, nil
		}
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB GetAll
func (ds *DataStore) GetAll(collection string, resultStruct interface{}) ([]bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(nil).All(resultStruct)
		if err != nil {
			if err == mgo.ErrNotFound {
				return nil, nil
			}
			return nil, err
		}
		return nil, nil
	}

	var data []bson.M
	err := c.Find(nil).All(&data)
	if err != nil {
		if err == mgo.ErrNotFound {
			return nil, nil
		}
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB GetOne
func (ds *DataStore) GetOne(collection string, conditions interface{}, resultStruct interface{}) (bson.M, error) {

	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	if resultStruct != nil {
		err := c.Find(conditions).One(resultStruct)
		if err != nil {
			if err == mgo.ErrNotFound {
				return nil, nil
			}
			return nil, err
		}
		return nil, nil
	}

	var data bson.M
	err := c.Find(conditions).One(&data)
	if err != nil {
		if err == mgo.ErrNotFound {
			return nil, nil
		}
		return nil, err
	}
	return data, nil
}

// ===

// Do a MongoDB Save
func (ds *DataStore) Save(collection string, data interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.Insert(data)
	if err != nil {
		Err.Log("db.go", "Save()", err)
		return err
	}
	return nil
}

// ===

// Do a MongoDB Update - multiple records
func (ds *DataStore) Update(collection string, condition, updateData interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.Update(condition, updateData)
	if err != nil {
		return err
	}

	return nil
}

// ===

// Do a MongoDB update - single record, by MongoID
func (ds *DataStore) UpdateId(collection string, _id, data interface{}) error {
	s, c := ds.GetSessionCollection(collection)
	defer s.Close()

	err := c.UpdateId(_id, data)
	if err != nil {
		return err
	}

	return nil
}

// ===

// Create a new data store
func newDataStore(user, pass, host, port, name string) (*DataStore, error) {
	session, err := mgo.Dial("mongodb://" + user + ":" + pass + "@" + host + ":" + port + "/" + name)
	if err != nil {
		return nil, err
	}
	session.SetSafe(&mgo.Safe{})

	return &DataStore{DbName: name, Session: session}, nil
}

// ===

// Initializing Mongo DB
func InitDB(user, pass, host, port, name string) *DataStore {
	dStore, err := newDataStore(user, pass, host, port, name)
	if err != nil {
		Err.Fatal("db.go", "InitDB()", err)
		return nil
	}
	return dStore
}

// ===
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

const (
	TRADB = ".trago.db"
	bytes = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	replicaId string
	version int
	files map[string]FileState
}

type FileState struct {
	size int
	mtime int64
	// TODO: use a hash as well
}

func main() {
	db, err := parseDbFile()
	checkError(err)

	fmt.Println(db)
}

func parseDbFile() (TraDb, error) {
	tradb := TraDb{}

	_, err := ioutil.ReadFile(TRADB)
	if os.IsNotExist(err) {
		fmt.Println("didn't find .trago.db")
		tradb = createDb()
		writeDb(tradb)
	} else {
		return tradb, err
	}

	return tradb, nil
}

func createDb() TraDb {
	replicaId := make([]byte, 16)
	version := 1

	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue			// ignore directories for now
		}
		fs := FileState{
			size: int(file.Size()),
			mtime: file.ModTime().UnixNano(),
		}
		filemap[file.Name()] = fs
	}

	return TraDb{
		replicaId: string(replicaId),
		version: version,
		files: filemap,
	}
}

func writeDb(tradb TraDb) {
	preamble := fmt.Sprintf(
		"replica %s\nversion %d\n# files\n",
		tradb.replicaId,
		tradb.version,
	)

	fileEntries := make([]string, len(tradb.files))

	i := 0
	for filename, info := range tradb.files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d",
			filename,
			info.size,
			info.mtime,
		)
		i = i+1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	checkError(err)
}

func findFiles(path string) {
	files, err := ioutil.ReadDir(path)
	checkError(err)

	for _, file := range files {
		fmt.Println(file.Name())
	}
}

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>parsing db file<commit_after>package main

import (
	"bufio"
	"math/rand"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

const (
	TRADB = ".trago.db"
	bytes = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	replicaId string
	version int
	files map[string]FileState
}

type FileState struct {
	size int
	mtime int64
	// TODO: use a hash as well
}

func main() {
	db, err := parseDbFile()
	checkError(err)

	fmt.Println(db)
}

func parseDbFile() (TraDb, error) {
	tradb := TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		fmt.Println("didn't find .trago.db")
		tradb = createDb()
		writeDb(tradb)
		return tradb, nil
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()
	tradb.files = make(map[string]FileState)

	scanner := bufio.NewScanner(dbfile)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch (fields[0]) {
			case "file":
			    if (len(fields) != 4) {
					continue
				}

			    size, err := strconv.Atoi(fields[2])
			    checkError(err)
			    mtime, err := strconv.ParseInt(fields[3], 10, 64)
			    checkError(err)

			    tradb.files[fields[1]] = FileState{
					size: size,
					mtime: mtime,
				}
			case "version":
			    if (len(fields) != 2) {
					continue
				}
			    version, err := strconv.Atoi(fields[1])
			    checkError(err)

			    tradb.version = version
			case "replica":
			    if (len(fields) != 2) {
					continue
				}
			    tradb.replicaId = fields[1]
		}
	}

	checkError(scanner.Err())

	return tradb, nil
}

func createDb() TraDb {
	replicaId := make([]byte, 16)
	version := 1

	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue			// ignore directories for now
		}
		fs := FileState{
			size: int(file.Size()),
			mtime: file.ModTime().UnixNano(),
		}
		filemap[file.Name()] = fs
	}

	return TraDb{
		replicaId: string(replicaId),
		version: version,
		files: filemap,
	}
}

func writeDb(tradb TraDb) {
	preamble := fmt.Sprintf(
		"replica %s\nversion %d\n# files\n",
		tradb.replicaId,
		tradb.version,
	)

	fileEntries := make([]string, len(tradb.files))

	i := 0
	for filename, info := range tradb.files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d",
			filename,
			info.size,
			info.mtime,
		)
		i = i+1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	checkError(err)
}

func findFiles(path string) {
	files, err := ioutil.ReadDir(path)
	checkError(err)

	for _, file := range files {
		fmt.Println(file.Name())
	}
}

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that consumes the output of a `comeback verify` run (--fast mode
// is okay), assuming that the roots were all backup jobs of interest. Any
// score that is in the bucket but not represented in the verify output is
// cloned to a garbage/ prefix in the bucket, and deleted from the blobs/
// prefix.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/verify"
	"github.com/jacobsa/comeback/wiring"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdGC = &Command{
	Name: "gc",
}

var fInput = cmdGC.Flags.String(
	"input",
	"",
	"Path to a file containing the output of a verify run.")

func init() {
	cmdGC.Run = runGC // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Parse the supplied input line, returning a list of all scores mentioned.
func parseGCInputLine(
	line []byte) (scores []blob.Score, err error) {
	// We expect space-separate components.
	components := bytes.Split(line, []byte{' '})
	if len(components) < 2 {
		err = fmt.Errorf(
			"Expected at least two components, got %d.",
			len(components))

		return
	}

	// The first should be the timestmap.
	_, err = time.Parse(time.RFC3339, string(components[0]))
	if err != nil {
		err = fmt.Errorf("time.Parse(%q): %v", components[0], err)
		return
	}

	// The rest are node names understood by package verify.
	for i := 1; i < len(components); i++ {
		node := string(components[i])

		var score blob.Score
		_, score, err = verify.ParseNodeName(node)
		if err != nil {
			err = fmt.Errorf("ParseNodeName(%q): %v", node, err)
			return
		}

		scores = append(scores, score)
	}

	return
}

// Parse the verify output, returning a list of all scores encountered.
func parseGCInput(
	r io.Reader) (scores []blob.Score, err error) {
	reader := bufio.NewReader(r)

	for {
		// Find the next line. EOF with no data means we are done; otherwise ignore
		// EOF in case the file doesn't end with a newline.
		var line []byte
		line, err = reader.ReadBytes('\n')
		if err == io.EOF {
			err = nil
			if len(line) == 0 {
				break
			}
		}

		// Propagate other errors.
		if err != nil {
			err = fmt.Errorf("ReadBytes: %v", err)
			return
		}

		// Trim the delimiter.
		line = line[:len(line)-1]

		// Parse the line.
		var lineScores []blob.Score
		lineScores, err = parseGCInputLine(line)
		if err != nil {
			err = fmt.Errorf("parseGCInputLine(%q): %v", line, err)
			return
		}

		scores = append(scores, lineScores...)
	}

	return
}

// Filter out scores that are in the list of non-garbage accessible scores,
// passing on only garbage.
func filterToGarbage(
	ctx context.Context,
	accessible []blob.Score,
	allScores <-chan blob.Score,
	garbageScores chan<- blob.Score) (err error) {
	// Create a map indexing the accessible scores.
	accessibleMap := make(map[blob.Score]struct{})
	for _, score := range accessible {
		accessibleMap[score] = struct{}{}
	}

	// Process each score.
	for score := range allScores {
		// Is this score accessible?
		if _, ok := accessibleMap[score]; ok {
			continue
		}

		// Send it down the garbage chute.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case garbageScores <- score:
		}
	}

	return
}

// Clone garbage blobs into a new location. Pass on the names of the source
// objects that were cloned.
func cloneGarbage(
	ctx context.Context,
	bucket gcs.Bucket,
	garbageScores <-chan blob.Score,
	garbageObjects chan<- string) (err error) {
	b := syncutil.NewBundle(ctx)

	const parallelism = 128
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			// Process each score.
			for score := range garbageScores {
				srcName := wiring.BlobObjectNamePrefix + score.Hex()

				// Clone the object.
				req := &gcs.CopyObjectRequest{
					SrcName: srcName,
					DstName: fmt.Sprintf("garbage/%s", score.Hex()),
				}

				_, err = bucket.CopyObject(ctx, req)
				if err != nil {
					err = fmt.Errorf("CopyObject: %v", err)
					return
				}

				// Write out the name of the object to be deleted.
				select {
				case <-ctx.Done():
					err = ctx.Err()

				case garbageObjects <- srcName:
				}
			}

			return
		})
	}

	err = b.Join()
	return
}

// Delete all objects whose name come in on the supplied channel.
func deleteObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	names <-chan string) (err error) {
	b := syncutil.NewBundle(ctx)

	const parallelism = 128
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			for name := range names {
				err = bucket.DeleteObject(ctx, name)
				if err != nil {
					err = fmt.Errorf("DeleteObject(%q): %v", name, err)
					return
				}
			}

			return
		})
	}

	err = b.Join()
	return
}

////////////////////////////////////////////////////////////////////////
// GC
////////////////////////////////////////////////////////////////////////

func runGC(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Grab dependencies.
	bucket := getBucket()

	// Open the input file.
	if *fInput == "" {
		err = fmt.Errorf("You must set --input.")
		return
	}

	inputFile, err := os.Open(*fInput)
	if err != nil {
		err = fmt.Errorf("Open: %v", err)
		return
	}

	// Parse it.
	accessibleScores, err := parseGCInput(inputFile)
	inputFile.Close()

	if err != nil {
		err = fmt.Errorf("parseGCInput: %v", err)
		return
	}

	b := syncutil.NewBundle(context.Background())

	// List all extant scores into a channel.
	allScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScores)
		err = blob.ListScores(ctx, bucket, wiring.BlobObjectNamePrefix, allScores)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Count the total number of scores, periodically printing status updates.
	var allScoresCount uint64
	var garbageScoresCount uint64

	allScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScoresAfterCounting)
		ticker := time.Tick(2 * time.Second)

		for score := range allScores {
			allScoresCount++

			// Print a status update?
			select {
			case <-ticker:
				g := atomic.LoadUint64(&garbageScoresCount)
				log.Printf("%d scores seen; %d garbage so far.", allScoresCount, g)

			default:
			}

			// Pass on the score.
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case allScoresAfterCounting <- score:
			}
		}

		return
	})

	// Filter to garbage scores.
	garbageScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScores)
		err = filterToGarbage(
			ctx,
			accessibleScores,
			allScoresAfterCounting,
			garbageScores)

		if err != nil {
			err = fmt.Errorf("filterToGarbage: %v", err)
			return
		}

		return
	})

	// Count the number of garbage scores.
	garbageScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScoresAfterCounting)
		for score := range garbageScores {
			atomic.AddUint64(&garbageScoresCount, 1)

			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case garbageScoresAfterCounting <- score:
			}
		}

		return
	})

	// Clone garbage blobs into a backup location.
	toDelete := make(chan string, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(toDelete)
		err = cloneGarbage(
			ctx,
			bucket,
			garbageScoresAfterCounting,
			toDelete)
		if err != nil {
			err = fmt.Errorf("cloneGarbage: %v", err)
			return
		}

		return
	})

	// Delete the original objects.
	b.Add(func(ctx context.Context) (err error) {
		err = deleteObjects(ctx, bucket, toDelete)
		if err != nil {
			err = fmt.Errorf("deleteObjects: %v", err)
			return
		}

		return
	})

	err = b.Join()
	if err != nil {
		return
	}

	// Print a summary.
	log.Printf(
		"Deleted %d objects out of %d total.",
		garbageScoresCount,
		allScoresCount)
}
<commit_msg>Use verifyRecord code in gc, too.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that consumes the output of a `comeback verify` run (--fast mode
// is okay), assuming that the roots were all backup jobs of interest. Any
// score that is in the bucket but not represented in the verify output is
// cloned to a garbage/ prefix in the bucket, and deleted from the blobs/
// prefix.

package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/wiring"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdGC = &Command{
	Name: "gc",
}

var fInput = cmdGC.Flags.String(
	"input",
	"",
	"Path to a file containing the output of a verify run.")

func init() {
	cmdGC.Run = runGC // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Parse the verify output, returning a list of all scores encountered.
func parseGCInput(
	ctx context.Context,
	r io.Reader) (scores []blob.Score, err error) {
	b := syncutil.NewBundle(ctx)

	// Parse the input, writing records into a channel.
	verifyRecords := make(chan verifyRecord, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(verifyRecords)
		err = parseVerifyOutput(ctx, r, verifyRecords)
		if err != nil {
			err = fmt.Errorf("parseVerifyOutput: %v", err)
			return
		}

		return
	})

	// Parse and save the scores from the records.
	b.Add(func(ctx context.Context) (err error) {
		for r := range verifyRecords {
			var recordScores []blob.Score
			recordScores, err = r.Scores()
			if err != nil {
				err = fmt.Errorf("Scores: %v", err)
				return
			}

			scores = append(scores, recordScores...)
		}

		return
	})

	err = b.Join()
	return
}

// Filter out scores that are in the list of non-garbage accessible scores,
// passing on only garbage.
func filterToGarbage(
	ctx context.Context,
	accessible []blob.Score,
	allScores <-chan blob.Score,
	garbageScores chan<- blob.Score) (err error) {
	// Create a map indexing the accessible scores.
	accessibleMap := make(map[blob.Score]struct{})
	for _, score := range accessible {
		accessibleMap[score] = struct{}{}
	}

	// Process each score.
	for score := range allScores {
		// Is this score accessible?
		if _, ok := accessibleMap[score]; ok {
			continue
		}

		// Send it down the garbage chute.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case garbageScores <- score:
		}
	}

	return
}

// Clone garbage blobs into a new location. Pass on the names of the source
// objects that were cloned.
func cloneGarbage(
	ctx context.Context,
	bucket gcs.Bucket,
	garbageScores <-chan blob.Score,
	garbageObjects chan<- string) (err error) {
	b := syncutil.NewBundle(ctx)

	const parallelism = 128
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			// Process each score.
			for score := range garbageScores {
				srcName := wiring.BlobObjectNamePrefix + score.Hex()

				// Clone the object.
				req := &gcs.CopyObjectRequest{
					SrcName: srcName,
					DstName: fmt.Sprintf("garbage/%s", score.Hex()),
				}

				_, err = bucket.CopyObject(ctx, req)
				if err != nil {
					err = fmt.Errorf("CopyObject: %v", err)
					return
				}

				// Write out the name of the object to be deleted.
				select {
				case <-ctx.Done():
					err = ctx.Err()

				case garbageObjects <- srcName:
				}
			}

			return
		})
	}

	err = b.Join()
	return
}

// Delete all objects whose name come in on the supplied channel.
func deleteObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	names <-chan string) (err error) {
	b := syncutil.NewBundle(ctx)

	const parallelism = 128
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			for name := range names {
				err = bucket.DeleteObject(ctx, name)
				if err != nil {
					err = fmt.Errorf("DeleteObject(%q): %v", name, err)
					return
				}
			}

			return
		})
	}

	err = b.Join()
	return
}

////////////////////////////////////////////////////////////////////////
// GC
////////////////////////////////////////////////////////////////////////

func runGC(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Grab dependencies.
	bucket := getBucket()

	// Open the input file.
	if *fInput == "" {
		err = fmt.Errorf("You must set --input.")
		return
	}

	inputFile, err := os.Open(*fInput)
	if err != nil {
		err = fmt.Errorf("Open: %v", err)
		return
	}

	// Parse it.
	accessibleScores, err := parseGCInput(context.Background(), inputFile)
	inputFile.Close()

	if err != nil {
		err = fmt.Errorf("parseGCInput: %v", err)
		return
	}

	b := syncutil.NewBundle(context.Background())

	// List all extant scores into a channel.
	allScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScores)
		err = blob.ListScores(ctx, bucket, wiring.BlobObjectNamePrefix, allScores)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Count the total number of scores, periodically printing status updates.
	var allScoresCount uint64
	var garbageScoresCount uint64

	allScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScoresAfterCounting)
		ticker := time.Tick(2 * time.Second)

		for score := range allScores {
			allScoresCount++

			// Print a status update?
			select {
			case <-ticker:
				g := atomic.LoadUint64(&garbageScoresCount)
				log.Printf("%d scores seen; %d garbage so far.", allScoresCount, g)

			default:
			}

			// Pass on the score.
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case allScoresAfterCounting <- score:
			}
		}

		return
	})

	// Filter to garbage scores.
	garbageScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScores)
		err = filterToGarbage(
			ctx,
			accessibleScores,
			allScoresAfterCounting,
			garbageScores)

		if err != nil {
			err = fmt.Errorf("filterToGarbage: %v", err)
			return
		}

		return
	})

	// Count the number of garbage scores.
	garbageScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScoresAfterCounting)
		for score := range garbageScores {
			atomic.AddUint64(&garbageScoresCount, 1)

			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case garbageScoresAfterCounting <- score:
			}
		}

		return
	})

	// Clone garbage blobs into a backup location.
	toDelete := make(chan string, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(toDelete)
		err = cloneGarbage(
			ctx,
			bucket,
			garbageScoresAfterCounting,
			toDelete)
		if err != nil {
			err = fmt.Errorf("cloneGarbage: %v", err)
			return
		}

		return
	})

	// Delete the original objects.
	b.Add(func(ctx context.Context) (err error) {
		err = deleteObjects(ctx, bucket, toDelete)
		if err != nil {
			err = fmt.Errorf("deleteObjects: %v", err)
			return
		}

		return
	})

	err = b.Join()
	if err != nil {
		return
	}

	// Print a summary.
	log.Printf(
		"Deleted %d objects out of %d total.",
		garbageScoresCount,
		allScoresCount)
}
<|endoftext|>


<commit_before>package pg

import (
	"database/sql/driver"
	"github.com/cevian/pq"
	"reflect"
	"stash.cloudflare.com/go-stream/cube"
	"stash.cloudflare.com/go-stream/util/slog"
)

type Executor struct {
	table *Table
	conn  driver.Conn
}

func NewExecutor(t *Table, c driver.Conn) *Executor {
	return &Executor{t, c}
}

func (e *Executor) ExecErr(sql string, args ...interface{}) (driver.Result, error) {
	exec := e.conn.(driver.Execer)

	dargs := make([]driver.Value, len(args))
	for n, arg := range args {
		var err error
		dargs[n], err = driver.DefaultParameterConverter.ConvertValue(arg)
		if err != nil {
			slog.Fatalf("sql: converting Exec argument #%d's type: %v", n, err)
		}
	}
	return exec.Exec(sql, dargs)
}

func (e *Executor) Exec(sql string, args ...interface{}) driver.Result {
	res, err := e.ExecErr(sql, args...)
	if err != nil {
		slog.Fatalf("Sql: %v Err: %v", sql, err)
	}
	return res
}

func (e *Executor) CreateBaseTable() {
	e.Exec(e.table.CreateTableSql(false))
}

func (e *Executor) DropAllTables() {
	e.Exec(e.table.DropTableSql())
}

func (e *Executor) CreateForeignTable(serverName string) {
	e.Exec(e.table.CreateForeignTableSql(serverName))
}

func (e *Executor) DropForeignTable(serverName string) {
	e.Exec(e.table.DropForeignTableSql(serverName))
}

func (e *Executor) CreateForeignTableView(serverNames []string, selfServerName string) {
	e.Exec(e.table.CreateForeignTableViewSql(serverNames, selfServerName))
}

func (e *Executor) DropForeignTableView() {
	e.Exec(e.table.DropForeignTableViewSql())
}

func getPartition(p cube.Partition) Partition {
	switch pt := p.(type) {
	case cube.TimePartition:
		return &TimePartition{&pt}
	default:
		slog.Fatalf("Unknown Partition Type %v", reflect.TypeOf(pt))
	}
	panic("Never Here")
}

func (e *Executor) DropPartition(p cube.Partition) {
	e.Exec(e.table.DropPartitionTableSql(getPartition(p)))
}

func (e *Executor) UpsertCube(p cube.Partition, c cube.Cuber) {
	e.UpsertCubes(p, []cube.Cuber{c})
}

func (e *Executor) UpsertCubes(p cube.Partition, c []cube.Cuber) {
	tx, err := e.conn.Begin()
	if err != nil {
		slog.Fatalf("Error starting transaction %v", err)
	}

	part := getPartition(p)

	//TODO: have a cache of existing partition tables...dont recreate if not necessary
	e.Exec(e.table.CreatePartitionTableSql(part))

	e.Exec(e.table.CreateTemporaryCopyTableSql(part))
	cy := pq.NewCopierFromConn(e.conn)
	err = cy.Start(e.table.CopyTableSql(part))
	if err != nil {
		slog.Fatalf("Error starting copy %v", err)
	}

	for _, cube := range c {
		err = cy.Send(e.table.CopyDataFull(cube))
		if err != nil {
			slog.Fatalf("Error copying %v", err)
		}
	}

	err = cy.Close()
	if err != nil {
		slog.Fatalf("Error Ending Copy %v", err)
	}

	e.Exec(e.table.MergeCopySql(part))

	err = tx.Commit()
	if err != nil {
		slog.Fatalf("Error Committing tx %v ", err)
	}

}
<commit_msg>Making sql errors non-fatal<commit_after>package pg

import (
	"database/sql/driver"
	"github.com/cevian/pq"
	"logger"
	"reflect"
	"stash.cloudflare.com/go-stream/cube"
	"stash.cloudflare.com/go-stream/util/slog"
)

type Executor struct {
	table *Table
	conn  driver.Conn
}

func NewExecutor(t *Table, c driver.Conn) *Executor {
	return &Executor{t, c}
}

func (e *Executor) ExecErr(sql string, args ...interface{}) (driver.Result, error) {
	exec := e.conn.(driver.Execer)

	dargs := make([]driver.Value, len(args))
	for n, arg := range args {
		var err error
		dargs[n], err = driver.DefaultParameterConverter.ConvertValue(arg)
		if err != nil {
			slog.Fatalf("sql: converting Exec argument #%d's type: %v", n, err)
		}
	}
	return exec.Exec(sql, dargs)
}

func (e *Executor) Exec(sql string, args ...interface{}) driver.Result {
	res, err := e.ExecErr(sql, args...)
	if err != nil {
		slog.Logf(logger.Levels.Error, "Sql: %v Err: %v", sql, err)
	}
	return res
}

func (e *Executor) CreateBaseTable() {
	e.Exec(e.table.CreateTableSql(false))
}

func (e *Executor) DropAllTables() {
	e.Exec(e.table.DropTableSql())
}

func (e *Executor) CreateForeignTable(serverName string) {
	e.Exec(e.table.CreateForeignTableSql(serverName))
}

func (e *Executor) DropForeignTable(serverName string) {
	e.Exec(e.table.DropForeignTableSql(serverName))
}

func (e *Executor) CreateForeignTableView(serverNames []string, selfServerName string) {
	e.Exec(e.table.CreateForeignTableViewSql(serverNames, selfServerName))
}

func (e *Executor) DropForeignTableView() {
	e.Exec(e.table.DropForeignTableViewSql())
}

func getPartition(p cube.Partition) Partition {
	switch pt := p.(type) {
	case cube.TimePartition:
		return &TimePartition{&pt}
	default:
		slog.Fatalf("Unknown Partition Type %v", reflect.TypeOf(pt))
	}
	panic("Never Here")
}

func (e *Executor) DropPartition(p cube.Partition) {
	e.Exec(e.table.DropPartitionTableSql(getPartition(p)))
}

func (e *Executor) UpsertCube(p cube.Partition, c cube.Cuber) {
	e.UpsertCubes(p, []cube.Cuber{c})
}

func (e *Executor) UpsertCubes(p cube.Partition, c []cube.Cuber) {
	tx, err := e.conn.Begin()
	if err != nil {
		slog.Fatalf("Error starting transaction %v", err)
	}

	part := getPartition(p)

	//TODO: have a cache of existing partition tables...dont recreate if not necessary
	e.Exec(e.table.CreatePartitionTableSql(part))

	e.Exec(e.table.CreateTemporaryCopyTableSql(part))
	cy := pq.NewCopierFromConn(e.conn)
	err = cy.Start(e.table.CopyTableSql(part))
	if err != nil {
		slog.Fatalf("Error starting copy %v", err)
	}

	for _, cube := range c {
		err = cy.Send(e.table.CopyDataFull(cube))
		if err != nil {
			slog.Fatalf("Error copying %v", err)
		}
	}

	err = cy.Close()
	if err != nil {
		slog.Fatalf("Error Ending Copy %v", err)
	}

	e.Exec(e.table.MergeCopySql(part))

	err = tx.Commit()
	if err != nil {
		slog.Fatalf("Error Committing tx %v ", err)
	}

}
<|endoftext|>