<commit_before>// +build linux freebsd solaris

package zfs

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/parsers"
	zfs "github.com/mistifyio/go-zfs"
	"github.com/opencontainers/runc/libcontainer/label"
)

type zfsOptions struct {
	fsName    string
	mountPath string
}

func init() {
	graphdriver.Register("zfs", Init)
}

// Logger returns a zfs logger implementation.
type Logger struct{}

// Log wraps log message from ZFS driver with a prefix '[zfs]'.
func (*Logger) Log(cmd []string) {
	logrus.Debugf("[zfs] %s", strings.Join(cmd, " "))
}

// Init returns a new ZFS driver.
// It takes base mount path and an array of options which are represented as key value pairs.
// Each option is in the for key=value. 'zfs.fsname' is expected to be a valid key in the options.
func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	var err error

	if _, err := exec.LookPath("zfs"); err != nil {
		logrus.Debugf("[zfs] zfs command is not available: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}

	file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600)
	if err != nil {
		logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}
	defer file.Close()

	options, err := parseOptions(opt)
	if err != nil {
		return nil, err
	}
	options.mountPath = base

	rootdir := path.Dir(base)

	if options.fsName == "" {
		err = checkRootdirFs(rootdir)
		if err != nil {
			return nil, err
		}
	}

	if options.fsName == "" {
		options.fsName, err = lookupZfsDataset(rootdir)
		if err != nil {
			return nil, err
		}
	}

	zfs.SetLogger(new(Logger))

	filesystems, err := zfs.Filesystems(options.fsName)
	if err != nil {
		return nil, fmt.Errorf("Cannot find root filesystem %s: %v", options.fsName, err)
	}

	filesystemsCache := make(map[string]bool, len(filesystems))
	var rootDataset *zfs.Dataset
	for _, fs := range filesystems {
		if fs.Name == options.fsName {
			rootDataset = fs
		}
		filesystemsCache[fs.Name] = true
	}

	if rootDataset == nil {
		return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName)
	}

	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
	if err != nil {
		return nil, fmt.Errorf("Failed to get root uid/guid: %v", err)
	}
	if err := idtools.MkdirAllAs(base, 0700, rootUID, rootGID); err != nil {
		return nil, fmt.Errorf("Failed to create '%s': %v", base, err)
	}

	if err := mount.MakePrivate(base); err != nil {
		return nil, err
	}
	d := &Driver{
		dataset:          rootDataset,
		options:          options,
		filesystemsCache: filesystemsCache,
		uidMaps:          uidMaps,
		gidMaps:          gidMaps,
		ctr:              graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
	}
	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func parseOptions(opt []string) (zfsOptions, error) {
	var options zfsOptions
	options.fsName = ""
	for _, option := range opt {
		key, val, err := parsers.ParseKeyValueOpt(option)
		if err != nil {
			return options, err
		}
		key = strings.ToLower(key)
		switch key {
		case "zfs.fsname":
			options.fsName = val
		default:
			return options, fmt.Errorf("Unknown option %s", key)
		}
	}
	return options, nil
}

func lookupZfsDataset(rootdir string) (string, error) {
	var stat syscall.Stat_t
	if err := syscall.Stat(rootdir, &stat); err != nil {
		return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err)
	}
	wantedDev := stat.Dev

	mounts, err := mount.GetMounts()
	if err != nil {
		return "", err
	}
	for _, m := range mounts {
		if err := syscall.Stat(m.Mountpoint, &stat); err != nil {
			logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err)
			continue // may fail on fuse file systems
		}

		if stat.Dev == wantedDev && m.Fstype == "zfs" {
			return m.Source, nil
		}
	}

	return "", fmt.Errorf("Failed to find zfs dataset mounted on '%s' in /proc/mounts", rootdir)
}

// Driver holds information about the driver, such as zfs dataset, options and cache.
type Driver struct {
	dataset          *zfs.Dataset
	options          zfsOptions
	sync.Mutex       // protects filesystem cache against concurrent access
	filesystemsCache map[string]bool
	uidMaps          []idtools.IDMap
	gidMaps          []idtools.IDMap
	ctr              *graphdriver.RefCounter
}

func (d *Driver) String() string {
	return "zfs"
}

// Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver.
func (d *Driver) Cleanup() error {
	return nil
}

// Status returns information about the ZFS filesystem. It returns a two dimensional array of information
// such as pool name, dataset name, disk usage, parent quota and compression used.
// Currently it return 'Zpool', 'Zpool Health', 'Parent Dataset', 'Space Used By Parent',
// 'Space Available', 'Parent Quota' and 'Compression'.
func (d *Driver) Status() [][2]string {
	parts := strings.Split(d.dataset.Name, "/")
	pool, err := zfs.GetZpool(parts[0])

	var poolName, poolHealth string
	if err == nil {
		poolName = pool.Name
		poolHealth = pool.Health
	} else {
		poolName = fmt.Sprintf("error while getting pool information %v", err)
		poolHealth = "not available"
	}

	quota := "no"
	if d.dataset.Quota != 0 {
		quota = strconv.FormatUint(d.dataset.Quota, 10)
	}

	return [][2]string{
		{"Zpool", poolName},
		{"Zpool Health", poolHealth},
		{"Parent Dataset", d.dataset.Name},
		{"Space Used By Parent", strconv.FormatUint(d.dataset.Used, 10)},
		{"Space Available", strconv.FormatUint(d.dataset.Avail, 10)},
		{"Parent Quota", quota},
		{"Compression", d.dataset.Compression},
	}
}

// GetMetadata returns image/container metadata related to graph driver
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return nil, nil
}

func (d *Driver) cloneFilesystem(name, parentName string) error {
	snapshotName := fmt.Sprintf("%d", time.Now().Nanosecond())
	parentDataset := zfs.Dataset{Name: parentName}
	snapshot, err := parentDataset.Snapshot(snapshotName /*recursive */, false)
	if err != nil {
		return err
	}

	_, err = snapshot.Clone(name, map[string]string{"mountpoint": "legacy"})
	if err == nil {
		d.Lock()
		d.filesystemsCache[name] = true
		d.Unlock()
	}

	if err != nil {
		snapshot.Destroy(zfs.DestroyDeferDeletion)
		return err
	}
	return snapshot.Destroy(zfs.DestroyDeferDeletion)
}

func (d *Driver) zfsPath(id string) string {
	return d.options.fsName + "/" + id
}

func (d *Driver) mountPath(id string) string {
	return path.Join(d.options.mountPath, "graph", getMountpoint(id))
}

// CreateReadWrite creates a layer that is writable for use as a container
// file system.
func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
	return d.Create(id, parent, mountLabel, storageOpt)
}

// Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent.
func (d *Driver) Create(id string, parent string, mountLabel string, storageOpt map[string]string) error {
	err := d.create(id, parent, storageOpt)
	if err == nil {
		return nil
	}
	if zfsError, ok := err.(*zfs.Error); ok {
		if !strings.HasSuffix(zfsError.Stderr, "dataset already exists\n") {
			return err
		}
		// aborted build -> cleanup
	} else {
		return err
	}

	dataset := zfs.Dataset{Name: d.zfsPath(id)}
	if err := dataset.Destroy(zfs.DestroyRecursiveClones); err != nil {
		return err
	}

	// retry
	return d.create(id, parent, storageOpt)
}

func (d *Driver) create(id, parent string, storageOpt map[string]string) error {
	name := d.zfsPath(id)
	quota, err := parseStorageOpt(storageOpt)
	if err != nil {
		return err
	}
	if parent == "" {
		mountoptions := map[string]string{"mountpoint": "legacy"}
		fs, err := zfs.CreateFilesystem(name, mountoptions)
		if err == nil {
			err = setQuota(name, quota)
			if err == nil {
				d.Lock()
				d.filesystemsCache[fs.Name] = true
				d.Unlock()
			}
		}
		return err
	}
	err = d.cloneFilesystem(name, d.zfsPath(parent))
	if err == nil {
		err = setQuota(name, quota)
	}
	return err
}

func parseStorageOpt(storageOpt map[string]string) (string, error) {
	// Read size to change the disk quota per container
	for k, v := range storageOpt {
		key := strings.ToLower(k)
		switch key {
		case "size":
			return v, nil
		default:
			return "0", fmt.Errorf("Unknown option %s", key)
		}
	}
	return "0", nil
}

func setQuota(name string, quota string) error {
	if quota == "0" {
		return nil
	}
	fs, err := zfs.GetDataset(name)
	if err != nil {
		return err
	}
	return fs.SetProperty("quota", quota)
}

// Remove deletes the dataset, filesystem and the cache for the given id.
func (d *Driver) Remove(id string) error {
	name := d.zfsPath(id)
	dataset := zfs.Dataset{Name: name}
	err := dataset.Destroy(zfs.DestroyRecursive)
	if err == nil {
		d.Lock()
		delete(d.filesystemsCache, name)
		d.Unlock()
	}
	return err
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	if count := d.ctr.Increment(mountpoint); count > 1 {
		return mountpoint, nil
	}

	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		d.ctr.Decrement(mountpoint)
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		d.ctr.Decrement(mountpoint)
		return "", err
	}

	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
		d.ctr.Decrement(mountpoint)
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}

	// this could be our first mount after creation of the filesystem, and the root dir may still have root
	// permissions instead of the remapped root uid:gid (if user namespaces are enabled):
	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
		mount.Unmount(mountpoint)
		d.ctr.Decrement(mountpoint)
		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
	}

	return mountpoint, nil
}

// Put removes the existing mountpoint for the given id if it exists.
func (d *Driver) Put(id string) error {
	mountpoint := d.mountPath(id)
	if count := d.ctr.Decrement(mountpoint); count > 0 {
		return nil
	}
	mounted, err := graphdriver.Mounted(graphdriver.FsMagicZfs, mountpoint)
	if err != nil || !mounted {
		return err
	}

	logrus.Debugf(`[zfs] unmount("%s")`, mountpoint)

	if err := mount.Unmount(mountpoint); err != nil {
		return fmt.Errorf("error unmounting to %s: %v", mountpoint, err)
	}
	return nil
}

// Exists checks to see if the cache entry exists for the given id.
func (d *Driver) Exists(id string) bool {
	d.Lock()
	defer d.Unlock()
	return d.filesystemsCache[d.zfsPath(id)] == true
}
<commit_msg>zfs: expose mountpoint/dataset on inspection<commit_after>// +build linux freebsd solaris

package zfs

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/parsers"
	zfs "github.com/mistifyio/go-zfs"
	"github.com/opencontainers/runc/libcontainer/label"
)

type zfsOptions struct {
	fsName    string
	mountPath string
}

func init() {
	graphdriver.Register("zfs", Init)
}

// Logger returns a zfs logger implementation.
type Logger struct{}

// Log wraps log message from ZFS driver with a prefix '[zfs]'.
func (*Logger) Log(cmd []string) {
	logrus.Debugf("[zfs] %s", strings.Join(cmd, " "))
}

// Init returns a new ZFS driver.
// It takes base mount path and an array of options which are represented as key value pairs.
// Each option is in the for key=value. 'zfs.fsname' is expected to be a valid key in the options.
func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	var err error

	if _, err := exec.LookPath("zfs"); err != nil {
		logrus.Debugf("[zfs] zfs command is not available: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}

	file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600)
	if err != nil {
		logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}
	defer file.Close()

	options, err := parseOptions(opt)
	if err != nil {
		return nil, err
	}
	options.mountPath = base

	rootdir := path.Dir(base)

	if options.fsName == "" {
		err = checkRootdirFs(rootdir)
		if err != nil {
			return nil, err
		}
	}

	if options.fsName == "" {
		options.fsName, err = lookupZfsDataset(rootdir)
		if err != nil {
			return nil, err
		}
	}

	zfs.SetLogger(new(Logger))

	filesystems, err := zfs.Filesystems(options.fsName)
	if err != nil {
		return nil, fmt.Errorf("Cannot find root filesystem %s: %v", options.fsName, err)
	}

	filesystemsCache := make(map[string]bool, len(filesystems))
	var rootDataset *zfs.Dataset
	for _, fs := range filesystems {
		if fs.Name == options.fsName {
			rootDataset = fs
		}
		filesystemsCache[fs.Name] = true
	}

	if rootDataset == nil {
		return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName)
	}

	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
	if err != nil {
		return nil, fmt.Errorf("Failed to get root uid/guid: %v", err)
	}
	if err := idtools.MkdirAllAs(base, 0700, rootUID, rootGID); err != nil {
		return nil, fmt.Errorf("Failed to create '%s': %v", base, err)
	}

	if err := mount.MakePrivate(base); err != nil {
		return nil, err
	}
	d := &Driver{
		dataset:          rootDataset,
		options:          options,
		filesystemsCache: filesystemsCache,
		uidMaps:          uidMaps,
		gidMaps:          gidMaps,
		ctr:              graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
	}
	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func parseOptions(opt []string) (zfsOptions, error) {
	var options zfsOptions
	options.fsName = ""
	for _, option := range opt {
		key, val, err := parsers.ParseKeyValueOpt(option)
		if err != nil {
			return options, err
		}
		key = strings.ToLower(key)
		switch key {
		case "zfs.fsname":
			options.fsName = val
		default:
			return options, fmt.Errorf("Unknown option %s", key)
		}
	}
	return options, nil
}

func lookupZfsDataset(rootdir string) (string, error) {
	var stat syscall.Stat_t
	if err := syscall.Stat(rootdir, &stat); err != nil {
		return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err)
	}
	wantedDev := stat.Dev

	mounts, err := mount.GetMounts()
	if err != nil {
		return "", err
	}
	for _, m := range mounts {
		if err := syscall.Stat(m.Mountpoint, &stat); err != nil {
			logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err)
			continue // may fail on fuse file systems
		}

		if stat.Dev == wantedDev && m.Fstype == "zfs" {
			return m.Source, nil
		}
	}

	return "", fmt.Errorf("Failed to find zfs dataset mounted on '%s' in /proc/mounts", rootdir)
}

// Driver holds information about the driver, such as zfs dataset, options and cache.
type Driver struct {
	dataset          *zfs.Dataset
	options          zfsOptions
	sync.Mutex       // protects filesystem cache against concurrent access
	filesystemsCache map[string]bool
	uidMaps          []idtools.IDMap
	gidMaps          []idtools.IDMap
	ctr              *graphdriver.RefCounter
}

func (d *Driver) String() string {
	return "zfs"
}

// Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver.
func (d *Driver) Cleanup() error {
	return nil
}

// Status returns information about the ZFS filesystem. It returns a two dimensional array of information
// such as pool name, dataset name, disk usage, parent quota and compression used.
// Currently it return 'Zpool', 'Zpool Health', 'Parent Dataset', 'Space Used By Parent',
// 'Space Available', 'Parent Quota' and 'Compression'.
func (d *Driver) Status() [][2]string {
	parts := strings.Split(d.dataset.Name, "/")
	pool, err := zfs.GetZpool(parts[0])

	var poolName, poolHealth string
	if err == nil {
		poolName = pool.Name
		poolHealth = pool.Health
	} else {
		poolName = fmt.Sprintf("error while getting pool information %v", err)
		poolHealth = "not available"
	}

	quota := "no"
	if d.dataset.Quota != 0 {
		quota = strconv.FormatUint(d.dataset.Quota, 10)
	}

	return [][2]string{
		{"Zpool", poolName},
		{"Zpool Health", poolHealth},
		{"Parent Dataset", d.dataset.Name},
		{"Space Used By Parent", strconv.FormatUint(d.dataset.Used, 10)},
		{"Space Available", strconv.FormatUint(d.dataset.Avail, 10)},
		{"Parent Quota", quota},
		{"Compression", d.dataset.Compression},
	}
}

// GetMetadata returns image/container metadata related to graph driver
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return map[string]string{
		"Mountpoint": d.mountPath(id),
		"Dataset":    d.zfsPath(id),
	}, nil
}

func (d *Driver) cloneFilesystem(name, parentName string) error {
	snapshotName := fmt.Sprintf("%d", time.Now().Nanosecond())
	parentDataset := zfs.Dataset{Name: parentName}
	snapshot, err := parentDataset.Snapshot(snapshotName /*recursive */, false)
	if err != nil {
		return err
	}

	_, err = snapshot.Clone(name, map[string]string{"mountpoint": "legacy"})
	if err == nil {
		d.Lock()
		d.filesystemsCache[name] = true
		d.Unlock()
	}

	if err != nil {
		snapshot.Destroy(zfs.DestroyDeferDeletion)
		return err
	}
	return snapshot.Destroy(zfs.DestroyDeferDeletion)
}

func (d *Driver) zfsPath(id string) string {
	return d.options.fsName + "/" + id
}

func (d *Driver) mountPath(id string) string {
	return path.Join(d.options.mountPath, "graph", getMountpoint(id))
}

// CreateReadWrite creates a layer that is writable for use as a container
// file system.
func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
	return d.Create(id, parent, mountLabel, storageOpt)
}

// Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent.
func (d *Driver) Create(id string, parent string, mountLabel string, storageOpt map[string]string) error {
	err := d.create(id, parent, storageOpt)
	if err == nil {
		return nil
	}
	if zfsError, ok := err.(*zfs.Error); ok {
		if !strings.HasSuffix(zfsError.Stderr, "dataset already exists\n") {
			return err
		}
		// aborted build -> cleanup
	} else {
		return err
	}

	dataset := zfs.Dataset{Name: d.zfsPath(id)}
	if err := dataset.Destroy(zfs.DestroyRecursiveClones); err != nil {
		return err
	}

	// retry
	return d.create(id, parent, storageOpt)
}

func (d *Driver) create(id, parent string, storageOpt map[string]string) error {
	name := d.zfsPath(id)
	quota, err := parseStorageOpt(storageOpt)
	if err != nil {
		return err
	}
	if parent == "" {
		mountoptions := map[string]string{"mountpoint": "legacy"}
		fs, err := zfs.CreateFilesystem(name, mountoptions)
		if err == nil {
			err = setQuota(name, quota)
			if err == nil {
				d.Lock()
				d.filesystemsCache[fs.Name] = true
				d.Unlock()
			}
		}
		return err
	}
	err = d.cloneFilesystem(name, d.zfsPath(parent))
	if err == nil {
		err = setQuota(name, quota)
	}
	return err
}

func parseStorageOpt(storageOpt map[string]string) (string, error) {
	// Read size to change the disk quota per container
	for k, v := range storageOpt {
		key := strings.ToLower(k)
		switch key {
		case "size":
			return v, nil
		default:
			return "0", fmt.Errorf("Unknown option %s", key)
		}
	}
	return "0", nil
}

func setQuota(name string, quota string) error {
	if quota == "0" {
		return nil
	}
	fs, err := zfs.GetDataset(name)
	if err != nil {
		return err
	}
	return fs.SetProperty("quota", quota)
}

// Remove deletes the dataset, filesystem and the cache for the given id.
func (d *Driver) Remove(id string) error {
	name := d.zfsPath(id)
	dataset := zfs.Dataset{Name: name}
	err := dataset.Destroy(zfs.DestroyRecursive)
	if err == nil {
		d.Lock()
		delete(d.filesystemsCache, name)
		d.Unlock()
	}
	return err
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	if count := d.ctr.Increment(mountpoint); count > 1 {
		return mountpoint, nil
	}

	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		d.ctr.Decrement(mountpoint)
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		d.ctr.Decrement(mountpoint)
		return "", err
	}

	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
		d.ctr.Decrement(mountpoint)
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}

	// this could be our first mount after creation of the filesystem, and the root dir may still have root
	// permissions instead of the remapped root uid:gid (if user namespaces are enabled):
	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
		mount.Unmount(mountpoint)
		d.ctr.Decrement(mountpoint)
		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
	}

	return mountpoint, nil
}

// Put removes the existing mountpoint for the given id if it exists.
func (d *Driver) Put(id string) error {
	mountpoint := d.mountPath(id)
	if count := d.ctr.Decrement(mountpoint); count > 0 {
		return nil
	}
	mounted, err := graphdriver.Mounted(graphdriver.FsMagicZfs, mountpoint)
	if err != nil || !mounted {
		return err
	}

	logrus.Debugf(`[zfs] unmount("%s")`, mountpoint)

	if err := mount.Unmount(mountpoint); err != nil {
		return fmt.Errorf("error unmounting to %s: %v", mountpoint, err)
	}
	return nil
}

// Exists checks to see if the cache entry exists for the given id.
func (d *Driver) Exists(id string) bool {
	d.Lock()
	defer d.Unlock()
	return d.filesystemsCache[d.zfsPath(id)] == true
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	"code.google.com/p/go.net/html"
)

type directory struct {
	node
	de map[string]dirent
	mu sync.Mutex
}

type dirent struct {
	fuse.Dirent
	mtime time.Time
}

func (d *directory) Attr() fuse.Attr {
	return fuse.Attr{
		Mode:  os.ModeDir | 0500,
		Uid:   MyUID,
		Gid:   MyGID,
		Nlink: 1,
		Mtime: d.mtime(),
	}
}

func NewDirectory(parent *directory, fs *filesystem, name string) *directory {
	return &directory{
		node: node{parent, fs, name},
		de:   map[string]dirent{},
	}
}

func (d *directory) populate() (err fuse.Error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	if len(d.de) > 0 {
		return nil
	}
	r, err := d.fs.client.Do(d.fs.NewRequest("GET", d.fullpath()+"/"))
	if err != nil {
		log.Println("ERROR:", err)
		return err
	}
	log.Println(d.fullpath(), r.Request.URL, r.Status)
	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		switch r.StatusCode {
		case http.StatusForbidden, http.StatusUnauthorized:
			err = fuse.EPERM
		default:
			err = fuse.ENOENT
		}
	}

	switch strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] {
	case "text/html":
		doc, err := html.Parse(r.Body)
		if err != nil {
			return err
		}
		var walk func(*html.Node)
		walk = func(n *html.Node) {
			if n.Type == html.ElementNode && n.Data == "a" {
				for _, a := range n.Attr {
					if a.Key == "href" {
						direntType := fuse.DT_File
						if a.Val[len(a.Val)-1] == '/' {
							direntType = fuse.DT_Dir
						}
						name, err := url.QueryUnescape(a.Val)
						if err != nil {
							log.Printf("url.QueryUnescape(%s): %s", a.Val, err)
						}
						name = strings.TrimRight(name, "/")
						d.de[name] = dirent{
							fuse.Dirent{
								Name: name,
								Type: direntType,
							},
							ParseTime(n.NextSibling),
						}
						log.Println(name, direntType)
						break
					}
				}
			}
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				walk(c)
			}
		}
		walk(doc)
	default:
		panic("Unsupported directory response.")
	}
	return
}

func (d *directory) ReadDir(_ fs.Intr) (de []fuse.Dirent, err fuse.Error) {
	if err = d.populate(); nil == err {
		de = make([]fuse.Dirent, len(d.de))
		i := 0
		for _, v := range d.de {
			de[i] = v.Dirent
			i++
		}
	}
	return
}

var ignoreNames = map[string]struct{}{
	"Backups.backupdb": struct{}{},
	"mach_kernel":      struct{}{},
	".DS_Store":        struct{}{},
	".hidden":          struct{}{},
	"._.":              struct{}{},
}

func (d *directory) Lookup(name string, _ fs.Intr) (fs.Node, fuse.Error) {
	if _, ignore := ignoreNames[name]; !ignore && !strings.HasPrefix(name, "._") {
		log.Printf("directory.Lookup(name=%s, path=%s)", name, d.fullpath())
		if err := d.populate(); err != nil {
			return nil, fuse.ENOENT
		}
		if de, ok := d.de[name]; ok {
			switch de.Type {
			case fuse.DT_Dir:
				return NewDirectory(d, d.fs, name), nil
			case fuse.DT_File:
				return NewFile(d, d.fs, name)
			default:
				panic("should not happen.")
			}
		}
	}
	return nil, fuse.ENOENT
}

func ParseTime(n *html.Node) (t time.Time) {
	if ts := prepareTimeString(n.Data); ts != "" {
		var err error
		t, err = time.Parse("2-Jan-2006 15:04", ts)
		if err != nil {
			log.Printf("ParseTime('%s'): %s", ts, err)
		}
	}
	return
}

func prepareTimeString(ts string) string {
	return strings.Trim(strings.Join(strings.SplitN(
		strings.Trim(ts, "\t "), " ", 3)[0:2], " "), "\r\n\t ")
}
<commit_msg>Reset (directory).de after 30 seconds.<commit_after>package main

import (
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	"code.google.com/p/go.net/html"
)

type directory struct {
	node
	de map[string]dirent
	mu sync.Mutex
}

type dirent struct {
	fuse.Dirent
	mtime time.Time
}

func (d *directory) Attr() fuse.Attr {
	return fuse.Attr{
		Mode:  os.ModeDir | 0500,
		Uid:   MyUID,
		Gid:   MyGID,
		Nlink: 1,
		Mtime: d.mtime(),
	}
}

func NewDirectory(parent *directory, fs *filesystem, name string) *directory {
	return &directory{
		node: node{parent, fs, name},
		de:   map[string]dirent{},
	}
}

func (d *directory) populate() (err fuse.Error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	if len(d.de) > 0 {
		return nil
	}
	r, err := d.fs.client.Do(d.fs.NewRequest("GET", d.fullpath()+"/"))
	if err != nil {
		log.Println("ERROR:", err)
		return err
	}
	log.Println(d.fullpath(), r.Request.URL, r.Status)
	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		switch r.StatusCode {
		case http.StatusForbidden, http.StatusUnauthorized:
			err = fuse.EPERM
		default:
			err = fuse.ENOENT
		}
	}

	switch strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] {
	case "text/html":
		doc, err := html.Parse(r.Body)
		if err != nil {
			return err
		}
		var walk func(*html.Node)
		walk = func(n *html.Node) {
			if n.Type == html.ElementNode && n.Data == "a" {
				for _, a := range n.Attr {
					if a.Key == "href" {
						direntType := fuse.DT_File
						if a.Val[len(a.Val)-1] == '/' {
							direntType = fuse.DT_Dir
						}
						name, err := url.QueryUnescape(a.Val)
						if err != nil {
							log.Printf("url.QueryUnescape(%s): %s", a.Val, err)
						}
						name = strings.TrimRight(name, "/")
						d.de[name] = dirent{
							fuse.Dirent{
								Name: name,
								Type: direntType,
							},
							ParseTime(n.NextSibling),
						}
						log.Println(name, direntType)
						break
					}
				}
			}
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				walk(c)
			}
		}
		walk(doc)
		time.AfterFunc(30*time.Second, func() {
			d.reset()
		})
	default:
		panic("Unsupported directory response.")
	}
	return
}

func (d *directory) ReadDir(_ fs.Intr) (de []fuse.Dirent, err fuse.Error) {
	if err = d.populate(); nil == err {
		de = make([]fuse.Dirent, len(d.de))
		i := 0
		for _, v := range d.de {
			de[i] = v.Dirent
			i++
		}
	}
	return
}

var ignoreNames = map[string]struct{}{
	"Backups.backupdb": struct{}{},
	"mach_kernel":      struct{}{},
	".DS_Store":        struct{}{},
	".hidden":          struct{}{},
	"._.":              struct{}{},
}

func (d *directory) Lookup(name string, _ fs.Intr) (fs.Node, fuse.Error) {
	if _, ignore := ignoreNames[name]; !ignore && !strings.HasPrefix(name, "._") {
		log.Printf("directory.Lookup(name=%s, path=%s)", name, d.fullpath())
		if err := d.populate(); err != nil {
			return nil, fuse.ENOENT
		}
		if de, ok := d.de[name]; ok {
			switch de.Type {
			case fuse.DT_Dir:
				return NewDirectory(d, d.fs, name), nil
			case fuse.DT_File:
				return NewFile(d, d.fs, name)
			default:
				panic("should not happen.")
			}
		}
	}
	return nil, fuse.ENOENT
}

func (d *directory) reset() {
	d.mu.Lock()
	size := len(d.de)
	log.Printf("directory.reset(%s): %d children", d.fullpath(), size)
	d.de = make(map[string]dirent, size)
	d.mu.Unlock()
}

func ParseTime(n *html.Node) (t time.Time) {
	if ts := prepareTimeString(n.Data); ts != "" {
		var err error
		t, err = time.Parse("2-Jan-2006 15:04", ts)
		if err != nil {
			log.Printf("ParseTime('%s'): %s", ts, err)
		}
	}
	return
}

func prepareTimeString(ts string) string {
	return strings.Trim(strings.Join(strings.SplitN(
		strings.Trim(ts, "\t "), " ", 3)[0:2], " "), "\r\n\t ")
}
<|endoftext|>


<commit_before>package repo

import (
	"os"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer string, sequence string) (string, error) {
	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash := string(md5.Sum(data))

	err = ioutils.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes)
}
<commit_msg>[repo] fix errors<commit_after>package repo

import (
	"os"
	"fmt"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer string, sequence string) (string, error) {
	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash_bytes := md5.Sum(data)
	hash := fmt.Sprintf("%s", hash_bytes)

	err = ioutil.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes), nil
}
<|endoftext|>


<commit_before>// Mgmt
// Copyright (C) 2013-2016+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"log"
)

//go:generate stringer -type=resState -output=resstate_stringer.go
type resState int

const (
	resStateNil resState = iota
	resStateWatching
	resStateEvent // an event has happened, but we haven't poked yet
	resStateCheckApply
	resStatePoking
)

// ResUUID is a unique identifier for a resource, namely it's name, and the kind ("type").
type ResUUID interface {
	GetName() string
	Kind() string
	IFF(ResUUID) bool

	Reversed() bool // true means this resource happens before the generator
}

// The BaseUUID struct is used to provide a unique resource identifier.
type BaseUUID struct {
	name string // name and kind are the values of where this is coming from
	kind string

	reversed *bool // piggyback edge information here
}

// The AutoEdge interface is used to implement the autoedges feature.
type AutoEdge interface {
	Next() []ResUUID  // call to get list of edges to add
	Test([]bool) bool // call until false
}

// MetaParams is a struct will all params that apply to every resource.
type MetaParams struct {
	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges? // XXX should default to true
	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
	Noop      bool `yaml:"noop"`
	// NOTE: there are separate Watch and CheckApply retry and delay values,
	// but I've decided to use the same ones for both until there's a proper
	// reason to want to do something differently for the Watch errors.
	Retry int16  `yaml:"retry"` // metaparam, number of times to retry on error. -1 for infinite
	Delay uint64 `yaml:"delay"` // metaparam, number of milliseconds to wait between retries
}

// The Base interface is everything that is common to all resources.
// Everything here only needs to be implemented once, in the BaseRes.
type Base interface {
	GetName() string // can't be named "Name()" because of struct field
	SetName(string)
	setKind(string)
	Kind() string
	Meta() *MetaParams
	Events() chan Event
	AssociateData(Converger)
	IsWatching() bool
	SetWatching(bool)
	GetState() resState
	SetState(resState)
	DoSend(chan Event, string) (bool, error)
	SendEvent(eventName, bool, bool) bool
	ReadEvent(*Event) (bool, bool) // TODO: optional here?
	GroupCmp(Res) bool             // TODO: is there a better name for this?
	GroupRes(Res) error            // group resource (arg) into self
	IsGrouped() bool               // am I grouped?
	SetGrouped(bool)               // set grouped bool
	GetGroup() []Res               // return everyone grouped inside me
	SetGroup([]Res)
}

// Res is the minimum interface you need to implement to define a new resource.
type Res interface {
	Base // include everything from the Base interface
	Init()
	//Validate() bool    // TODO: this might one day be added
	GetUUIDs() []ResUUID    // most resources only return one
	Watch(chan Event) error // send on channel to signal process() events
	CheckApply(bool) (bool, error)
	AutoEdges() AutoEdge
	Compare(Res) bool
	CollectPattern(string) // XXX: temporary until Res collection is more advanced
}

// BaseRes is the base struct that gets used in every resource.
type BaseRes struct {
	Name       string     `yaml:"name"`
	MetaParams MetaParams `yaml:"meta"` // struct of all the metaparams
	kind       string
	events     chan Event
	converger  Converger // converged tracking
	state      resState
	watching   bool  // is Watch() loop running ?
	isStateOK  bool  // whether the state is okay based on events or not
	isGrouped  bool  // am i contained within a group?
	grouped    []Res // list of any grouped resources
}

// UUIDExistsInUUIDs wraps the IFF method when used with a list of UUID's.
func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
	for _, u := range uuids {
		if uuid.IFF(u) {
			return true
		}
	}
	return false
}

// GetName returns the name of the resource.
func (obj *BaseUUID) GetName() string {
	return obj.name
}

// Kind returns the kind of resource.
func (obj *BaseUUID) Kind() string {
	return obj.kind
}

// IFF looks at two UUID's and if and only if they are equivalent, returns true.
// If they are not equivalent, it returns false.
// Most resources will want to override this method, since it does the important
// work of actually discerning if two resources are identical in function.
func (obj *BaseUUID) IFF(uuid ResUUID) bool {
	res, ok := uuid.(*BaseUUID)
	if !ok {
		return false
	}
	return obj.name == res.name
}

// Reversed is part of the ResUUID interface, and true means this resource
// happens before the generator.
func (obj *BaseUUID) Reversed() bool {
	if obj.reversed == nil {
		log.Fatal("Programming error!")
	}
	return *obj.reversed
}

// Init initializes structures like channels if created without New constructor.
func (obj *BaseRes) Init() {
	obj.events = make(chan Event) // unbuffered chan size to avoid stale events
}

// GetName is used by all the resources to Get the name.
func (obj *BaseRes) GetName() string {
	return obj.Name
}

// SetName is used to set the name of the resource.
func (obj *BaseRes) SetName(name string) {
	obj.Name = name
}

// setKind sets the kind. This is used internally for exported resources.
func (obj *BaseRes) setKind(kind string) {
	obj.kind = kind
}

// Kind returns the kind of resource this is.
func (obj *BaseRes) Kind() string {
	return obj.kind
}

// Meta returns the MetaParams as a reference, which we can then get/set on.
func (obj *BaseRes) Meta() *MetaParams {
	return &obj.MetaParams
}

// Events returns the channel of events to listen on.
func (obj *BaseRes) Events() chan Event {
	return obj.events
}

// AssociateData associates some data with the object in question.
func (obj *BaseRes) AssociateData(converger Converger) {
	obj.converger = converger
}

// IsWatching tells us if the Watch() function is running.
func (obj *BaseRes) IsWatching() bool {
	return obj.watching
}

// SetWatching stores the status of if the Watch() function is running.
func (obj *BaseRes) SetWatching(b bool) {
	obj.watching = b
}

// GetState returns the state of the resource.
func (obj *BaseRes) GetState() resState {
	return obj.state
}

// SetState sets the state of the resource.
func (obj *BaseRes) SetState(state resState) {
	if DEBUG {
		log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
	}
	obj.state = state
}

// DoSend sends off an event, but doesn't block the incoming event queue. It can
// also recursively call itself when events need processing during the wait.
// I'm not completely comfortable with this fn, but it will have to do for now.
func (obj *BaseRes) DoSend(processChan chan Event, comment string) (bool, error) {
	resp := NewResp()
	processChan <- Event{eventNil, resp, comment, true} // trigger process
	select {
	case e := <-resp: // wait for the ACK()
		if e != nil { // we got a NACK
			return true, e // exit with error
		}

	case event := <-obj.events:
		// NOTE: this code should match the similar code below!
		//cuuid.SetConverged(false) // TODO ?
		if exit, send := obj.ReadEvent(&event); exit {
			return true, nil // exit, without error
		} else if send {
			return obj.DoSend(processChan, comment) // recurse
		}
	}
	return false, nil // return, no error or exit signal
}

// SendEvent pushes an event into the message queue for a particular vertex
func (obj *BaseRes) SendEvent(event eventName, sync bool, activity bool) bool {
	// TODO: isn't this race-y ?
	if !obj.IsWatching() { // element has already exited
		return false // if we don't return, we'll block on the send
	}
	if !sync {
		obj.events <- Event{event, nil, "", activity}
		return true
	}

	resp := NewResp()
	obj.events <- Event{event, resp, "", activity}
	resp.ACKWait() // waits until true (nil) value
	return true
}

// ReadEvent processes events when a select gets one, and handles the pause
// code too! The return values specify if we should exit and poke respectively.
func (obj *BaseRes) ReadEvent(event *Event) (exit, poke bool) {
	event.ACK()
	switch event.Name {
	case eventStart:
		return false, true

	case eventPoke:
		return false, true

	case eventBackPoke:
		return false, true // forward poking in response to a back poke!

	case eventExit:
		return true, false

	case eventPause:
		// wait for next event to continue
		select {
		case e := <-obj.events:
			e.ACK()
			if e.Name == eventExit {
				return true, false
			} else if e.Name == eventStart { // eventContinue
				return false, false // don't poke on unpause!
			} else {
				// if we get a poke event here, it's a bug!
				log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
			}
		}

	default:
		log.Fatal("Unknown event: ", event)
	}
	return true, false // required to keep the stupid go compiler happy
}

// GroupCmp compares two resources and decides if they're suitable for grouping
// You'll probably want to override this method when implementing a resource...
func (obj *BaseRes) GroupCmp(res Res) bool {
	return false // base implementation assumes false, override me!
}

// GroupRes groups resource (arg) into self.
func (obj *BaseRes) GroupRes(res Res) error {
	if l := len(res.GetGroup()); l > 0 {
		return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
	}
	if res.IsGrouped() {
		return fmt.Errorf("Res: %v is already grouped!", res)
	}

	obj.grouped = append(obj.grouped, res)
	res.SetGrouped(true) // i am contained _in_ a group
	return nil
}

// IsGrouped determines if we are grouped.
func (obj *BaseRes) IsGrouped() bool { // am I grouped?
	return obj.isGrouped
}

// SetGrouped sets a flag to tell if we are grouped.
func (obj *BaseRes) SetGrouped(b bool) {
	obj.isGrouped = b
}

// GetGroup returns everyone grouped inside me.
func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
	return obj.grouped
}

// SetGroup sets the grouped resources into me.
func (obj *BaseRes) SetGroup(g []Res) {
	obj.grouped = g
}

// Compare is the base compare method, which also handles the metaparams cmp
func (obj *BaseRes) Compare(res Res) bool {
	if obj.Meta().Noop != res.Meta().Noop {
		// obj is the existing res, res is the *new* resource
		// if we go from no-noop -> noop, we can re-use the obj
		// if we go from noop -> no-noop, we need to regenerate
		if obj.Meta().Noop { // asymmetrical
			return false // going from noop to no-noop!
		}
	}
	return true
}

// CollectPattern is used for resource collection.
func (obj *BaseRes) CollectPattern(pattern string) {
	// XXX: default method is empty
}

// ResToB64 encodes a resource to a base64 encoded string (after serialization)
func ResToB64(res Res) (string, error) {
	b := bytes.Buffer{}
	e := gob.NewEncoder(&b)
	err := e.Encode(&res) // pass with &
	if err != nil {
		return "", fmt.Errorf("Gob failed to encode: %v", err)
	}
	return base64.StdEncoding.EncodeToString(b.Bytes()), nil
}

// B64ToRes decodes a resource from a base64 encoded string (after deserialization)
func B64ToRes(str string) (Res, error) {
	var output interface{}
	bb, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return nil, fmt.Errorf("Base64 failed to decode: %v", err)
	}
	b := bytes.NewBuffer(bb)
	d := gob.NewDecoder(b)
	err = d.Decode(&output) // pass with &
	if err != nil {
		return nil, fmt.Errorf("Gob failed to decode: %v", err)
	}
	res, ok := output.(Res)
	if !ok {
		return nil, fmt.Errorf("Output %v is not a Res", res)

	}
	return res, nil
}
<commit_msg>resources: Compare Delay and Retry metaparams on graph change<commit_after>// Mgmt
// Copyright (C) 2013-2016+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"log"
)

//go:generate stringer -type=resState -output=resstate_stringer.go
type resState int

const (
	resStateNil resState = iota
	resStateWatching
	resStateEvent // an event has happened, but we haven't poked yet
	resStateCheckApply
	resStatePoking
)

// ResUUID is a unique identifier for a resource, namely it's name, and the kind ("type").
type ResUUID interface {
	GetName() string
	Kind() string
	IFF(ResUUID) bool

	Reversed() bool // true means this resource happens before the generator
}

// The BaseUUID struct is used to provide a unique resource identifier.
type BaseUUID struct {
	name string // name and kind are the values of where this is coming from
	kind string

	reversed *bool // piggyback edge information here
}

// The AutoEdge interface is used to implement the autoedges feature.
type AutoEdge interface {
	Next() []ResUUID  // call to get list of edges to add
	Test([]bool) bool // call until false
}

// MetaParams is a struct will all params that apply to every resource.
type MetaParams struct {
	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges? // XXX should default to true
	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
	Noop      bool `yaml:"noop"`
	// NOTE: there are separate Watch and CheckApply retry and delay values,
	// but I've decided to use the same ones for both until there's a proper
	// reason to want to do something differently for the Watch errors.
	Retry int16  `yaml:"retry"` // metaparam, number of times to retry on error. -1 for infinite
	Delay uint64 `yaml:"delay"` // metaparam, number of milliseconds to wait between retries
}

// The Base interface is everything that is common to all resources.
// Everything here only needs to be implemented once, in the BaseRes.
type Base interface {
	GetName() string // can't be named "Name()" because of struct field
	SetName(string)
	setKind(string)
	Kind() string
	Meta() *MetaParams
	Events() chan Event
	AssociateData(Converger)
	IsWatching() bool
	SetWatching(bool)
	GetState() resState
	SetState(resState)
	DoSend(chan Event, string) (bool, error)
	SendEvent(eventName, bool, bool) bool
	ReadEvent(*Event) (bool, bool) // TODO: optional here?
	GroupCmp(Res) bool             // TODO: is there a better name for this?
	GroupRes(Res) error            // group resource (arg) into self
	IsGrouped() bool               // am I grouped?
	SetGrouped(bool)               // set grouped bool
	GetGroup() []Res               // return everyone grouped inside me
	SetGroup([]Res)
}

// Res is the minimum interface you need to implement to define a new resource.
type Res interface {
	Base // include everything from the Base interface
	Init()
	//Validate() bool    // TODO: this might one day be added
	GetUUIDs() []ResUUID    // most resources only return one
	Watch(chan Event) error // send on channel to signal process() events
	CheckApply(bool) (bool, error)
	AutoEdges() AutoEdge
	Compare(Res) bool
	CollectPattern(string) // XXX: temporary until Res collection is more advanced
}

// BaseRes is the base struct that gets used in every resource.
type BaseRes struct {
	Name       string     `yaml:"name"`
	MetaParams MetaParams `yaml:"meta"` // struct of all the metaparams
	kind       string
	events     chan Event
	converger  Converger // converged tracking
	state      resState
	watching   bool  // is Watch() loop running ?
	isStateOK  bool  // whether the state is okay based on events or not
	isGrouped  bool  // am i contained within a group?
	grouped    []Res // list of any grouped resources
}

// UUIDExistsInUUIDs wraps the IFF method when used with a list of UUID's.
func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
	for _, u := range uuids {
		if uuid.IFF(u) {
			return true
		}
	}
	return false
}

// GetName returns the name of the resource.
func (obj *BaseUUID) GetName() string {
	return obj.name
}

// Kind returns the kind of resource.
func (obj *BaseUUID) Kind() string {
	return obj.kind
}

// IFF looks at two UUID's and if and only if they are equivalent, returns true.
// If they are not equivalent, it returns false.
// Most resources will want to override this method, since it does the important
// work of actually discerning if two resources are identical in function.
func (obj *BaseUUID) IFF(uuid ResUUID) bool {
	res, ok := uuid.(*BaseUUID)
	if !ok {
		return false
	}
	return obj.name == res.name
}

// Reversed is part of the ResUUID interface, and true means this resource
// happens before the generator.
func (obj *BaseUUID) Reversed() bool {
	if obj.reversed == nil {
		log.Fatal("Programming error!")
	}
	return *obj.reversed
}

// Init initializes structures like channels if created without New constructor.
func (obj *BaseRes) Init() {
	obj.events = make(chan Event) // unbuffered chan size to avoid stale events
}

// GetName is used by all the resources to Get the name.
func (obj *BaseRes) GetName() string {
	return obj.Name
}

// SetName is used to set the name of the resource.
func (obj *BaseRes) SetName(name string) {
	obj.Name = name
}

// setKind sets the kind. This is used internally for exported resources.
func (obj *BaseRes) setKind(kind string) {
	obj.kind = kind
}

// Kind returns the kind of resource this is.
func (obj *BaseRes) Kind() string {
	return obj.kind
}

// Meta returns the MetaParams as a reference, which we can then get/set on.
func (obj *BaseRes) Meta() *MetaParams {
	return &obj.MetaParams
}

// Events returns the channel of events to listen on.
func (obj *BaseRes) Events() chan Event {
	return obj.events
}

// AssociateData associates some data with the object in question.
func (obj *BaseRes) AssociateData(converger Converger) {
	obj.converger = converger
}

// IsWatching tells us if the Watch() function is running.
func (obj *BaseRes) IsWatching() bool {
	return obj.watching
}

// SetWatching stores the status of if the Watch() function is running.
func (obj *BaseRes) SetWatching(b bool) {
	obj.watching = b
}

// GetState returns the state of the resource.
func (obj *BaseRes) GetState() resState {
	return obj.state
}

// SetState sets the state of the resource.
func (obj *BaseRes) SetState(state resState) {
	if DEBUG {
		log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
	}
	obj.state = state
}

// DoSend sends off an event, but doesn't block the incoming event queue. It can
// also recursively call itself when events need processing during the wait.
// I'm not completely comfortable with this fn, but it will have to do for now.
func (obj *BaseRes) DoSend(processChan chan Event, comment string) (bool, error) {
	resp := NewResp()
	processChan <- Event{eventNil, resp, comment, true} // trigger process
	select {
	case e := <-resp: // wait for the ACK()
		if e != nil { // we got a NACK
			return true, e // exit with error
		}

	case event := <-obj.events:
		// NOTE: this code should match the similar code below!
		//cuuid.SetConverged(false) // TODO ?
		if exit, send := obj.ReadEvent(&event); exit {
			return true, nil // exit, without error
		} else if send {
			return obj.DoSend(processChan, comment) // recurse
		}
	}
	return false, nil // return, no error or exit signal
}

// SendEvent pushes an event into the message queue for a particular vertex
func (obj *BaseRes) SendEvent(event eventName, sync bool, activity bool) bool {
	// TODO: isn't this race-y ?
	if !obj.IsWatching() { // element has already exited
		return false // if we don't return, we'll block on the send
	}
	if !sync {
		obj.events <- Event{event, nil, "", activity}
		return true
	}

	resp := NewResp()
	obj.events <- Event{event, resp, "", activity}
	resp.ACKWait() // waits until true (nil) value
	return true
}

// ReadEvent processes events when a select gets one, and handles the pause
// code too! The return values specify if we should exit and poke respectively.
func (obj *BaseRes) ReadEvent(event *Event) (exit, poke bool) {
	event.ACK()
	switch event.Name {
	case eventStart:
		return false, true

	case eventPoke:
		return false, true

	case eventBackPoke:
		return false, true // forward poking in response to a back poke!

	case eventExit:
		return true, false

	case eventPause:
		// wait for next event to continue
		select {
		case e := <-obj.events:
			e.ACK()
			if e.Name == eventExit {
				return true, false
			} else if e.Name == eventStart { // eventContinue
				return false, false // don't poke on unpause!
			} else {
				// if we get a poke event here, it's a bug!
				log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
			}
		}

	default:
		log.Fatal("Unknown event: ", event)
	}
	return true, false // required to keep the stupid go compiler happy
}

// GroupCmp compares two resources and decides if they're suitable for grouping
// You'll probably want to override this method when implementing a resource...
func (obj *BaseRes) GroupCmp(res Res) bool {
	return false // base implementation assumes false, override me!
}

// GroupRes groups resource (arg) into self.
func (obj *BaseRes) GroupRes(res Res) error {
	if l := len(res.GetGroup()); l > 0 {
		return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
	}
	if res.IsGrouped() {
		return fmt.Errorf("Res: %v is already grouped!", res)
	}

	obj.grouped = append(obj.grouped, res)
	res.SetGrouped(true) // i am contained _in_ a group
	return nil
}

// IsGrouped determines if we are grouped.
func (obj *BaseRes) IsGrouped() bool { // am I grouped?
	return obj.isGrouped
}

// SetGrouped sets a flag to tell if we are grouped.
func (obj *BaseRes) SetGrouped(b bool) {
	obj.isGrouped = b
}

// GetGroup returns everyone grouped inside me.
func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
	return obj.grouped
}

// SetGroup sets the grouped resources into me.
func (obj *BaseRes) SetGroup(g []Res) {
	obj.grouped = g
}

// Compare is the base compare method, which also handles the metaparams cmp
func (obj *BaseRes) Compare(res Res) bool {
	// TODO: should the AutoEdge values be compared?
	if obj.Meta().AutoEdge != res.Meta().AutoEdge {
		return false
	}
	if obj.Meta().AutoGroup != res.Meta().AutoGroup {
		return false
	}
	if obj.Meta().Noop != res.Meta().Noop {
		// obj is the existing res, res is the *new* resource
		// if we go from no-noop -> noop, we can re-use the obj
		// if we go from noop -> no-noop, we need to regenerate
		if obj.Meta().Noop { // asymmetrical
			return false // going from noop to no-noop!
		}
	}
	if obj.Meta().Retry != res.Meta().Retry {
		return false
	}
	if obj.Meta().Delay != res.Meta().Delay {
		return false
	}
	return true
}

// CollectPattern is used for resource collection.
func (obj *BaseRes) CollectPattern(pattern string) {
	// XXX: default method is empty
}

// ResToB64 encodes a resource to a base64 encoded string (after serialization)
func ResToB64(res Res) (string, error) {
	b := bytes.Buffer{}
	e := gob.NewEncoder(&b)
	err := e.Encode(&res) // pass with &
	if err != nil {
		return "", fmt.Errorf("Gob failed to encode: %v", err)
	}
	return base64.StdEncoding.EncodeToString(b.Bytes()), nil
}

// B64ToRes decodes a resource from a base64 encoded string (after deserialization)
func B64ToRes(str string) (Res, error) {
	var output interface{}
	bb, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return nil, fmt.Errorf("Base64 failed to decode: %v", err)
	}
	b := bytes.NewBuffer(bb)
	d := gob.NewDecoder(b)
	err = d.Decode(&output) // pass with &
	if err != nil {
		return nil, fmt.Errorf("Gob failed to decode: %v", err)
	}
	res, ok := output.(Res)
	if !ok {
		return nil, fmt.Errorf("Output %v is not a Res", res)

	}
	return res, nil
}
<|endoftext|>


<commit_before>package cpp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type lexerState struct {
	pos  FilePos
	brdr *bufio.Reader
	//At the beginning on line not including whitespace
	bol bool
	//If this channel recieves a value, the lexing goroutines should close
	//its output channel and its error channel and terminate.
	cancel chan struct{}
	stream chan *Token
}

type breakout struct {
}

//Lex starts a goroutine which lexes the contents of the reader.
//fname is used for error messages when showing the source location.
//No preprocessing is done, this is just pure reading of the unprocessed
//source file.
//The goroutine will not stop until all tokens are read
func Lex(fname string, r io.Reader) chan *Token {
	ls := new(lexerState)
	ls.pos.File = fname
	ls.pos.Line = 1
	ls.pos.Col = 1
	ls.stream = make(chan *Token)
	ls.brdr = bufio.NewReader(r)
	ls.bol = true
	go ls.lex()
	return ls.stream
}

func (ls *lexerState) lexError(e error) {
	eWithPos := fmt.Sprintf("Error while reading %s. %s", ls.pos, e.Error())
	ls.sendTok(ERROR, eWithPos)
	close(ls.stream)
	//recover exits the lexer cleanly
	panic(&breakout{})
}

func (ls *lexerState) lex() {

	//This recovery happens if lexError is called.
	defer func() {
		//XXX is this correct way to retrigger non breakout?
		if e := recover(); e != nil {
			_ = e.(breakout) // Will re-panic if not a parse error.
		}

	}()

	first, _, err := ls.brdr.ReadRune()

	for {
		if err == io.EOF {
			break
		}
		if err != nil {
			ls.lexError(err)
		}
		switch {
		case isAlpha(first) || first == '_':
			ls.brdr.UnreadRune()
			ls.readIdentOrKeyword()
		case isNumeric(first):
			ls.brdr.UnreadRune()
			ls.readConstantInt()
		case isWhiteSpace(first):
			ls.brdr.UnreadRune()
			ls.skipWhiteSpace()
		default:
			switch first {
			case '#':
				ls.readDirective()
			case '!':
				ls.sendTok(NOT, "!")
			case '?':
				ls.sendTok('?', "?")
			case ':':
				ls.sendTok(COLON, ":")
			case '"':
				ls.brdr.UnreadRune()
				ls.readCString()
			case '(':
				ls.sendTok(LPAREN, "(")
			case ')':
				ls.sendTok(RPAREN, ")")
			case '{':
				ls.sendTok(LBRACE, "{")
			case '}':
				ls.sendTok(RBRACE, "}")
			case '[':
				ls.sendTok(LBRACK, "[")
			case ']':
				ls.sendTok(RBRACE, "]")
			case '<':
				ls.sendTok(LSS, "<")
			case '>':
				ls.sendTok(GTR, ">")
			case '+':
				second, _, _ := ls.brdr.ReadRune()
				if second == '+' {
					ls.sendTok(INC, "++")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ADD, "+")
			case '.':
				ls.sendTok(PERIOD, ".")
			case '-':
				second, _, _ := ls.brdr.ReadRune()
				if second == '>' {
					ls.sendTok(ARROW, "->")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(SUB, "-")
			case ',':
				ls.sendTok(COMMA, ",")
			case '*':
				ls.sendTok(MUL, "*")
			case '/':
				second, _, _ := ls.brdr.ReadRune()
				if second == '*' { // C comment
					for {
						endChar, _, err := ls.brdr.ReadRune()
						if err == io.EOF {
							ls.lexError(fmt.Errorf("Unclosed comment."))
						}
						if err != nil {
							ls.lexError(err)
						}
						if endChar == '*' {
							closeBar, _, _ := ls.brdr.ReadRune()
							if closeBar == '/' {
								break
							}
						}
					}
				} else {
					ls.sendTok(QUO, "/")
				}
			case '|':
				second, _, _ := ls.brdr.ReadRune()
				if second == '|' {
					ls.sendTok(LOR, "||")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(OR, "|")
			case '&':
				second, _, _ := ls.brdr.ReadRune()
				if second == '&' {
					ls.sendTok(LAND, "&&")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(AND, "&")
			case '=':
				second, _, _ := ls.brdr.ReadRune()
				if second == '=' {
					ls.sendTok(EQL, "==")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ASSIGN, "=")
			case ';':
				ls.sendTok(SEMICOLON, ";")
			default:
				//ls.lexError(fmt.Errorf("Internal Error - bad char code '%d'", first))
			}
		}
		first, _, err = ls.brdr.ReadRune()
	}
	close(ls.stream)
}

func (ls *lexerState) sendTok(kind TokenKind, val string) {
	var tok Token
	tok.Kind = kind
	tok.Val = val
	tok.Pos.Line = ls.pos.Line
	tok.Pos.Col = ls.pos.Col
	tok.Pos.File = ls.pos.File
	ls.bol = false
	ls.stream <- &tok
}

func (ls *lexerState) readDirective() {

	if !ls.isAtLineStart() {
		ls.lexError(fmt.Errorf("Directive not at beginning of line."))
	}

	//Skip
	for {
		ws, _, err := ls.brdr.ReadRune()
		if err != nil {
			ls.lexError(err)
		}

		if ws == ' ' || ws == '\t' {
			continue
		}

		if ws == '\n' {
			ls.lexError(fmt.Errorf("Empty directive"))
		}

		break
	}

}

func (ls *lexerState) readIdentOrKeyword() {
	var buff bytes.Buffer
	first, _, _ := ls.brdr.ReadRune()
	if !isValidIdentStart(first) {
		panic("internal error")
	}
	buff.WriteRune(first)
	for {
		b, _, err := ls.brdr.ReadRune()
		if isAlphaNumeric(b) || b == '_' {
			buff.WriteRune(b)
		} else {
			if err == nil {
				ls.brdr.UnreadByte()
			}
			str := buff.String()
			tokType, ok := keywordLUT[str]
			if !ok {
				tokType = IDENT
			}

			ls.sendTok(tokType, str)
			ls.pos.Col += len(str)
			break
		}
	}
}

func (ls *lexerState) skipWhiteSpace() {
	for {
		r, _, err := ls.brdr.ReadRune()
		if !isWhiteSpace(r) {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			break
		}
		if r == '\n' {
			ls.pos.Line += 1
			ls.pos.Col = 1
		} else {
			ls.pos.Col += 1
		}
	}
}

func (ls *lexerState) readConstantInt() {
	var buff bytes.Buffer
	for {
		r, _, err := ls.brdr.ReadRune()
		if isNumeric(r) {
			buff.WriteRune(r)
		} else {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			str := buff.String()
			ls.sendTok(INT_CONSTANT, str)
			ls.pos.Col += len(str)
			break
		}
	}
}

func (ls *lexerState) readCString() {
	first, _, err := ls.brdr.ReadRune()

	if err != nil {
		ls.lexError(err)
	}

	if first != '"' {
		ls.lexError(fmt.Errorf("internal error"))
	}

	escaped := false
	for {
		b, _, err := ls.brdr.ReadRune()
		if err == io.EOF {
			ls.lexError(fmt.Errorf("Unterminated string literal."))
		}
		if err != nil {
			ls.lexError(err)
		}
		if b == '"' && !escaped {
			break
		}
		if !escaped {
			if b == '\\' {
				escaped = true
			}
		} else {
			escaped = false
		}
	}
	ls.sendTok(STRING, "")
}

func (ls *lexerState) isAtLineStart() bool {
	return ls.bol
}

func isValidIdentStart(b rune) bool {
	return b == '_' || isAlpha(b)
}

func isAlpha(b rune) bool {
	if b >= 'a' && b <= 'z' {
		return true
	}
	if b >= 'A' && b <= 'Z' {
		return true
	}
	return false
}

func isWhiteSpace(b rune) bool {
	return b == ' ' || b == '\r' || b == '\n' || b == '\t'
}

func isNumeric(b rune) bool {
	if b >= '0' && b <= '9' {
		return true
	}
	return false
}

func isAlphaNumeric(b rune) bool {
	return isNumeric(b) || isAlpha(b)
}
<commit_msg>untested c++ comment<commit_after>package cpp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type lexerState struct {
	pos  FilePos
	brdr *bufio.Reader
	//At the beginning on line not including whitespace
	bol bool
	//If this channel recieves a value, the lexing goroutines should close
	//its output channel and its error channel and terminate.
	cancel chan struct{}
	stream chan *Token
}

type breakout struct {
}

//Lex starts a goroutine which lexes the contents of the reader.
//fname is used for error messages when showing the source location.
//No preprocessing is done, this is just pure reading of the unprocessed
//source file.
//The goroutine will not stop until all tokens are read
func Lex(fname string, r io.Reader) chan *Token {
	ls := new(lexerState)
	ls.pos.File = fname
	ls.pos.Line = 1
	ls.pos.Col = 1
	ls.stream = make(chan *Token)
	ls.brdr = bufio.NewReader(r)
	ls.bol = true
	go ls.lex()
	return ls.stream
}

func (ls *lexerState) lexError(e error) {
	eWithPos := fmt.Sprintf("Error while reading %s. %s", ls.pos, e.Error())
	ls.sendTok(ERROR, eWithPos)
	close(ls.stream)
	//recover exits the lexer cleanly
	panic(&breakout{})
}

func (ls *lexerState) lex() {

	//This recovery happens if lexError is called.
	defer func() {
		//XXX is this correct way to retrigger non breakout?
		if e := recover(); e != nil {
			_ = e.(breakout) // Will re-panic if not a parse error.
		}

	}()

	first, _, err := ls.brdr.ReadRune()

	for {
		if err == io.EOF {
			break
		}
		if err != nil {
			ls.lexError(err)
		}
		switch {
		case isAlpha(first) || first == '_':
			ls.brdr.UnreadRune()
			ls.readIdentOrKeyword()
		case isNumeric(first):
			ls.brdr.UnreadRune()
			ls.readConstantInt()
		case isWhiteSpace(first):
			ls.brdr.UnreadRune()
			ls.skipWhiteSpace()
		default:
			switch first {
			case '#':
				ls.readDirective()
			case '!':
				ls.sendTok(NOT, "!")
			case '?':
				ls.sendTok('?', "?")
			case ':':
				ls.sendTok(COLON, ":")
			case '"':
				ls.brdr.UnreadRune()
				ls.readCString()
			case '(':
				ls.sendTok(LPAREN, "(")
			case ')':
				ls.sendTok(RPAREN, ")")
			case '{':
				ls.sendTok(LBRACE, "{")
			case '}':
				ls.sendTok(RBRACE, "}")
			case '[':
				ls.sendTok(LBRACK, "[")
			case ']':
				ls.sendTok(RBRACE, "]")
			case '<':
				ls.sendTok(LSS, "<")
			case '>':
				ls.sendTok(GTR, ">")
			case '+':
				second, _, _ := ls.brdr.ReadRune()
				if second == '+' {
					ls.sendTok(INC, "++")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ADD, "+")
			case '.':
				ls.sendTok(PERIOD, ".")
			case '-':
				second, _, _ := ls.brdr.ReadRune()
				if second == '>' {
					ls.sendTok(ARROW, "->")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(SUB, "-")
			case ',':
				ls.sendTok(COMMA, ",")
			case '*':
				ls.sendTok(MUL, "*")
			case '/':
				second, _, _ := ls.brdr.ReadRune()
				if second == '*' { // C comment.
					for {
						endChar, _, err := ls.brdr.ReadRune()
						if err == io.EOF {
							ls.lexError(fmt.Errorf("Unclosed comment."))
						}
						if err != nil {
							ls.lexError(err)
						}
						if endChar == '*' {
							closeBar, _, _ := ls.brdr.ReadRune()
							if closeBar == '/' {
								break
							}
						}
					}
				} else if second == '/' { // C++ comment.
					for {
						c, _, err := ls.brdr.ReadRune()
						if err != nil {
							break
						}
						if c == '\n' {
							break
						}
					}
				} else {
					ls.sendTok(QUO, "/")
				}
			case '|':
				second, _, _ := ls.brdr.ReadRune()
				if second == '|' {
					ls.sendTok(LOR, "||")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(OR, "|")
			case '&':
				second, _, _ := ls.brdr.ReadRune()
				if second == '&' {
					ls.sendTok(LAND, "&&")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(AND, "&")
			case '=':
				second, _, _ := ls.brdr.ReadRune()
				if second == '=' {
					ls.sendTok(EQL, "==")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ASSIGN, "=")
			case ';':
				ls.sendTok(SEMICOLON, ";")
			default:
				//ls.lexError(fmt.Errorf("Internal Error - bad char code '%d'", first))
			}
		}
		first, _, err = ls.brdr.ReadRune()
	}
	close(ls.stream)
}

func (ls *lexerState) sendTok(kind TokenKind, val string) {
	var tok Token
	tok.Kind = kind
	tok.Val = val
	tok.Pos.Line = ls.pos.Line
	tok.Pos.Col = ls.pos.Col
	tok.Pos.File = ls.pos.File
	ls.bol = false
	ls.stream <- &tok
}

func (ls *lexerState) readDirective() {

	if !ls.isAtLineStart() {
		ls.lexError(fmt.Errorf("Directive not at beginning of line."))
	}

	//Skip
	for {
		ws, _, err := ls.brdr.ReadRune()
		if err != nil {
			ls.lexError(err)
		}

		if ws == ' ' || ws == '\t' {
			continue
		}

		if ws == '\n' {
			ls.lexError(fmt.Errorf("Empty directive"))
		}

		break
	}

}

func (ls *lexerState) readIdentOrKeyword() {
	var buff bytes.Buffer
	first, _, _ := ls.brdr.ReadRune()
	if !isValidIdentStart(first) {
		panic("internal error")
	}
	buff.WriteRune(first)
	for {
		b, _, err := ls.brdr.ReadRune()
		if isAlphaNumeric(b) || b == '_' {
			buff.WriteRune(b)
		} else {
			if err == nil {
				ls.brdr.UnreadByte()
			}
			str := buff.String()
			tokType, ok := keywordLUT[str]
			if !ok {
				tokType = IDENT
			}

			ls.sendTok(tokType, str)
			ls.pos.Col += len(str)
			break
		}
	}
}

func (ls *lexerState) skipWhiteSpace() {
	for {
		r, _, err := ls.brdr.ReadRune()
		if !isWhiteSpace(r) {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			break
		}
		if r == '\n' {
			ls.pos.Line += 1
			ls.pos.Col = 1
		} else {
			ls.pos.Col += 1
		}
	}
}

func (ls *lexerState) readConstantInt() {
	var buff bytes.Buffer
	for {
		r, _, err := ls.brdr.ReadRune()
		if isNumeric(r) {
			buff.WriteRune(r)
		} else {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			str := buff.String()
			ls.sendTok(INT_CONSTANT, str)
			ls.pos.Col += len(str)
			break
		}
	}
}

func (ls *lexerState) readCString() {
	first, _, err := ls.brdr.ReadRune()

	if err != nil {
		ls.lexError(err)
	}

	if first != '"' {
		ls.lexError(fmt.Errorf("internal error"))
	}

	escaped := false
	for {
		b, _, err := ls.brdr.ReadRune()
		if err == io.EOF {
			ls.lexError(fmt.Errorf("Unterminated string literal."))
		}
		if err != nil {
			ls.lexError(err)
		}
		if b == '"' && !escaped {
			break
		}
		if !escaped {
			if b == '\\' {
				escaped = true
			}
		} else {
			escaped = false
		}
	}
	ls.sendTok(STRING, "")
}

func (ls *lexerState) isAtLineStart() bool {
	return ls.bol
}

func isValidIdentStart(b rune) bool {
	return b == '_' || isAlpha(b)
}

func isAlpha(b rune) bool {
	if b >= 'a' && b <= 'z' {
		return true
	}
	if b >= 'A' && b <= 'Z' {
		return true
	}
	return false
}

func isWhiteSpace(b rune) bool {
	return b == ' ' || b == '\r' || b == '\n' || b == '\t'
}

func isNumeric(b rune) bool {
	if b >= '0' && b <= '9' {
		return true
	}
	return false
}

func isAlphaNumeric(b rune) bool {
	return isNumeric(b) || isAlpha(b)
}
<|endoftext|>


<commit_before>package dlna

import (
	"fmt"
	//"math"
	"strconv"
	"strings"
	"time"
)

const (
	TimeSeekRangeDomain   = "TimeSeekRange.dlna.org"
	ContentFeaturesDomain = "contentFeatures.dlna.org"
	TransferModeDomain    = "transferMode.dlna.org"
)

type ContentFeatures struct {
	SupportTimeSeek bool
	SupportRange    bool
	Transcoded      bool
}

// flags are in hex. trailing 24 zeroes, 26 are after the space
// "DLNA.ORG_OP=" time-seek-range-supp bytes-range-header-supp

func (cf ContentFeatures) String() string {
	return fmt.Sprintf("DLNA.ORG_OP=%02b;DLNA.ORG_CI=%b;DLNA.ORG_FLAGS=017000 00000000000000000000000000000000", func() (ret uint) {
		if cf.SupportTimeSeek {
			ret |= 2
		}
		if cf.SupportRange {
			ret |= 1
		}
		return
	}(), func() uint {
		if cf.Transcoded {
			return 1
		}
		return 0
	}())
}

func ParseNPTDuration(s string) (ret time.Duration, err error) {
	ss := strings.SplitN(s, ":", 3)
	var (
		h   int64
		m   uint64
		sec float64
	)
	h, err = strconv.ParseInt(ss[0], 0, 64)
	if err != nil {
		return
	}
	m, err = strconv.ParseUint(ss[1], 0, 0)
	if err != nil {
		return
	}
	sec, err = strconv.ParseFloat(ss[2], 64)
	if err != nil {
		return
	}
	ret = time.Duration(h) * time.Hour
	ret += time.Duration(m) * time.Minute
	ret += time.Duration(sec * float64(time.Second))
	return
}

func NPTDurationAsString(d time.Duration)

type NPTRange struct {
	Start, End time.Duration
}

func ParseNPTRange(s string) (ret NPTRange, err error) {
	ss := strings.SplitN(s, "-", 2)
	if ss[0] != "" {
		ret.Start, err = ParseNPTDuration(ss[0])
		if err != nil {
			return
		}
	}
	if ss[1] != "" {
		ret.End, err = ParseNPTDuration(ss[1])
		if err != nil {
			return
		}
	}
	return
}

func (me NPTRange) String() (ret string) {
	ret = me.Start.String() + "-"
	if me.End >= 0 {
		ret += me.End.String()
	}
	return
}
<commit_msg>Tighten conformance to to NPT time specs.<commit_after>package dlna

import (
	"fmt"
	"strings"
	"time"
)

const (
	TimeSeekRangeDomain   = "TimeSeekRange.dlna.org"
	ContentFeaturesDomain = "contentFeatures.dlna.org"
	TransferModeDomain    = "transferMode.dlna.org"
)

type ContentFeatures struct {
	//ProfileName string
	SupportTimeSeek bool
	SupportRange    bool
	//play speeds, DLNA.ORG_PS
	Transcoded bool
}

// flags are in hex. trailing 24 zeroes, 26 are after the space
// "DLNA.ORG_OP=" time-seek-range-supp bytes-range-header-supp

func (cf ContentFeatures) String() string {
	//DLNA.ORG_PN=[a-zA-Z0-9_]*
	return fmt.Sprintf("DLNA.ORG_OP=%02b;DLNA.ORG_CI=%b", func() (ret uint) {
		if cf.SupportTimeSeek {
			ret |= 2
		}
		if cf.SupportRange {
			ret |= 1
		}
		return
	}(), func() uint {
		if cf.Transcoded {
			return 1
		}
		return 0
	}())
}

func ParseNPTTime(s string) (time.Duration, error) {
	var h, m, sec, ms time.Duration
	n, err := fmt.Sscanf(s, "%d:%2d:%2d.%3d", &h, &m, &sec, &ms)
	if err != nil {
		return -1, err
	}
	if n < 3 {
		return -1, fmt.Errorf("invalid npt time: %s", s)
	}
	ret := time.Duration(h) * time.Hour
	ret += time.Duration(m) * time.Minute
	ret += sec * time.Second
	ret += ms * time.Millisecond
	return ret, nil
}

func FormatNPTTime(npt time.Duration) string {
	npt /= time.Millisecond
	ms := npt % 1000
	npt /= 1000
	s := npt % 60
	npt /= 60
	m := npt % 60
	npt /= 60
	h := npt
	return fmt.Sprintf("%02d:%02d:%02d.%03d", h, m, s, ms)
}

type NPTRange struct {
	Start, End time.Duration
}

func ParseNPTRange(s string) (ret NPTRange, err error) {
	ss := strings.SplitN(s, "-", 2)
	if ss[0] != "" {
		ret.Start, err = ParseNPTTime(ss[0])
		if err != nil {
			return
		}
	}
	if ss[1] != "" {
		ret.End, err = ParseNPTTime(ss[1])
		if err != nil {
			return
		}
	}
	return
}

func (me NPTRange) String() (ret string) {
	ret = me.Start.String() + "-"
	if me.End >= 0 {
		ret += me.End.String()
	}
	return
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package searcher

import (
	"regexp"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
)

type RegexpSearcher struct {
	indexReader index.IndexReader
	pattern     *regexp.Regexp
	field       string
	explain     bool
	searcher    *DisjunctionSearcher
}

func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, field string, boost float64, explain bool) (*RegexpSearcher, error) {

	prefixTerm, complete := pattern.LiteralPrefix()
	var candidateTerms []string
	if complete {
		// there is no pattern
		candidateTerms = []string{prefixTerm}
	} else {
		var err error
		candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, prefixTerm)
		if err != nil {
			return nil, err
		}
	}

	// enumerate all the terms in the range
	qsearchers := make([]search.Searcher, 0, len(candidateTerms))
	qsearchersClose := func() {
		for _, searcher := range qsearchers {
			_ = searcher.Close()
		}
	}
	for _, cterm := range candidateTerms {
		qsearcher, err := NewTermSearcher(indexReader, cterm, field, boost, explain)
		if err != nil {
			qsearchersClose()
			return nil, err
		}
		qsearchers = append(qsearchers, qsearcher)
	}

	// build disjunction searcher of these ranges
	searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain)
	if err != nil {
		qsearchersClose()
		return nil, err
	}

	return &RegexpSearcher{
		indexReader: indexReader,
		pattern:     pattern,
		field:       field,
		explain:     explain,
		searcher:    searcher,
	}, nil
}

func findRegexpCandidateTerms(indexReader index.IndexReader, pattern *regexp.Regexp, field, prefixTerm string) (rv []string, err error) {
	rv = make([]string, 0)
	var fieldDict index.FieldDict
	if len(prefixTerm) > 0 {
		fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))
	} else {
		fieldDict, err = indexReader.FieldDict(field)
	}
	defer func() {
		if cerr := fieldDict.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()

	// enumerate the terms and check against regexp
	tfd, err := fieldDict.Next()
	for err == nil && tfd != nil {
		if pattern.MatchString(tfd.Term) {
			rv = append(rv, tfd.Term)
			if tooManyClauses(len(rv)) {
				return rv, tooManyClausesErr()
			}
		}
		tfd, err = fieldDict.Next()
	}

	return rv, err
}

func (s *RegexpSearcher) Count() uint64 {
	return s.searcher.Count()
}

func (s *RegexpSearcher) Weight() float64 {
	return s.searcher.Weight()
}

func (s *RegexpSearcher) SetQueryNorm(qnorm float64) {
	s.searcher.SetQueryNorm(qnorm)
}

func (s *RegexpSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
	return s.searcher.Next(ctx)

}

func (s *RegexpSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
	return s.searcher.Advance(ctx, ID)
}

func (s *RegexpSearcher) Close() error {
	return s.searcher.Close()
}

func (s *RegexpSearcher) Min() int {
	return 0
}

func (s *RegexpSearcher) DocumentMatchPoolSize() int {
	return s.searcher.DocumentMatchPoolSize()
}
<commit_msg>optimize NewRegexpSearcher to return its disjunction searcher<commit_after>//  Copyright (c) 2015 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package searcher

import (
	"regexp"

	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/search"
)

func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, field string, boost float64, explain bool) (search.Searcher, error) {

	prefixTerm, complete := pattern.LiteralPrefix()
	var candidateTerms []string
	if complete {
		// there is no pattern
		candidateTerms = []string{prefixTerm}
	} else {
		var err error
		candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, prefixTerm)
		if err != nil {
			return nil, err
		}
	}

	// enumerate all the terms in the range
	qsearchers := make([]search.Searcher, 0, len(candidateTerms))
	qsearchersClose := func() {
		for _, searcher := range qsearchers {
			_ = searcher.Close()
		}
	}
	for _, cterm := range candidateTerms {
		qsearcher, err := NewTermSearcher(indexReader, cterm, field, boost, explain)
		if err != nil {
			qsearchersClose()
			return nil, err
		}
		qsearchers = append(qsearchers, qsearcher)
	}

	// build disjunction searcher of these ranges
	searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain)
	if err != nil {
		qsearchersClose()
		return nil, err
	}

	return searcher, err
}

func findRegexpCandidateTerms(indexReader index.IndexReader, pattern *regexp.Regexp, field, prefixTerm string) (rv []string, err error) {
	rv = make([]string, 0)
	var fieldDict index.FieldDict
	if len(prefixTerm) > 0 {
		fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))
	} else {
		fieldDict, err = indexReader.FieldDict(field)
	}
	defer func() {
		if cerr := fieldDict.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()

	// enumerate the terms and check against regexp
	tfd, err := fieldDict.Next()
	for err == nil && tfd != nil {
		if pattern.MatchString(tfd.Term) {
			rv = append(rv, tfd.Term)
			if tooManyClauses(len(rv)) {
				return rv, tooManyClausesErr()
			}
		}
		tfd, err = fieldDict.Next()
	}

	return rv, err
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gorilla/mux"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/fs111/simpleconfig"
	godis "github.com/simonz05/godis/redis"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	// special key in redis, that is our global counter
	COUNTER = "__counter__"
	HTTP    = "http"
)

var (
	redis  *godis.Client
	config *simpleconfig.Config
	filenotfound string
)

type KurzUrl struct {
	Key          string
	ShortUrl     string
	LongUrl      string
	CreationDate int64
	Clicks       int64
}

// Converts the KurzUrl to JSON.
func (k KurzUrl) Json() []byte {
	b, _ := json.Marshal(k)
	return b
}

// Creates a new KurzUrl instance. The Given key, shorturl and longurl will
// be used. Clicks will be set to 0 and CreationDate to time.Nanoseconds()
func NewKurzUrl(key, shorturl, longurl string) *KurzUrl {
	kurl := new(KurzUrl)
	kurl.CreationDate = time.Now().UnixNano()
	kurl.Key = key
	kurl.LongUrl = longurl
	kurl.ShortUrl = shorturl
	kurl.Clicks = 0
	return kurl
}

// stores a new KurzUrl for the given key, shorturl and longurl. Existing
// ones with the same url will be overwritten
func store(key, shorturl, longurl string) *KurzUrl {
	kurl := NewKurzUrl(key, shorturl, longurl)
	go redis.Hset(kurl.Key, "LongUrl", kurl.LongUrl)
	go redis.Hset(kurl.Key, "ShortUrl", kurl.ShortUrl)
	go redis.Hset(kurl.Key, "CreationDate", kurl.CreationDate)
	go redis.Hset(kurl.Key, "Clicks", kurl.Clicks)
	return kurl
}

// loads a KurzUrl instance for the given key. If the key is
// not found, os.Error is returned.
func load(key string) (*KurzUrl, error) {
	if ok, _ := redis.Hexists(key, "ShortUrl"); ok {
		kurl := new(KurzUrl)
		kurl.Key = key
		reply, _ := redis.Hmget(key, "LongUrl", "ShortUrl", "CreationDate", "Clicks")
		kurl.LongUrl, kurl.ShortUrl, kurl.CreationDate, kurl.Clicks =
			reply.Elems[0].Elem.String(), reply.Elems[1].Elem.String(),
			reply.Elems[2].Elem.Int64(), reply.Elems[3].Elem.Int64()
		return kurl, nil
	}
	return nil, errors.New("unknown key: " + key)
}

func fileExists(dir string) bool {
	info, err := os.Stat(dir)
	if err != nil {
		return false
	}

	return !info.IsDir()
}

// function to display the info about a KurzUrl given by it's Key
func info(w http.ResponseWriter, r *http.Request) {
	short := mux.Vars(r)["short"]
	if strings.HasSuffix(short, "+") {
		short = strings.Replace(short, "+", "", 1)
	}

	kurl, err := load(short)
	if err == nil {
		w.Header().Set("Content-Type", "application/json")
		w.Write(kurl.Json())
		io.WriteString(w, "\n")
	} else {
		http.Redirect(w, r, filenotfound, http.StatusNotFound)
	}
}

// function to resolve a shorturl and redirect
func resolve(w http.ResponseWriter, r *http.Request) {

	short := mux.Vars(r)["short"]
	kurl, err := load(short)
	if err == nil {
		go redis.Hincrby(kurl.Key, "Clicks", 1)
		http.Redirect(w, r, kurl.LongUrl, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, r, filenotfound, http.StatusMovedPermanently)
	}
}

// Determines if the string rawurl is a valid URL to be stored.
func isValidUrl(rawurl string) (u *url.URL, err error) {
	if len(rawurl) == 0 {
		return nil, errors.New("empty url")
	}
	// XXX this needs some love...
	if !strings.HasPrefix(rawurl, HTTP) {
		rawurl = fmt.Sprintf("%s://%s", HTTP, rawurl)
	}
	return url.Parse(rawurl)
}

// function to shorten and store a url
func shorten(w http.ResponseWriter, r *http.Request) {
	host := config.GetStringDefault("hostname", "localhost")
	leUrl := r.FormValue("url")
	theUrl, err := isValidUrl(string(leUrl))
	if err == nil {
		ctr, _ := redis.Incr(COUNTER)
		encoded := Encode(ctr)
		location := fmt.Sprintf("%s://%s/%s", HTTP, host, encoded)
		store(encoded, location, theUrl.String())

		home := r.FormValue("home")
		if home != "" {
			http.Redirect(w, r, "/", http.StatusMovedPermanently)
		} else {
			// redirect to the info page
			http.Redirect(w, r, location+"+", http.StatusMovedPermanently)
		}
	} else {
		http.Redirect(w, r, filenotfound, http.StatusNotFound)
	}
}

//Returns a json array with information about the last shortened urls. If data
// is a valid integer, that's the amount of data it will return, otherwise
// a maximum of 10 entries will be returned.
func latest(w http.ResponseWriter, r *http.Request) {
	data := mux.Vars(r)["data"]
	howmany, err := strconv.ParseInt(data, 10, 64)
	if err != nil {
		howmany = 10
	}
	c, _ := redis.Get(COUNTER)

	last := c.Int64()
	upTo := (last - howmany)

	w.Header().Set("Content-Type", "application/json")

	var kurls = []*KurzUrl{}

	for i := last; i > upTo && i > 0; i -= 1 {
		kurl, err := load(Encode(i))
		if err == nil {
			kurls = append(kurls, kurl)
		}
	}
	s, _ := json.Marshal(kurls)
	w.Write(s)
}

func static(w http.ResponseWriter, r *http.Request) {
	fname := mux.Vars(r)["fileName"]
	// empty means, we want to serve the index file. Due to a bug in http.serveFile
	// the file cannot be called index.html, anything else is fine.
	if fname == "" {
		fname = "index.htm"
	}
	staticDir := config.GetStringDefault("static-directory", "")
	staticFile := path.Join(staticDir, fname)
	if fileExists(staticFile) {
		http.ServeFile(w, r, staticFile)
	}
}

func main() {
	flag.Parse()
	path := flag.Arg(0)

	config, _ = simpleconfig.NewConfig(path)

	host := config.GetStringDefault("redis.address", "tcp:localhost:6379")
	db := config.GetIntDefault("redis.database", 0)
	passwd := config.GetStringDefault("redis.password", "")

	filenotfound = config.GetStringDefault("filenotfound", "https://www.youtube.com/watch?v=oHg5SJYRHA0")

	redis = godis.New(host, db, passwd)

	router := mux.NewRouter()
	router.HandleFunc("/shorten/{url:(.*$)}", shorten)

	router.HandleFunc("/{short:([a-zA-Z0-9]+$)}", resolve)
	router.HandleFunc("/{short:([a-zA-Z0-9]+)\\+$}", info)
	router.HandleFunc("/info/{short:[a-zA-Z0-9]+}", info)
	router.HandleFunc("/latest/{data:[0-9]+}", latest)

	router.HandleFunc("/{fileName:(.*$)}", static)

	listen := config.GetStringDefault("listen", "0.0.0.0")
	port := config.GetStringDefault("port", "9999")
	s := &http.Server{
		Addr:    listen + ":" + port,
		Handler: router,
	}
	s.ListenAndServe()
}
<commit_msg>fix redis connection string<commit_after>package main

import (
	"code.google.com/p/gorilla/mux"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/fs111/simpleconfig"
	godis "github.com/simonz05/godis/redis"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	// special key in redis, that is our global counter
	COUNTER = "__counter__"
	HTTP    = "http"
)

var (
	redis  *godis.Client
	config *simpleconfig.Config
	filenotfound string
)

type KurzUrl struct {
	Key          string
	ShortUrl     string
	LongUrl      string
	CreationDate int64
	Clicks       int64
}

// Converts the KurzUrl to JSON.
func (k KurzUrl) Json() []byte {
	b, _ := json.Marshal(k)
	return b
}

// Creates a new KurzUrl instance. The Given key, shorturl and longurl will
// be used. Clicks will be set to 0 and CreationDate to time.Nanoseconds()
func NewKurzUrl(key, shorturl, longurl string) *KurzUrl {
	kurl := new(KurzUrl)
	kurl.CreationDate = time.Now().UnixNano()
	kurl.Key = key
	kurl.LongUrl = longurl
	kurl.ShortUrl = shorturl
	kurl.Clicks = 0
	return kurl
}

// stores a new KurzUrl for the given key, shorturl and longurl. Existing
// ones with the same url will be overwritten
func store(key, shorturl, longurl string) *KurzUrl {
	kurl := NewKurzUrl(key, shorturl, longurl)
	go redis.Hset(kurl.Key, "LongUrl", kurl.LongUrl)
	go redis.Hset(kurl.Key, "ShortUrl", kurl.ShortUrl)
	go redis.Hset(kurl.Key, "CreationDate", kurl.CreationDate)
	go redis.Hset(kurl.Key, "Clicks", kurl.Clicks)
	return kurl
}

// loads a KurzUrl instance for the given key. If the key is
// not found, os.Error is returned.
func load(key string) (*KurzUrl, error) {
	if ok, _ := redis.Hexists(key, "ShortUrl"); ok {
		kurl := new(KurzUrl)
		kurl.Key = key
		reply, _ := redis.Hmget(key, "LongUrl", "ShortUrl", "CreationDate", "Clicks")
		kurl.LongUrl, kurl.ShortUrl, kurl.CreationDate, kurl.Clicks =
			reply.Elems[0].Elem.String(), reply.Elems[1].Elem.String(),
			reply.Elems[2].Elem.Int64(), reply.Elems[3].Elem.Int64()
		return kurl, nil
	}
	return nil, errors.New("unknown key: " + key)
}

func fileExists(dir string) bool {
	info, err := os.Stat(dir)
	if err != nil {
		return false
	}

	return !info.IsDir()
}

// function to display the info about a KurzUrl given by it's Key
func info(w http.ResponseWriter, r *http.Request) {
	short := mux.Vars(r)["short"]
	if strings.HasSuffix(short, "+") {
		short = strings.Replace(short, "+", "", 1)
	}

	kurl, err := load(short)
	if err == nil {
		w.Header().Set("Content-Type", "application/json")
		w.Write(kurl.Json())
		io.WriteString(w, "\n")
	} else {
		http.Redirect(w, r, filenotfound, http.StatusNotFound)
	}
}

// function to resolve a shorturl and redirect
func resolve(w http.ResponseWriter, r *http.Request) {

	short := mux.Vars(r)["short"]
	kurl, err := load(short)
	if err == nil {
		go redis.Hincrby(kurl.Key, "Clicks", 1)
		http.Redirect(w, r, kurl.LongUrl, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, r, filenotfound, http.StatusMovedPermanently)
	}
}

// Determines if the string rawurl is a valid URL to be stored.
func isValidUrl(rawurl string) (u *url.URL, err error) {
	if len(rawurl) == 0 {
		return nil, errors.New("empty url")
	}
	// XXX this needs some love...
	if !strings.HasPrefix(rawurl, HTTP) {
		rawurl = fmt.Sprintf("%s://%s", HTTP, rawurl)
	}
	return url.Parse(rawurl)
}

// function to shorten and store a url
func shorten(w http.ResponseWriter, r *http.Request) {
	host := config.GetStringDefault("hostname", "localhost")
	leUrl := r.FormValue("url")
	theUrl, err := isValidUrl(string(leUrl))
	if err == nil {
		ctr, _ := redis.Incr(COUNTER)
		encoded := Encode(ctr)
		location := fmt.Sprintf("%s://%s/%s", HTTP, host, encoded)
		store(encoded, location, theUrl.String())

		home := r.FormValue("home")
		if home != "" {
			http.Redirect(w, r, "/", http.StatusMovedPermanently)
		} else {
			// redirect to the info page
			http.Redirect(w, r, location+"+", http.StatusMovedPermanently)
		}
	} else {
		http.Redirect(w, r, filenotfound, http.StatusNotFound)
	}
}

//Returns a json array with information about the last shortened urls. If data
// is a valid integer, that's the amount of data it will return, otherwise
// a maximum of 10 entries will be returned.
func latest(w http.ResponseWriter, r *http.Request) {
	data := mux.Vars(r)["data"]
	howmany, err := strconv.ParseInt(data, 10, 64)
	if err != nil {
		howmany = 10
	}
	c, _ := redis.Get(COUNTER)

	last := c.Int64()
	upTo := (last - howmany)

	w.Header().Set("Content-Type", "application/json")

	var kurls = []*KurzUrl{}

	for i := last; i > upTo && i > 0; i -= 1 {
		kurl, err := load(Encode(i))
		if err == nil {
			kurls = append(kurls, kurl)
		}
	}
	s, _ := json.Marshal(kurls)
	w.Write(s)
}

func static(w http.ResponseWriter, r *http.Request) {
	fname := mux.Vars(r)["fileName"]
	// empty means, we want to serve the index file. Due to a bug in http.serveFile
	// the file cannot be called index.html, anything else is fine.
	if fname == "" {
		fname = "index.htm"
	}
	staticDir := config.GetStringDefault("static-directory", "")
	staticFile := path.Join(staticDir, fname)
	if fileExists(staticFile) {
		http.ServeFile(w, r, staticFile)
	}
}

func main() {
	flag.Parse()
	path := flag.Arg(0)

	config, _ = simpleconfig.NewConfig(path)

	host := config.GetStringDefault("redis.netaddress", "tcp:localhost:6379")
	db := config.GetIntDefault("redis.database", 0)
	passwd := config.GetStringDefault("redis.password", "")

	filenotfound = config.GetStringDefault("filenotfound", "https://www.youtube.com/watch?v=oHg5SJYRHA0")

	redis = godis.New(host, db, passwd)

	router := mux.NewRouter()
	router.HandleFunc("/shorten/{url:(.*$)}", shorten)

	router.HandleFunc("/{short:([a-zA-Z0-9]+$)}", resolve)
	router.HandleFunc("/{short:([a-zA-Z0-9]+)\\+$}", info)
	router.HandleFunc("/info/{short:[a-zA-Z0-9]+}", info)
	router.HandleFunc("/latest/{data:[0-9]+}", latest)

	router.HandleFunc("/{fileName:(.*$)}", static)

	listen := config.GetStringDefault("listen", "0.0.0.0")
	port := config.GetStringDefault("port", "9999")
	s := &http.Server{
		Addr:    listen + ":" + port,
		Handler: router,
	}
	s.ListenAndServe()
}
<|endoftext|>