<commit_before>package migrate

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/marcosnils/cmt/cmd"
	"github.com/marcosnils/cmt/validate"
)

var Command = cli.Command{
	Name:  "migrate",
	Usage: "Migrate running container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "src",
			Usage: "Source host where the container is running",
		},
		cli.StringFlag{
			Name:  "dst",
			Usage: "Target host to migrate the container",
		},
		cli.BoolFlag{
			Name:  "pre-dump",
			Usage: "Perform a pre-dump to minimize downtime",
		},
		cli.StringFlag{
			Name:  "force",
			Usage: "Doesn't fail of validations related to different versions of executable binaries and cpu differences",
		},
		cli.StringFlag{
			Name:  "hook-pre-restore",
			Usage: "Command to run right before restoring process",
		},
		cli.StringFlag{
			Name:  "hook-post-restore",
			Usage: "Command to run right after a successful process restoration",
		},
		cli.StringFlag{
			Name:  "hook-failed-restore",
			Usage: "Command to run right after a failed process restoration",
		},
	},
	Action: func(c *cli.Context) {
		srcUrl := validate.ParseURL(c.String("src"))
		dstUrl := validate.ParseURL(c.String("dst"))

		log.Println("Performing validations")
		src, dst := validate.Validate(srcUrl, dstUrl, c.Bool("force"))

		log.Println("Preparing everything to do a checkpoint")
		containerId := getContainerId(srcUrl.Path)
		var imagesPath string
		var restoreCmd cmd.Cmd
		var migrateStart time.Time
		var downtime time.Duration

		if c.Bool("pre-dump") {
			// Process pre-dump
			predumpPath := fmt.Sprintf("%s/images/0", srcUrl.Path)
			prepareDir(src, predumpPath)

			checkpoint(src, containerId, predumpPath, true)

			srcTarFile := fmt.Sprintf("%s/predump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, predumpPath)

			prepareDir(dst, fmt.Sprintf("%s/images/0", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/0", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/0/predump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/0", dstUrl.Path))

			// Process final image
			migrateStart = time.Now()

			imagesPath = fmt.Sprintf("%s/images/1", srcUrl.Path)
			log.Println("Performing the checkpoint")
			_, _, err = src.Run("sudo", "runc", "--id", containerId, "checkpoint", "--image-path", imagesPath, "--prev-images-dir", predumpPath)
			if err != nil {
				log.Fatal("Error performing checkpoint:", err)
			}

			srcTarFile = fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)
			prepareDir(dst, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err = cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/1", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile = fmt.Sprintf("%s/images/1/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Performing the restore")
			TriggerHook(c.String("hook-pre-restore"))
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images/1", dstUrl.Path)

			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}
		} else {
			imagesPath = fmt.Sprintf("%s/images", srcUrl.Path)
			prepareDir(src, imagesPath)

			migrateStart = time.Now()
			checkpoint(src, containerId, imagesPath, false)

			srcTarFile := fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)

			prepareDir(dst, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Copying checkpoint image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Performing the restore")
			TriggerHook(c.String("hook-pre-restore"))
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		}

		var restoreSucceed bool
		var restoreError error
		var wg sync.WaitGroup
		wg.Add(1)

		go func() {
			restoreError = restoreCmd.Wait()
			wg.Done()
		}()

		go func() {
			log.Println("Waiting for container to start...")
			// We make a fast check so we don't wait for the first ticker internal
			if isRunning(containerId, dst) {
				restoreSucceed = true
				wg.Done()
				return
			}
			ticker := time.NewTicker(200 * time.Millisecond)
			go func() {
				for _ = range ticker.C {
					if isRunning(containerId, dst) {
						restoreSucceed = true
						break
					}

				}
				ticker.Stop()
				wg.Done()
			}()
		}()

		wg.Wait()

		downtime = time.Since(migrateStart)

		if restoreSucceed {
			log.Printf("Restore finished successfully, total downtime: %dms", downtime/time.Millisecond)
			TriggerHook(c.String("hook-post-restore"))
		} else {
			log.Println("Error performing restore:", restoreError)
			TriggerHook(c.String("hook-failed-restore"))
		}

	},
}

func isRunning(containerId string, dstCmd cmd.Cmd) bool {
	_, _, err := dstCmd.Run("stat", fmt.Sprintf("/var/run/opencontainer/containers/%s", containerId))
	if err != nil {
		return true
	}

	return false
}

func unpackTar(cmd cmd.Cmd, tarFile, workDir string) {
	log.Println("Preparing image at destination host")
	_, _, err := cmd.Run("sudo", "tar", "-C", workDir, "-xvzf", tarFile)
	if err != nil {
		log.Fatal("Error uncompressing image in destination:", err)
	}
}

func prepareTar(cmd cmd.Cmd, tarFile, workDir string) {
	_, _, err := cmd.Run("sudo", "tar", "-czf", tarFile, "-C", fmt.Sprintf("%s/", workDir), ".")
	if err != nil {
		log.Fatal("Error compressing image in source:", err)
	}
}

func checkpoint(cmd cmd.Cmd, containerId, imagesPath string, predump bool) {
	log.Printf("Performing the checkpoint predump = %t\n", predump)
	args := []string{"runc", "--id", containerId, "checkpoint", "--image-path", imagesPath}
	if predump {
		args = append(args, "--pre-dump")
	}
	_, _, err := cmd.Run("sudo", args...)
	if err != nil {
		log.Fatal("Error performing checkpoint:", err)
	}
}

func prepareDir(cmd cmd.Cmd, path string) {
	_, _, err := cmd.Run("mkdir", "-p", path)
	if err != nil {
		log.Fatal("Error preparing pre-dump dir:", err)
	}
}

func getContainerId(path string) string {
	_, id := filepath.Split(path)
	return id
}

func TriggerHook(command string) error {
	if command == "" {
		return nil
	}

	args := strings.Fields(command)
	c := cmd.NewLocal()
	_, _, err := c.Run(args[0], args[1:]...)

	return err
}
<commit_msg>Force flag should be bool<commit_after>package migrate

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/marcosnils/cmt/cmd"
	"github.com/marcosnils/cmt/validate"
)

var Command = cli.Command{
	Name:  "migrate",
	Usage: "Migrate running container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "src",
			Usage: "Source host where the container is running",
		},
		cli.StringFlag{
			Name:  "dst",
			Usage: "Target host to migrate the container",
		},
		cli.BoolFlag{
			Name:  "pre-dump",
			Usage: "Perform a pre-dump to minimize downtime",
		},
		cli.BoolFlag{
			Name:  "force",
			Usage: "Doesn't fail of validations related to different versions of executable binaries and cpu differences",
		},
		cli.StringFlag{
			Name:  "hook-pre-restore",
			Usage: "Command to run right before restoring process",
		},
		cli.StringFlag{
			Name:  "hook-post-restore",
			Usage: "Command to run right after a successful process restoration",
		},
		cli.StringFlag{
			Name:  "hook-failed-restore",
			Usage: "Command to run right after a failed process restoration",
		},
	},
	Action: func(c *cli.Context) {
		srcUrl := validate.ParseURL(c.String("src"))
		dstUrl := validate.ParseURL(c.String("dst"))

		log.Println("Performing validations")
		src, dst := validate.Validate(srcUrl, dstUrl, c.Bool("force"))

		log.Println("Preparing everything to do a checkpoint")
		containerId := getContainerId(srcUrl.Path)
		var imagesPath string
		var restoreCmd cmd.Cmd
		var migrateStart time.Time
		var downtime time.Duration

		if c.Bool("pre-dump") {
			// Process pre-dump
			predumpPath := fmt.Sprintf("%s/images/0", srcUrl.Path)
			prepareDir(src, predumpPath)

			checkpoint(src, containerId, predumpPath, true)

			srcTarFile := fmt.Sprintf("%s/predump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, predumpPath)

			prepareDir(dst, fmt.Sprintf("%s/images/0", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/0", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/0/predump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/0", dstUrl.Path))

			// Process final image
			migrateStart = time.Now()

			imagesPath = fmt.Sprintf("%s/images/1", srcUrl.Path)
			log.Println("Performing the checkpoint")
			_, _, err = src.Run("sudo", "runc", "--id", containerId, "checkpoint", "--image-path", imagesPath, "--prev-images-dir", predumpPath)
			if err != nil {
				log.Fatal("Error performing checkpoint:", err)
			}

			srcTarFile = fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)
			prepareDir(dst, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err = cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/1", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile = fmt.Sprintf("%s/images/1/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Performing the restore")
			TriggerHook(c.String("hook-pre-restore"))
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images/1", dstUrl.Path)

			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}
		} else {
			imagesPath = fmt.Sprintf("%s/images", srcUrl.Path)
			prepareDir(src, imagesPath)

			migrateStart = time.Now()
			checkpoint(src, containerId, imagesPath, false)

			srcTarFile := fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)

			prepareDir(dst, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Copying checkpoint image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Performing the restore")
			TriggerHook(c.String("hook-pre-restore"))
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		}

		var restoreSucceed bool
		var restoreError error
		var wg sync.WaitGroup
		wg.Add(1)

		go func() {
			restoreError = restoreCmd.Wait()
			wg.Done()
		}()

		go func() {
			log.Println("Waiting for container to start...")
			// We make a fast check so we don't wait for the first ticker internal
			if isRunning(containerId, dst) {
				restoreSucceed = true
				wg.Done()
				return
			}
			ticker := time.NewTicker(200 * time.Millisecond)
			go func() {
				for _ = range ticker.C {
					if isRunning(containerId, dst) {
						restoreSucceed = true
						break
					}

				}
				ticker.Stop()
				wg.Done()
			}()
		}()

		wg.Wait()

		downtime = time.Since(migrateStart)

		if restoreSucceed {
			log.Printf("Restore finished successfully, total downtime: %dms", downtime/time.Millisecond)
			TriggerHook(c.String("hook-post-restore"))
		} else {
			log.Println("Error performing restore:", restoreError)
			TriggerHook(c.String("hook-failed-restore"))
		}

	},
}

func isRunning(containerId string, dstCmd cmd.Cmd) bool {
	_, _, err := dstCmd.Run("stat", fmt.Sprintf("/var/run/opencontainer/containers/%s", containerId))
	if err != nil {
		return true
	}

	return false
}

func unpackTar(cmd cmd.Cmd, tarFile, workDir string) {
	log.Println("Preparing image at destination host")
	_, _, err := cmd.Run("sudo", "tar", "-C", workDir, "-xvzf", tarFile)
	if err != nil {
		log.Fatal("Error uncompressing image in destination:", err)
	}
}

func prepareTar(cmd cmd.Cmd, tarFile, workDir string) {
	_, _, err := cmd.Run("sudo", "tar", "-czf", tarFile, "-C", fmt.Sprintf("%s/", workDir), ".")
	if err != nil {
		log.Fatal("Error compressing image in source:", err)
	}
}

func checkpoint(cmd cmd.Cmd, containerId, imagesPath string, predump bool) {
	log.Printf("Performing the checkpoint predump = %t\n", predump)
	args := []string{"runc", "--id", containerId, "checkpoint", "--image-path", imagesPath}
	if predump {
		args = append(args, "--pre-dump")
	}
	_, _, err := cmd.Run("sudo", args...)
	if err != nil {
		log.Fatal("Error performing checkpoint:", err)
	}
}

func prepareDir(cmd cmd.Cmd, path string) {
	_, _, err := cmd.Run("mkdir", "-p", path)
	if err != nil {
		log.Fatal("Error preparing pre-dump dir:", err)
	}
}

func getContainerId(path string) string {
	_, id := filepath.Split(path)
	return id
}

func TriggerHook(command string) error {
	if command == "" {
		return nil
	}

	args := strings.Fields(command)
	c := cmd.NewLocal()
	_, _, err := c.Run(args[0], args[1:]...)

	return err
}
<|endoftext|>


<commit_before>package article

import (
	"bytes"

	"github.com/ptt/pttweb/ansi"
	"github.com/ptt/pttweb/pttbbs"
)

const (
	kPreviewContentLines = 5
)

type Renderer struct {
	buf    bytes.Buffer
	lineNo int

	mapper    *IndexMapper
	lineBuf   bytes.Buffer
	lineSegs  []Segment
	segIndex  int
	segOffset int
	segClosed bool

	terminalState TerminalState

	acceptMetaLines bool

	title string

	previewContent   string
	previewLineCount int
}

func NewRenderer() *Renderer {
	ar := &Renderer{
		mapper:   NewIndexMapper(2),
		lineSegs: make([]Segment, 0, 8),
	}
	ar.Reset()
	return ar
}

func (r *Renderer) Reset() {
	r.buf.Reset()
	r.lineNo = 1

	r.mapper.Reset()
	r.lineBuf.Reset()
	r.lineSegs = r.lineSegs[0:0]
	r.segIndex = 0
	r.segOffset = 0
	r.segClosed = true

	r.terminalState.Reset()

	r.acceptMetaLines = true

	r.title = ""

	r.previewContent = ""
	r.previewLineCount = 0
}

func (r *Renderer) ParsedTitle() string {
	return r.title
}

func (r *Renderer) PreviewContent() string {
	return r.previewContent
}

func (r *Renderer) Render(content []byte) (*bytes.Buffer, error) {
	converter := &ansi.AnsiParser{
		Rune: func(ru rune) {
			r.oneRune(ru)
		},
		Escape: func(esc ansi.EscapeSequence) {
			r.escape(esc)
		},
	}
	if err := converter.ConvertFromUTF8(content); err != nil {
		return nil, err
	}
	return &r.buf, nil
}

func (r *Renderer) currSeg() *Segment {
	if len(r.lineSegs) == 0 || r.segClosed {
		r.startSegment()
	}
	return &r.lineSegs[len(r.lineSegs)-1]
}

func (r *Renderer) escape(esc ansi.EscapeSequence) {
	r.terminalState.ApplyEscapeSequence(esc)
	if r.segClosed || !r.terminalState.Equal(&r.currSeg().TermState) {
		r.startSegment()
	}
}

func (r *Renderer) startSegment() {
	if !r.segClosed {
		r.endSegment()
	}
	r.lineSegs = append(r.lineSegs, Segment{
		Tag:       "span",
		TermState: r.terminalState,
		Buffer:    &bytes.Buffer{},
	})
	r.segClosed = false
}

func (r *Renderer) endSegment() {
	// Remove empty segment
	if r.lineSegs[len(r.lineSegs)-1].Len() == 0 {
		r.lineSegs = r.lineSegs[:len(r.lineSegs)-1]
	}

	r.segClosed = true
}

func (r *Renderer) oneRune(ru rune) {
	seg := r.currSeg()
	r.mapper.Record(r.lineBuf.Len(), len(r.lineSegs)-1, seg.Len())
	fastWriteHtmlEscapedRune(seg.Buffer, ru)
	r.lineBuf.WriteRune(ru)

	if ru == '\n' {
		r.endOfLine()
	}
}

func (r *Renderer) outputToSegment(i, off int) {
	for ; r.segIndex < i; r.segIndex++ {
		s := &r.lineSegs[r.segIndex]
		if r.segOffset > 0 {
			// half output
			if r.segClosed {
				s.WriteOpen(&r.buf)
			}
			r.buf.Write(s.InnerBytes()[r.segOffset:])
			s.WriteClose(&r.buf)
			r.segOffset = 0
			r.segClosed = true
		} else {
			if !r.segClosed {
				panic("Segment not closed at beginning of a segment is not allowed")
			}
			s.WriteTo(&r.buf)
			r.segClosed = true
		}
	}
	if off > 0 {
		s := &r.lineSegs[r.segIndex]
		if r.segClosed {
			s.WriteOpen(&r.buf)
		}
		r.buf.Write(s.InnerBytes()[r.segOffset:off])
		r.segOffset = off
		r.segClosed = false
	}
}

func (r *Renderer) skipToSegment(i, off int) {
	r.prematureCloseSegment()
	r.segIndex = i
	r.segOffset = off
}

func (r *Renderer) prematureCloseSegment() {
	if !r.segClosed {
		r.lineSegs[r.segIndex].WriteClose(&r.buf)
		r.segClosed = true
	}
}

func (r *Renderer) matchFirstLineAndOutput(line []byte) bool {
	tag1, val1, tag2, val2, ok := pttbbs.ParseArticleFirstLine(r.lineBuf.Bytes())
	if !ok {
		return false
	}

	r.writeMetaLine(tag1, val1, ClassArticleMetaLine)
	r.writeMetaLine(tag2, val2, ClassArticleMetaLineRight)
	return true
}

func (r *Renderer) writeMetaLine(tag, val []byte, divClass string) {
	r.buf.WriteString(`<div class="` + divClass + `"><span class="` + ClassArticleMetaTag + `">`)
	fastWriteHtmlEscaped(&r.buf, string(tag))
	r.buf.WriteString(`</span>`)
	r.buf.WriteString(`<span class="` + ClassArticleMetaValue + `">`)
	fastWriteHtmlEscaped(&r.buf, string(val))
	r.buf.WriteString(`</span></div>`)
}

func (r *Renderer) endOfLine() {
	r.segClosed = true

	line := r.lineBuf.Bytes()
	parsed := false

	if r.acceptMetaLines && r.lineNo < 5 {
		if r.lineNo == 1 && r.matchFirstLineAndOutput(line) {
			parsed = true
		} else if tag, val, ok := pttbbs.ParseArticleMetaLine(line); ok {
			if bytes.Equal(tag, []byte(pttbbs.ArticleTitle)) {
				r.title = string(val)
			}
			r.writeMetaLine(tag, val, ClassArticleMetaLine)
			parsed = true
		} else {
			r.acceptMetaLines = false
		}
	}

	if !parsed {
		if len(r.lineSegs) > 0 {
			if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.QuotePrefixStrings) {
				r.lineSegs[0].TermState.SetColor(6, DefaultBg, NoFlags)
			} else if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.SignaturePrefixStrings) {
				r.lineSegs[0].TermState.SetColor(2, DefaultBg, NoFlags)
			}
		}

		if r.previewLineCount < kPreviewContentLines {
			r.previewContent += string(line)
			r.previewLineCount++
		}
		r.processNormalContentLine(line)
	}

	// Reset and update variables
	r.mapper.Reset()
	r.lineBuf.Reset()
	r.lineSegs = r.lineSegs[0:0]
	r.segIndex = 0
	r.segOffset = 0
	r.segClosed = true
	r.lineNo++
}

func (r *Renderer) processNormalContentLine(line []byte) {
	// Detect push line
	isPush := false
	if matchPushLine(r.lineSegs) {
		r.lineSegs[0].ExtraFlags |= PushTag
		r.lineSegs[1].ExtraFlags |= PushUserId
		r.lineSegs[2].ExtraFlags |= PushContent
		r.lineSegs[3].ExtraFlags |= PushIpDateTime
		// Remove trailing spaces
		r.lineSegs[2].TrimRight(" ")
		r.buf.WriteString(`<div class="` + ClassPushDiv + `">`)
		isPush = true
	}

	for _, u := range FindAllUrlsIndex(line) {
		urlString := string(line[u[0]:u[1]])
		linkBegin, linkEnd := makeExternalUrlLink(urlString)

		begin := r.mapper.Get(u[0])
		end := r.mapper.Get(u[1])
		r.outputToSegment(begin[0], begin[1])
		if begin[0] == end[0] {
			// same segment: embed
			r.buf.WriteString(linkBegin)
			r.outputToSegment(end[0], end[1])
			r.buf.WriteString(linkEnd)
		} else {
			// different segments: split, wrap-around
			r.prematureCloseSegment()
			r.buf.WriteString(linkBegin)
			r.outputToSegment(end[0], end[1])
			r.prematureCloseSegment()
			r.buf.WriteString(linkEnd)
		}
	}
	r.outputToSegment(len(r.lineSegs), 0)

	if isPush {
		r.buf.WriteString(`</div>`)
	}

	if rcs, err := FindRichContents(r.lineBuf.Bytes()); err == nil {
		for _, rc := range rcs {
			r.buf.WriteString(`<div class="richcontent">` + rc.ContentHtml + `</div>`)
		}
	}
}
<commit_msg>article: use method values.<commit_after>package article

import (
	"bytes"

	"github.com/ptt/pttweb/ansi"
	"github.com/ptt/pttweb/pttbbs"
)

const (
	kPreviewContentLines = 5
)

type Renderer struct {
	buf    bytes.Buffer
	lineNo int

	mapper    *IndexMapper
	lineBuf   bytes.Buffer
	lineSegs  []Segment
	segIndex  int
	segOffset int
	segClosed bool

	terminalState TerminalState

	acceptMetaLines bool

	title string

	previewContent   string
	previewLineCount int
}

func NewRenderer() *Renderer {
	ar := &Renderer{
		mapper:   NewIndexMapper(2),
		lineSegs: make([]Segment, 0, 8),
	}
	ar.Reset()
	return ar
}

func (r *Renderer) Reset() {
	r.buf.Reset()
	r.lineNo = 1

	r.mapper.Reset()
	r.lineBuf.Reset()
	r.lineSegs = r.lineSegs[0:0]
	r.segIndex = 0
	r.segOffset = 0
	r.segClosed = true

	r.terminalState.Reset()

	r.acceptMetaLines = true

	r.title = ""

	r.previewContent = ""
	r.previewLineCount = 0
}

func (r *Renderer) ParsedTitle() string {
	return r.title
}

func (r *Renderer) PreviewContent() string {
	return r.previewContent
}

func (r *Renderer) Render(content []byte) (*bytes.Buffer, error) {
	converter := &ansi.AnsiParser{
		Rune:   r.oneRune,
		Escape: r.escape,
	}
	if err := converter.ConvertFromUTF8(content); err != nil {
		return nil, err
	}
	return &r.buf, nil
}

func (r *Renderer) currSeg() *Segment {
	if len(r.lineSegs) == 0 || r.segClosed {
		r.startSegment()
	}
	return &r.lineSegs[len(r.lineSegs)-1]
}

func (r *Renderer) escape(esc ansi.EscapeSequence) {
	r.terminalState.ApplyEscapeSequence(esc)
	if r.segClosed || !r.terminalState.Equal(&r.currSeg().TermState) {
		r.startSegment()
	}
}

func (r *Renderer) startSegment() {
	if !r.segClosed {
		r.endSegment()
	}
	r.lineSegs = append(r.lineSegs, Segment{
		Tag:       "span",
		TermState: r.terminalState,
		Buffer:    &bytes.Buffer{},
	})
	r.segClosed = false
}

func (r *Renderer) endSegment() {
	// Remove empty segment
	if r.lineSegs[len(r.lineSegs)-1].Len() == 0 {
		r.lineSegs = r.lineSegs[:len(r.lineSegs)-1]
	}

	r.segClosed = true
}

func (r *Renderer) oneRune(ru rune) {
	seg := r.currSeg()
	r.mapper.Record(r.lineBuf.Len(), len(r.lineSegs)-1, seg.Len())
	fastWriteHtmlEscapedRune(seg.Buffer, ru)
	r.lineBuf.WriteRune(ru)

	if ru == '\n' {
		r.endOfLine()
	}
}

func (r *Renderer) outputToSegment(i, off int) {
	for ; r.segIndex < i; r.segIndex++ {
		s := &r.lineSegs[r.segIndex]
		if r.segOffset > 0 {
			// half output
			if r.segClosed {
				s.WriteOpen(&r.buf)
			}
			r.buf.Write(s.InnerBytes()[r.segOffset:])
			s.WriteClose(&r.buf)
			r.segOffset = 0
			r.segClosed = true
		} else {
			if !r.segClosed {
				panic("Segment not closed at beginning of a segment is not allowed")
			}
			s.WriteTo(&r.buf)
			r.segClosed = true
		}
	}
	if off > 0 {
		s := &r.lineSegs[r.segIndex]
		if r.segClosed {
			s.WriteOpen(&r.buf)
		}
		r.buf.Write(s.InnerBytes()[r.segOffset:off])
		r.segOffset = off
		r.segClosed = false
	}
}

func (r *Renderer) skipToSegment(i, off int) {
	r.prematureCloseSegment()
	r.segIndex = i
	r.segOffset = off
}

func (r *Renderer) prematureCloseSegment() {
	if !r.segClosed {
		r.lineSegs[r.segIndex].WriteClose(&r.buf)
		r.segClosed = true
	}
}

func (r *Renderer) matchFirstLineAndOutput(line []byte) bool {
	tag1, val1, tag2, val2, ok := pttbbs.ParseArticleFirstLine(r.lineBuf.Bytes())
	if !ok {
		return false
	}

	r.writeMetaLine(tag1, val1, ClassArticleMetaLine)
	r.writeMetaLine(tag2, val2, ClassArticleMetaLineRight)
	return true
}

func (r *Renderer) writeMetaLine(tag, val []byte, divClass string) {
	r.buf.WriteString(`<div class="` + divClass + `"><span class="` + ClassArticleMetaTag + `">`)
	fastWriteHtmlEscaped(&r.buf, string(tag))
	r.buf.WriteString(`</span>`)
	r.buf.WriteString(`<span class="` + ClassArticleMetaValue + `">`)
	fastWriteHtmlEscaped(&r.buf, string(val))
	r.buf.WriteString(`</span></div>`)
}

func (r *Renderer) endOfLine() {
	r.segClosed = true

	line := r.lineBuf.Bytes()
	parsed := false

	if r.acceptMetaLines && r.lineNo < 5 {
		if r.lineNo == 1 && r.matchFirstLineAndOutput(line) {
			parsed = true
		} else if tag, val, ok := pttbbs.ParseArticleMetaLine(line); ok {
			if bytes.Equal(tag, []byte(pttbbs.ArticleTitle)) {
				r.title = string(val)
			}
			r.writeMetaLine(tag, val, ClassArticleMetaLine)
			parsed = true
		} else {
			r.acceptMetaLines = false
		}
	}

	if !parsed {
		if len(r.lineSegs) > 0 {
			if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.QuotePrefixStrings) {
				r.lineSegs[0].TermState.SetColor(6, DefaultBg, NoFlags)
			} else if pttbbs.MatchPrefixBytesToStrings(line, pttbbs.SignaturePrefixStrings) {
				r.lineSegs[0].TermState.SetColor(2, DefaultBg, NoFlags)
			}
		}

		if r.previewLineCount < kPreviewContentLines {
			r.previewContent += string(line)
			r.previewLineCount++
		}
		r.processNormalContentLine(line)
	}

	// Reset and update variables
	r.mapper.Reset()
	r.lineBuf.Reset()
	r.lineSegs = r.lineSegs[0:0]
	r.segIndex = 0
	r.segOffset = 0
	r.segClosed = true
	r.lineNo++
}

func (r *Renderer) processNormalContentLine(line []byte) {
	// Detect push line
	isPush := false
	if matchPushLine(r.lineSegs) {
		r.lineSegs[0].ExtraFlags |= PushTag
		r.lineSegs[1].ExtraFlags |= PushUserId
		r.lineSegs[2].ExtraFlags |= PushContent
		r.lineSegs[3].ExtraFlags |= PushIpDateTime
		// Remove trailing spaces
		r.lineSegs[2].TrimRight(" ")
		r.buf.WriteString(`<div class="` + ClassPushDiv + `">`)
		isPush = true
	}

	for _, u := range FindAllUrlsIndex(line) {
		urlString := string(line[u[0]:u[1]])
		linkBegin, linkEnd := makeExternalUrlLink(urlString)

		begin := r.mapper.Get(u[0])
		end := r.mapper.Get(u[1])
		r.outputToSegment(begin[0], begin[1])
		if begin[0] == end[0] {
			// same segment: embed
			r.buf.WriteString(linkBegin)
			r.outputToSegment(end[0], end[1])
			r.buf.WriteString(linkEnd)
		} else {
			// different segments: split, wrap-around
			r.prematureCloseSegment()
			r.buf.WriteString(linkBegin)
			r.outputToSegment(end[0], end[1])
			r.prematureCloseSegment()
			r.buf.WriteString(linkEnd)
		}
	}
	r.outputToSegment(len(r.lineSegs), 0)

	if isPush {
		r.buf.WriteString(`</div>`)
	}

	if rcs, err := FindRichContents(r.lineBuf.Bytes()); err == nil {
		for _, rc := range rcs {
			r.buf.WriteString(`<div class="richcontent">` + rc.ContentHtml + `</div>`)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"golang.org/x/net/webdav"
	"github.com/fsouza/go-dockerclient"
)

const (
	DOCKER_TAR_PREFIX = "rootfs/"
	OWNER_PERM_RW     = 0600
	VERSION_PREFIX    = "/v1"
	CONTENT_PREFIX    = VERSION_PREFIX + "/content/"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("docker-fleece-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to fleece")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to fleece must be specified")
	}
	if *path == "" {
		log.Fatalf("Destination path for image files must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	log.Printf("Pulling image %s", *image)
	imagePullOption := docker.PullImageOptions{Repository: *image}
	imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
	if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
		log.Fatalf("Unable to pull docker image: %v\n", err)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	err = os.Mkdir(*path, 0755)
	if err != nil {
		if !os.IsExist(err) {
			log.Fatalf("Unable to create destination path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	_ = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_PREFIX)

		http.Handle(CONTENT_PREFIX, webdav.StripPrefix(CONTENT_PREFIX, &webdav.Handler{
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		}))

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<commit_msg>add support for serving image metadata<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"github.com/fsouza/go-dockerclient"
	"golang.org/x/net/webdav"
)

const (
	DOCKER_TAR_PREFIX  = "rootfs/"
	OWNER_PERM_RW      = 0600
	VERSION_PREFIX     = "/v1"
	CONTENT_URL_PREFIX = VERSION_PREFIX + "/content/"
	METADATA_URL_PATH  = VERSION_PREFIX + "/metadata"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("docker-fleece-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to fleece")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to fleece must be specified")
	}
	if *path == "" {
		log.Fatalf("Destination path for image files must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	log.Printf("Pulling image %s", *image)
	imagePullOption := docker.PullImageOptions{Repository: *image}
	imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
	if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
		log.Fatalf("Unable to pull docker image: %v\n", err)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	containerMetadata, err := client.InspectContainer(container.ID)
	if err != nil {
		log.Fatalf("Unable to get docker container information: %v\n", err)
	}

	imageMetadata, err := client.InspectImage(containerMetadata.Image)
	if err != nil {
		log.Fatalf("Unable to get docker image information: %v\n", err)
	}

	err = os.Mkdir(*path, 0755)
	if err != nil {
		if !os.IsExist(err) {
			log.Fatalf("Unable to create destination path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	_ = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_URL_PREFIX)

		http.Handle(CONTENT_URL_PREFIX, webdav.StripPrefix(CONTENT_URL_PREFIX, &webdav.Handler{
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		}))

		http.HandleFunc(METADATA_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.Marshal(imageMetadata)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<|endoftext|>


<commit_before>package model

import (
	"github.com/slyrz/newscat/html"
	"math"
	"strings"
)

// ChunkExtractor utilizes the trained model to extract relevant html.Chunks from
// an html.Article.
type ChunkExtractor struct {
	ChunkFeatures []chunkFeature
	BoostFeatures []boostFeature
}

// LinkExtractor extracts possible links to news articles from a html.Website.
type LinkExtractor struct {
}

// NewChunkExtractor creates and initializes a new ChunkExtractor.
func NewChunkExtractor() *ChunkExtractor {
	return new(ChunkExtractor)
}

// Extract returns a list of relevant text chunks found in article.
//
// How it works
//
// This function creates a feature vector for each chunk found in article.
// A feature vector contains a numerical representation of the chunk's
// properties like HTML element type, parent element type, number of words,
// number of sentences and stuff like this.
//
// A logistic regression model is used to calculate scores based on these
// feature vectors. Then, in some kind of meta / ensemble learning approach,
// a second type of feature vector is created based on these scores.
// This feature vector is fed to our random forest and finally
// the random forest's predictions are used to generate the result.
//
// By now you might have noticed that I'm exceptionally bad at naming and
// describing things properly.
func (ext *ChunkExtractor) Extract(article *html.Article) []*html.Chunk {
	ext.ChunkFeatures = nil
	ext.BoostFeatures = nil
	if len(article.Chunks) == 0 {
		return nil
	}

	chunkFeatures := make([]chunkFeature, len(article.Chunks))
	boostFeatures := make([]boostFeature, len(article.Chunks))

	// Count the number of words and sentences we encountered for each
	// class. This helps us to detect elements that contain the article text.
	classStats := article.GetClassStats()
	clusterStats := article.GetClusterStats()

	chunkFeatureWriter := new(chunkFeatureWriter)
	for i, chunk := range article.Chunks {
		chunkFeatureWriter.Assign(chunkFeatures[i][:])
		chunkFeatureWriter.WriteElementType(chunk)
		chunkFeatureWriter.WriteParentType(chunk)
		chunkFeatureWriter.WriteSiblingTypes(chunk)
		chunkFeatureWriter.WriteAncestors(chunk)
		chunkFeatureWriter.WriteTextStat(chunk)
		chunkFeatureWriter.WriteTextStatSiblings(chunk)
		chunkFeatureWriter.WriteClassStat(chunk, classStats)
		chunkFeatureWriter.WriteClusterStat(chunk, clusterStats)
	}

	// Detect the minimum and maximum value for each element in the
	// feature vector.
	empMin := chunkFeature{}
	empMax := chunkFeature{}
	for i := range chunkFeatures {
		for j, val := range chunkFeatures[i] {
			switch {
			case val < empMin[j]:
				empMin[j] = val
			case val > empMax[j]:
				empMax[j] = val
			}
		}
	}

	// Perform MinMax normalization.
	for i := range chunkFeatures {
		feature := &chunkFeatures[i]
		for j, val := range chunkFeatures[i] {
			// If the maximum value is not greater than one, we assume that the feature is
			// already normalized and leave it untouched.
			if empMax[j] > 1.0 {
				feature[j] = (val - empMin[j]) / (empMax[j] - empMin[j])
			}
		}
	}

	// Now cluster chunks by containers to calculate average score per
	// container.
	clusterContainer := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterContainer.Add(chunk.Container, chunk, chunkFeatures[i].Score())
	}

	boostFeatureWriter := new(boostFeatureWriter)
	for i, chunk := range article.Chunks {
		boostFeatureWriter.Assign(boostFeatures[i][:])
		boostFeatureWriter.WriteChunk(chunk)
		boostFeatureWriter.WriteCluster(chunk, clusterContainer[chunk.Container])
		boostFeatureWriter.WriteTitleSimilarity(chunk, article.Title)
	}

	// Cluster chunks by block and add those blocks to the result whose average
	// score is above prediction level. This makes sure that we don't split large
	// blocks.
	clusterBlock := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterBlock.Add(chunk.Block, chunk, boostFeatures[i].Score(), float32(chunk.Text.Len()))
	}

	// Keep blocks together.
	result := make([]*html.Chunk, 0, 8)
	for _, chunk := range article.Chunks {
		if clusterBlock[chunk.Block].Score() > 0.5 {
			result = append(result, chunk)
		}
	}

	// Make them accessible.
	ext.ChunkFeatures = chunkFeatures
	ext.BoostFeatures = boostFeatures
	return result
}

// NewLinkExtractor creates and initializes a new LinkExtractor.
func NewLinkExtractor() *LinkExtractor {
	return new(LinkExtractor)
}

// Extract returns a list of links to possible news articles found on website.
//
// TODO
func (ext *LinkExtractor) Extract(website *html.Website) []*html.Link {
	if len(website.Links) == 0 {
		return nil
	}

	// Calculate the host name frequency. We assume that the more often
	// we encounter a host name, the more important are its links.
	hostFreq := make(map[string]float64)
	unitFrac := 1.0 / float64(len(website.Links))
	for _, link := range website.Links {
		hostFreq[link.URL.Host] = hostFreq[link.URL.Host] + unitFrac
	}

	// TODO
	result := make([]*html.Link, 0, 8)
	for _, link := range website.Links {
		score := calcEntropy(link.URL.Path) * hostFreq[link.URL.Host]
		if score >= 3.0 {
			result = append(result, link)
		}
	}
	return result
}

var (
	entropy = map[rune]float64{
		'a': -0.10634380971489953,
		'b': -0.1530425750819142,
		'c': -0.054512771770650156,
		'd': -0.2277081908035247,
		'e': -0.019143424124981535,
		'f': -0.10631901817532326,
		'g': -0.08950958783746302,
		'h': -0.02138890867431747,
		'i': -0.2040842535452831,
		'j': -0.08066894356623576,
		'k': -0.14676065483416048,
		'l': -0.054715501033775864,
		'm': -0.22756818123774952,
		'n': -0.2036576755124059,
		'o': -0.011796162629390488,
		'p': -0.19352586509733832,
		'q': -0.061206381465562155,
		'r': -0.12701158290463668,
		's': -0.10100191941724014,
		't': -0.022872200444912123,
		'u': -0.05095402954111724,
		'v': -0.09120240867794169,
		'w': -0.20676837174037171,
		'x': -0.0856356305628621,
		'y': -0.052914228421479595,
		'z': -0.08290376493347784,
		'0': -0.20509019606654505,
		'1': -0.04936506198036805,
		'2': -0.052649170776553546,
		'3': -0.28439304075529853,
		'4': -0.11790007547597285,
		'5': -0.09902560480951333,
		'6': -0.0624913286633471,
		'7': -0.1247640940220319,
		'8': -0.17807011198199005,
		'9': -0.2233673033869976,
	}
)

// calcEntropy returns the entropy of a string.
func calcEntropy(s string) float64 {
	result := 0.0
	for _, r := range strings.ToLower(s) {
		result += entropy[r]
	}
	return math.Abs(result)
}
<commit_msg>added explanation<commit_after>package model

import (
	"github.com/slyrz/newscat/html"
	"math"
	"strings"
)

// ChunkExtractor utilizes the trained model to extract relevant html.Chunks from
// an html.Article.
type ChunkExtractor struct {
	ChunkFeatures []chunkFeature
	BoostFeatures []boostFeature
}

// LinkExtractor extracts possible links to news articles from a html.Website.
type LinkExtractor struct {
}

// NewChunkExtractor creates and initializes a new ChunkExtractor.
func NewChunkExtractor() *ChunkExtractor {
	return new(ChunkExtractor)
}

// Extract returns a list of relevant text chunks found in article.
//
// How it works
//
// This function creates a feature vector for each chunk found in article.
// A feature vector contains a numerical representation of the chunk's
// properties like HTML element type, parent element type, number of words,
// number of sentences and stuff like this.
//
// A logistic regression model is used to calculate scores based on these
// feature vectors. Then, in some kind of meta / ensemble learning approach,
// a second type of feature vector is created based on these scores.
// This feature vector is fed to our random forest and finally
// the random forest's predictions are used to generate the result.
//
// By now you might have noticed that I'm exceptionally bad at naming and
// describing things properly.
func (ext *ChunkExtractor) Extract(article *html.Article) []*html.Chunk {
	ext.ChunkFeatures = nil
	ext.BoostFeatures = nil
	if len(article.Chunks) == 0 {
		return nil
	}

	chunkFeatures := make([]chunkFeature, len(article.Chunks))
	boostFeatures := make([]boostFeature, len(article.Chunks))

	// Count the number of words and sentences we encountered for each
	// class. This helps us to detect elements that contain the article text.
	classStats := article.GetClassStats()
	clusterStats := article.GetClusterStats()

	chunkFeatureWriter := new(chunkFeatureWriter)
	for i, chunk := range article.Chunks {
		chunkFeatureWriter.Assign(chunkFeatures[i][:])
		chunkFeatureWriter.WriteElementType(chunk)
		chunkFeatureWriter.WriteParentType(chunk)
		chunkFeatureWriter.WriteSiblingTypes(chunk)
		chunkFeatureWriter.WriteAncestors(chunk)
		chunkFeatureWriter.WriteTextStat(chunk)
		chunkFeatureWriter.WriteTextStatSiblings(chunk)
		chunkFeatureWriter.WriteClassStat(chunk, classStats)
		chunkFeatureWriter.WriteClusterStat(chunk, clusterStats)
	}

	// Detect the minimum and maximum value for each element in the
	// feature vector.
	empMin := chunkFeature{}
	empMax := chunkFeature{}
	for i := range chunkFeatures {
		for j, val := range chunkFeatures[i] {
			switch {
			case val < empMin[j]:
				empMin[j] = val
			case val > empMax[j]:
				empMax[j] = val
			}
		}
	}

	// Perform MinMax normalization.
	for i := range chunkFeatures {
		feature := &chunkFeatures[i]
		for j, val := range chunkFeatures[i] {
			// If the maximum value is not greater than one, we assume that the feature is
			// already normalized and leave it untouched.
			if empMax[j] > 1.0 {
				feature[j] = (val - empMin[j]) / (empMax[j] - empMin[j])
			}
		}
	}

	// Now cluster chunks by containers to calculate average score per
	// container.
	clusterContainer := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterContainer.Add(chunk.Container, chunk, chunkFeatures[i].Score())
	}

	boostFeatureWriter := new(boostFeatureWriter)
	for i, chunk := range article.Chunks {
		boostFeatureWriter.Assign(boostFeatures[i][:])
		boostFeatureWriter.WriteChunk(chunk)
		boostFeatureWriter.WriteCluster(chunk, clusterContainer[chunk.Container])
		boostFeatureWriter.WriteTitleSimilarity(chunk, article.Title)
	}

	// Cluster chunks by block and add those blocks to the result whose average
	// score is above prediction level. This makes sure that we don't split large
	// blocks.
	clusterBlock := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterBlock.Add(chunk.Block, chunk, boostFeatures[i].Score(), float32(chunk.Text.Len()))
	}

	// Keep blocks together.
	result := make([]*html.Chunk, 0, 8)
	for _, chunk := range article.Chunks {
		if clusterBlock[chunk.Block].Score() > 0.5 {
			result = append(result, chunk)
		}
	}

	// Make them accessible.
	ext.ChunkFeatures = chunkFeatures
	ext.BoostFeatures = boostFeatures
	return result
}

// NewLinkExtractor creates and initializes a new LinkExtractor.
func NewLinkExtractor() *LinkExtractor {
	return new(LinkExtractor)
}

// Extract returns a list of links to possible news articles found on website.
//
// How it works
//
// This function calculates a score for each link found on website. The score
// is the product of the host name frequency and the entropy of the
// path component. If the score exceeds a fixed threshold, the link is assumed
// to be relevant and included in the result.
//
// This is very simple and works surpsingly well since for most news sites,
// the following assumptions hold:
// 1. The majority of links lead to relevant content.
// 2. The majority of relevant links share the same host name.
// 3. The path of a relevant link tends to be longer than the path of a
//    non-relevant link.
func (ext *LinkExtractor) Extract(website *html.Website) []*html.Link {
	if len(website.Links) == 0 {
		return nil
	}

	// Calculate the host name frequency. We assume that the more often
	// we encounter a host name, the more important are its links.
	hostFreq := make(map[string]float64)
	unitFrac := 1.0 / float64(len(website.Links))
	for _, link := range website.Links {
		hostFreq[link.URL.Host] = hostFreq[link.URL.Host] + unitFrac
	}

	result := make([]*html.Link, 0, 8)
	for _, link := range website.Links {
		score := calcEntropy(link.URL.Path) * hostFreq[link.URL.Host]
		if score >= 3.0 {
			result = append(result, link)
		}
	}
	return result
}

var (
	entropy = map[rune]float64{
		'a': -0.10634380971489953,
		'b': -0.1530425750819142,
		'c': -0.054512771770650156,
		'd': -0.2277081908035247,
		'e': -0.019143424124981535,
		'f': -0.10631901817532326,
		'g': -0.08950958783746302,
		'h': -0.02138890867431747,
		'i': -0.2040842535452831,
		'j': -0.08066894356623576,
		'k': -0.14676065483416048,
		'l': -0.054715501033775864,
		'm': -0.22756818123774952,
		'n': -0.2036576755124059,
		'o': -0.011796162629390488,
		'p': -0.19352586509733832,
		'q': -0.061206381465562155,
		'r': -0.12701158290463668,
		's': -0.10100191941724014,
		't': -0.022872200444912123,
		'u': -0.05095402954111724,
		'v': -0.09120240867794169,
		'w': -0.20676837174037171,
		'x': -0.0856356305628621,
		'y': -0.052914228421479595,
		'z': -0.08290376493347784,
		'0': -0.20509019606654505,
		'1': -0.04936506198036805,
		'2': -0.052649170776553546,
		'3': -0.28439304075529853,
		'4': -0.11790007547597285,
		'5': -0.09902560480951333,
		'6': -0.0624913286633471,
		'7': -0.1247640940220319,
		'8': -0.17807011198199005,
		'9': -0.2233673033869976,
	}
)

// calcEntropy returns the entropy of a string.
func calcEntropy(s string) float64 {
	result := 0.0
	for _, r := range strings.ToLower(s) {
		result += entropy[r]
	}
	return math.Abs(result)
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"strconv"

	"github.com/hashicorp/consul/agent/checks"
	"github.com/hashicorp/consul/agent/consul"
	"github.com/hashicorp/consul/version"
	"github.com/hashicorp/raft"
)

// DefaultSource is the default agent configuration.
// This needs to be merged first in the head.
// todo(fs): The values are sourced from multiple sources.
// todo(fs): IMO, this should be the definitive default for all configurable values
// todo(fs): and whatever is in here should clobber every default value. Hence, no sourcing.
func DefaultSource() Source {
	cfg := consul.DefaultConfig()
	serfLAN := cfg.SerfLANConfig.MemberlistConfig
	serfWAN := cfg.SerfWANConfig.MemberlistConfig

	// DEPRECATED (ACL-Legacy-Compat) - when legacy ACL support is removed these defaults
	//   the acl_* config entries here should be transitioned to their counterparts in the
	//   acl stanza for now we need to be able to detect the new entries not being set (not
	//   just set to the defaults here) so that we can use the old entries. So the true
	//   default still needs to reside in the original config values
	return Source{
		Name:   "default",
		Format: "hcl",
		Data: `
		acl_default_policy = "allow"
		acl_down_policy = "extend-cache"
		acl_ttl = "30s"
		acl = {
			policy_ttl = "30s"
		}
		bind_addr = "0.0.0.0"
		bootstrap = false
		bootstrap_expect = 0
		check_output_max_size = ` + strconv.Itoa(checks.DefaultBufSize) + `
		check_update_interval = "5m"
		client_addr = "127.0.0.1"
		datacenter = "` + consul.DefaultDC + `"
		default_query_time = "300s"
		disable_coordinates = false
		disable_host_node_id = true
		disable_remote_exec = true
		domain = "consul."
		encrypt_verify_incoming = true
		encrypt_verify_outgoing = true
		log_level = "INFO"
		max_query_time = "600s"
		primary_gateways_interval = "30s"
		protocol = ` + strconv.Itoa(consul.DefaultRPCProtocol) + `
		retry_interval = "30s"
		retry_interval_wan = "30s"
		server = false
		syslog_facility = "LOCAL0"
		tls_min_version = "tls12"

		// TODO (slackpad) - Until #3744 is done, we need to keep these
		// in sync with agent/consul/config.go.
		autopilot = {
			cleanup_dead_servers = true
			last_contact_threshold = "200ms"
			max_trailing_logs = 250
			server_stabilization_time = "10s"
		}
		gossip_lan = {
			gossip_interval = "` + serfLAN.GossipInterval.String() + `"
			gossip_nodes = ` + strconv.Itoa(serfLAN.GossipNodes) + `
			retransmit_mult = ` + strconv.Itoa(serfLAN.RetransmitMult) + `
			probe_interval = "` + serfLAN.ProbeInterval.String() + `"
			probe_timeout = "` + serfLAN.ProbeTimeout.String() + `"
			suspicion_mult = ` + strconv.Itoa(serfLAN.SuspicionMult) + `
		}
		gossip_wan = {
			gossip_interval = "` + serfWAN.GossipInterval.String() + `"
			gossip_nodes = ` + strconv.Itoa(serfLAN.GossipNodes) + `
			retransmit_mult = ` + strconv.Itoa(serfLAN.RetransmitMult) + `
			probe_interval = "` + serfWAN.ProbeInterval.String() + `"
			probe_timeout = "` + serfWAN.ProbeTimeout.String() + `"
			suspicion_mult = ` + strconv.Itoa(serfWAN.SuspicionMult) + `
		}
		dns_config = {
			allow_stale = true
			a_record_limit = 0
			udp_answer_limit = 3
			max_stale = "87600h"
			recursor_timeout = "2s"
		}
		limits = {
			http_max_conns_per_client = 200
			https_handshake_timeout = "5s"
			rpc_handshake_timeout = "5s"
			rpc_rate = -1
			rpc_max_burst = 1000
			rpc_max_conns_per_client = 100
			kv_max_value_size = ` + strconv.FormatInt(raft.SuggestedMaxDataSize, 10) + `
			txn_max_req_len = ` + strconv.FormatInt(raft.SuggestedMaxDataSize, 10) + `
		}
		performance = {
			leave_drain_time = "5s"
			raft_multiplier = ` + strconv.Itoa(int(consul.DefaultRaftMultiplier)) + `
			rpc_hold_timeout = "7s"
		}
		ports = {
			dns = 8600
			http = 8500
			https = -1
			grpc = -1
			serf_lan = ` + strconv.Itoa(consul.DefaultLANSerfPort) + `
			serf_wan = ` + strconv.Itoa(consul.DefaultWANSerfPort) + `
			server = ` + strconv.Itoa(consul.DefaultRPCPort) + `
			proxy_min_port = 20000
			proxy_max_port = 20255
			sidecar_min_port = 21000
			sidecar_max_port = 21255
			expose_min_port = 21500
			expose_max_port = 21755
		}
		telemetry = {
			metrics_prefix = "consul"
			filter_default = true
		}

	`,
	}
}

// DevSource is the additional default configuration for dev mode.
// This should be merged in the head after the default configuration.
func DevSource() Source {
	return Source{
		Name:   "dev",
		Format: "hcl",
		Data: `
		bind_addr = "127.0.0.1"
		disable_anonymous_signature = true
		disable_keyring_file = true
		enable_debug = true
		ui = true
		log_level = "DEBUG"
		server = true

		gossip_lan = {
			gossip_interval = "100ms"
			probe_interval = "100ms"
			probe_timeout = "100ms"
			suspicion_mult = 3
		}
		gossip_wan = {
			gossip_interval = "100ms"
			probe_interval = "100ms"
			probe_timeout = "100ms"
			suspicion_mult = 3
		}
		connect = {
			enabled = true
		}
		performance = {
			raft_multiplier = 1
		}
		ports = {
			grpc = 8502
		}
	`,
	}
}

// NonUserSource contains the values the user cannot configure.
// This needs to be merged in the tail.
func NonUserSource() Source {
	return Source{
		Name:   "non-user",
		Format: "hcl",
		Data: `
		acl = {
			disabled_ttl = "120s"
		}
		check_deregister_interval_min = "1m"
		check_reap_interval = "30s"
		ae_interval = "1m"
		sync_coordinate_rate_target = 64
		sync_coordinate_interval_min = "15s"

		# segment_limit is the maximum number of network segments that may be declared.
		segment_limit = 64

		# SegmentNameLimit is the maximum segment name length.
		segment_name_limit = 64

		connect = {
			# 0s causes the value to be ignored and operate without capping
			# the max time before leaf certs can be generated after a roots change.
			test_ca_leaf_root_change_spread = "0s"
		}
	`,
	}
}

// VersionSource creates a config source for the version parameters.
// This should be merged in the tail since these values are not
// user configurable.
func VersionSource(rev, ver, verPre string) Source {
	return Source{
		Name:   "version",
		Format: "hcl",
		Data:   fmt.Sprintf(`revision = %q version = %q version_prerelease = %q`, rev, ver, verPre),
	}
}

// DefaultVersionSource returns the version config source for the embedded
// version numbers.
func DefaultVersionSource() Source {
	return VersionSource(version.GitCommit, version.Version, version.VersionPrerelease)
}

// DefaultConsulSource returns the default configuration for the consul agent.
// This should be merged in the tail since these values are not user configurable.
func DefaultConsulSource() Source {
	cfg := consul.DefaultConfig()
	raft := cfg.RaftConfig
	return Source{
		Name:   "consul",
		Format: "hcl",
		Data: `
		consul = {
			coordinate = {
				update_batch_size = ` + strconv.Itoa(cfg.CoordinateUpdateBatchSize) + `
				update_max_batches = ` + strconv.Itoa(cfg.CoordinateUpdateMaxBatches) + `
				update_period = "` + cfg.CoordinateUpdatePeriod.String() + `"
			}
			raft = {
				election_timeout = "` + raft.ElectionTimeout.String() + `"
				heartbeat_timeout = "` + raft.HeartbeatTimeout.String() + `"
				leader_lease_timeout = "` + raft.LeaderLeaseTimeout.String() + `"
			}
			server = {
				health_interval = "` + cfg.ServerHealthInterval.String() + `"
			}
		}
	`,
	}
}

// DevConsulSource returns the consul agent configuration for the dev mode.
// This should be merged in the tail after the DefaultConsulSource.
func DevConsulSource() Source {
	return Source{
		Name:   "consul-dev",
		Format: "hcl",
		Data: `
		consul = {
			coordinate = {
				update_period = "100ms"
			}
			raft = {
				election_timeout = "52ms"
				heartbeat_timeout = "35ms"
				leader_lease_timeout = "20ms"
			}
			server = {
				health_interval = "10ms"
			}
		}
	`,
	}
}

func DefaultRuntimeConfig(hcl string) *RuntimeConfig {
	b, err := NewBuilder(BuilderOpts{HCL: []string{hcl}})
	if err != nil {
		panic(err)
	}
	rt, err := b.BuildAndValidate()
	if err != nil {
		panic(err)
	}
	return &rt
}
<commit_msg>Add default prefix_filter<commit_after>package config

import (
	"fmt"
	"strconv"

	"github.com/hashicorp/consul/agent/checks"
	"github.com/hashicorp/consul/agent/consul"
	"github.com/hashicorp/consul/version"
	"github.com/hashicorp/raft"
)

// DefaultSource is the default agent configuration.
// This needs to be merged first in the head.
// todo(fs): The values are sourced from multiple sources.
// todo(fs): IMO, this should be the definitive default for all configurable values
// todo(fs): and whatever is in here should clobber every default value. Hence, no sourcing.
func DefaultSource() Source {
	cfg := consul.DefaultConfig()
	serfLAN := cfg.SerfLANConfig.MemberlistConfig
	serfWAN := cfg.SerfWANConfig.MemberlistConfig

	// DEPRECATED (ACL-Legacy-Compat) - when legacy ACL support is removed these defaults
	//   the acl_* config entries here should be transitioned to their counterparts in the
	//   acl stanza for now we need to be able to detect the new entries not being set (not
	//   just set to the defaults here) so that we can use the old entries. So the true
	//   default still needs to reside in the original config values
	return Source{
		Name:   "default",
		Format: "hcl",
		Data: `
		acl_default_policy = "allow"
		acl_down_policy = "extend-cache"
		acl_ttl = "30s"
		acl = {
			policy_ttl = "30s"
		}
		bind_addr = "0.0.0.0"
		bootstrap = false
		bootstrap_expect = 0
		check_output_max_size = ` + strconv.Itoa(checks.DefaultBufSize) + `
		check_update_interval = "5m"
		client_addr = "127.0.0.1"
		datacenter = "` + consul.DefaultDC + `"
		default_query_time = "300s"
		disable_coordinates = false
		disable_host_node_id = true
		disable_remote_exec = true
		domain = "consul."
		encrypt_verify_incoming = true
		encrypt_verify_outgoing = true
		log_level = "INFO"
		max_query_time = "600s"
		primary_gateways_interval = "30s"
		protocol = ` + strconv.Itoa(consul.DefaultRPCProtocol) + `
		retry_interval = "30s"
		retry_interval_wan = "30s"
		server = false
		syslog_facility = "LOCAL0"
		tls_min_version = "tls12"

		// TODO (slackpad) - Until #3744 is done, we need to keep these
		// in sync with agent/consul/config.go.
		autopilot = {
			cleanup_dead_servers = true
			last_contact_threshold = "200ms"
			max_trailing_logs = 250
			server_stabilization_time = "10s"
		}
		gossip_lan = {
			gossip_interval = "` + serfLAN.GossipInterval.String() + `"
			gossip_nodes = ` + strconv.Itoa(serfLAN.GossipNodes) + `
			retransmit_mult = ` + strconv.Itoa(serfLAN.RetransmitMult) + `
			probe_interval = "` + serfLAN.ProbeInterval.String() + `"
			probe_timeout = "` + serfLAN.ProbeTimeout.String() + `"
			suspicion_mult = ` + strconv.Itoa(serfLAN.SuspicionMult) + `
		}
		gossip_wan = {
			gossip_interval = "` + serfWAN.GossipInterval.String() + `"
			gossip_nodes = ` + strconv.Itoa(serfLAN.GossipNodes) + `
			retransmit_mult = ` + strconv.Itoa(serfLAN.RetransmitMult) + `
			probe_interval = "` + serfWAN.ProbeInterval.String() + `"
			probe_timeout = "` + serfWAN.ProbeTimeout.String() + `"
			suspicion_mult = ` + strconv.Itoa(serfWAN.SuspicionMult) + `
		}
		dns_config = {
			allow_stale = true
			a_record_limit = 0
			udp_answer_limit = 3
			max_stale = "87600h"
			recursor_timeout = "2s"
		}
		limits = {
			http_max_conns_per_client = 200
			https_handshake_timeout = "5s"
			rpc_handshake_timeout = "5s"
			rpc_rate = -1
			rpc_max_burst = 1000
			rpc_max_conns_per_client = 100
			kv_max_value_size = ` + strconv.FormatInt(raft.SuggestedMaxDataSize, 10) + `
			txn_max_req_len = ` + strconv.FormatInt(raft.SuggestedMaxDataSize, 10) + `
		}
		performance = {
			leave_drain_time = "5s"
			raft_multiplier = ` + strconv.Itoa(int(consul.DefaultRaftMultiplier)) + `
			rpc_hold_timeout = "7s"
		}
		ports = {
			dns = 8600
			http = 8500
			https = -1
			grpc = -1
			serf_lan = ` + strconv.Itoa(consul.DefaultLANSerfPort) + `
			serf_wan = ` + strconv.Itoa(consul.DefaultWANSerfPort) + `
			server = ` + strconv.Itoa(consul.DefaultRPCPort) + `
			proxy_min_port = 20000
			proxy_max_port = 20255
			sidecar_min_port = 21000
			sidecar_max_port = 21255
			expose_min_port = 21500
			expose_max_port = 21755
		}
		telemetry = {
			metrics_prefix = "consul"
			filter_default = true
			prefix_filter = [ "-consul.api.http" ]
		}

	`,
	}
}

// DevSource is the additional default configuration for dev mode.
// This should be merged in the head after the default configuration.
func DevSource() Source {
	return Source{
		Name:   "dev",
		Format: "hcl",
		Data: `
		bind_addr = "127.0.0.1"
		disable_anonymous_signature = true
		disable_keyring_file = true
		enable_debug = true
		ui = true
		log_level = "DEBUG"
		server = true

		gossip_lan = {
			gossip_interval = "100ms"
			probe_interval = "100ms"
			probe_timeout = "100ms"
			suspicion_mult = 3
		}
		gossip_wan = {
			gossip_interval = "100ms"
			probe_interval = "100ms"
			probe_timeout = "100ms"
			suspicion_mult = 3
		}
		connect = {
			enabled = true
		}
		performance = {
			raft_multiplier = 1
		}
		ports = {
			grpc = 8502
		}
	`,
	}
}

// NonUserSource contains the values the user cannot configure.
// This needs to be merged in the tail.
func NonUserSource() Source {
	return Source{
		Name:   "non-user",
		Format: "hcl",
		Data: `
		acl = {
			disabled_ttl = "120s"
		}
		check_deregister_interval_min = "1m"
		check_reap_interval = "30s"
		ae_interval = "1m"
		sync_coordinate_rate_target = 64
		sync_coordinate_interval_min = "15s"

		# segment_limit is the maximum number of network segments that may be declared.
		segment_limit = 64

		# SegmentNameLimit is the maximum segment name length.
		segment_name_limit = 64

		connect = {
			# 0s causes the value to be ignored and operate without capping
			# the max time before leaf certs can be generated after a roots change.
			test_ca_leaf_root_change_spread = "0s"
		}
	`,
	}
}

// VersionSource creates a config source for the version parameters.
// This should be merged in the tail since these values are not
// user configurable.
func VersionSource(rev, ver, verPre string) Source {
	return Source{
		Name:   "version",
		Format: "hcl",
		Data:   fmt.Sprintf(`revision = %q version = %q version_prerelease = %q`, rev, ver, verPre),
	}
}

// DefaultVersionSource returns the version config source for the embedded
// version numbers.
func DefaultVersionSource() Source {
	return VersionSource(version.GitCommit, version.Version, version.VersionPrerelease)
}

// DefaultConsulSource returns the default configuration for the consul agent.
// This should be merged in the tail since these values are not user configurable.
func DefaultConsulSource() Source {
	cfg := consul.DefaultConfig()
	raft := cfg.RaftConfig
	return Source{
		Name:   "consul",
		Format: "hcl",
		Data: `
		consul = {
			coordinate = {
				update_batch_size = ` + strconv.Itoa(cfg.CoordinateUpdateBatchSize) + `
				update_max_batches = ` + strconv.Itoa(cfg.CoordinateUpdateMaxBatches) + `
				update_period = "` + cfg.CoordinateUpdatePeriod.String() + `"
			}
			raft = {
				election_timeout = "` + raft.ElectionTimeout.String() + `"
				heartbeat_timeout = "` + raft.HeartbeatTimeout.String() + `"
				leader_lease_timeout = "` + raft.LeaderLeaseTimeout.String() + `"
			}
			server = {
				health_interval = "` + cfg.ServerHealthInterval.String() + `"
			}
		}
	`,
	}
}

// DevConsulSource returns the consul agent configuration for the dev mode.
// This should be merged in the tail after the DefaultConsulSource.
func DevConsulSource() Source {
	return Source{
		Name:   "consul-dev",
		Format: "hcl",
		Data: `
		consul = {
			coordinate = {
				update_period = "100ms"
			}
			raft = {
				election_timeout = "52ms"
				heartbeat_timeout = "35ms"
				leader_lease_timeout = "20ms"
			}
			server = {
				health_interval = "10ms"
			}
		}
	`,
	}
}

func DefaultRuntimeConfig(hcl string) *RuntimeConfig {
	b, err := NewBuilder(BuilderOpts{HCL: []string{hcl}})
	if err != nil {
		panic(err)
	}
	rt, err := b.BuildAndValidate()
	if err != nil {
		panic(err)
	}
	return &rt
}
<|endoftext|>


<commit_before><commit_msg>Add 70. Climbing Stairs of golang version<commit_after><|endoftext|>


<commit_before>package data

type LedgerEntrySlice []LedgerEntry

type leBase struct {
	LedgerEntryType   LedgerEntryType
	LedgerIndex       *Hash256 `json:"index,omitempty"`
	PreviousTxnID     *Hash256 `json:",omitempty"`
	PreviousTxnLgrSeq *uint32  `json:",omitempty"`
	Hash              Hash256  `json:"-"`
	Id                Hash256  `json:"-"`
}

type AccountRoot struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	Account       *Account         `json:",omitempty"`
	Sequence      *uint32          `json:",omitempty"`
	Balance       *Value           `json:",omitempty"`
	OwnerCount    *uint32          `json:",omitempty"`
	AccountTxnID  *Hash256         `json:",omitempty"`
	RegularKey    *RegularKey      `json:",omitempty"`
	EmailHash     *Hash128         `json:",omitempty"`
	WalletLocator *Hash256         `json:",omitempty"`
	WalletSize    *uint32          `json:",omitempty"`
	MessageKey    *VariableLength  `json:",omitempty"`
	TickSize      *uint8           `json:",omitempty"`
	TransferRate  *uint32          `json:",omitempty"`
	Domain        *VariableLength  `json:",omitempty"`
	Signers       *VariableLength  `json:",omitempty"`
}

type RippleState struct {
	leBase
	Flags          *LedgerEntryFlag `json:",omitempty"`
	LowLimit       *Amount          `json:",omitempty"`
	HighLimit      *Amount          `json:",omitempty"`
	Balance        *Amount          `json:",omitempty"`
	LowNode        *NodeIndex       `json:",omitempty"`
	HighNode       *NodeIndex       `json:",omitempty"`
	LowQualityIn   *uint32          `json:",omitempty"`
	LowQualityOut  *uint32          `json:",omitempty"`
	HighQualityIn  *uint32          `json:",omitempty"`
	HighQualityOut *uint32          `json:",omitempty"`
}

type Offer struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	Account       *Account         `json:",omitempty"`
	Sequence      *uint32          `json:",omitempty"`
	TakerPays     *Amount          `json:",omitempty"`
	TakerGets     *Amount          `json:",omitempty"`
	BookDirectory *Hash256         `json:",omitempty"`
	BookNode      *NodeIndex       `json:",omitempty"`
	OwnerNode     *NodeIndex       `json:",omitempty"`
	Expiration    *uint32          `json:",omitempty"`
}

type Directory struct {
	leBase
	Flags             *LedgerEntryFlag `json:",omitempty"`
	RootIndex         *Hash256         `json:",omitempty"`
	Indexes           *Vector256       `json:",omitempty"`
	Owner             *Account         `json:",omitempty"`
	TakerPaysCurrency *Hash160         `json:",omitempty"`
	TakerPaysIssuer   *Hash160         `json:",omitempty"`
	TakerGetsCurrency *Hash160         `json:",omitempty"`
	TakerGetsIssuer   *Hash160         `json:",omitempty"`
	ExchangeRate      *ExchangeRate    `json:",omitempty"`
	IndexNext         *NodeIndex       `json:",omitempty"`
	IndexPrevious     *NodeIndex       `json:",omitempty"`
}

type LedgerHashes struct {
	leBase
	Flags               *LedgerEntryFlag `json:",omitempty"`
	FirstLedgerSequence *uint32          `json:",omitempty"`
	LastLedgerSequence  *uint32          `json:",omitempty"`
	Hashes              *Vector256       `json:",omitempty"`
}

type Majority struct {
	Amendment *Hash256 `json:",omitempty"`
	CloseTime *uint32  `json:",omitempty"`
}

type Amendments struct {
	leBase
	Flags      *LedgerEntryFlag `json:",omitempty"`
	Amendments *Vector256       `json:",omitempty"`
	Majorities []Majority       `json:",omitempty"`
}

type FeeSettings struct {
	leBase
	Flags             *LedgerEntryFlag `json:",omitempty"`
	BaseFee           *Uint64Hex       `json:",omitempty"`
	ReferenceFeeUnits *uint32          `json:",omitempty"`
	ReserveBase       *uint32          `json:",omitempty"`
	ReserveIncrement  *uint32          `json:",omitempty"`
}

type Escrow struct {
	leBase
	Flags           *LedgerEntryFlag `json:",omitempty"`
	Account         Account          `json:",omitempty"`
	Destination     Account          `json:",omitempty"`
	Amount          Amount           `json:",omitempty"`
	Condition       *VariableLength  `json:",omitempty"`
	CancelAfter     *uint32          `json:",omitempty"`
	FinishAfter     *uint32          `json:",omitempty"`
	SourceTag       *uint32          `json:",omitempty"`
	DestinationTag  *uint32          `json:",omitempty"`
	OwnerNode       *NodeIndex       `json:",omitempty"`
	DestinationNode *NodeIndex       `json:",omitempty"`
}

type SignerEntry struct {
	Account      *Account `json:",omitempty"`
	SignerWeight *uint16  `json:",omitempty"`
}

type SignerList struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	OwnerNode     *NodeIndex       `json:",omitempty"`
	SignerQuorum  *uint32          `json:",omitempty"`
	SignerEntries []SignerEntry    `json:",omitempty"`
	SignerListID  *uint32          `json:",omitempty"`
}

type Ticket struct {
	leBase
	Flags      *LedgerEntryFlag `json:",omitempty"`
	Account    *Account         `json:",omitempty"`
	Sequence   *uint32          `json:",omitempty"`
	OwnerNode  *NodeIndex       `json:",omitempty"`
	Target     *Account         `json:",omitempty"`
	Expiration *uint32          `json:",omitempty"`
}

type PayChannel struct {
	leBase
	Flags          *LedgerEntryFlag `json:",omitempty"`
	Account        *Account         `json:",omitempty"`
	Destination    *Account         `json:",omitempty"`
	Amount         *Amount          `json:",omitempty"`
	Balance        *Amount          `json:",omitempty"`
	PublicKey      *PublicKey       `json:",omitempty"`
	SettleDelay    *uint32          `json:",omitempty"`
	OwnerNode      *NodeIndex       `json:",omitempty"`
	Expiration     *uint32          `json:",omitempty"`
	CancelAfter    *uint32          `json:",omitempty"`
	DestinationTag *uint32          `json:",omitempty"`
	SourceTag      *uint32          `json:",omitempty"`
}

type Check struct {
	leBase
	Flags           *LedgerEntryFlag `json:",omitempty"`
	Account         *Account         `json:",omitempty"`
	Destination     *Account         `json:",omitempty"`
	OwnerNode       *NodeIndex       `json:",omitempty"`
	SendMax         *Amount          `json:",omitempty"`
	Sequence        *uint32          `json:",omitempty"`
	DestinationNode *NodeIndex       `json:",omitempty"`
	DestinationTag  *uint32          `json:",omitempty"`
	Expiration      *uint32          `json:",omitempty"`
	SourceTag       *uint32          `json:",omitempty"`
	InvoiceID       *Hash256         `json:",omitempty"`
}

type DepositPreAuth struct {
	leBase
	Account   *Account         `json:",omitempty"`
	Authorize *Account         `json:",omitempty"`
	Flags     *LedgerEntryFlag `json:",omitempty"`
	OwnerNode *NodeIndex       `json:",omitempty"`
}

func (a *AccountRoot) Affects(account Account) bool {
	return a.Account != nil && a.Account.Equals(account)
}
func (r *RippleState) Affects(account Account) bool {
	return r.LowLimit.Issuer.Equals(account) || r.HighLimit.Issuer.Equals(account)
}
func (o *Offer) Affects(account Account) bool        { return o.Account.Equals(account) }
func (d *Directory) Affects(account Account) bool    { return false }
func (l *LedgerHashes) Affects(account Account) bool { return false }
func (a *Amendments) Affects(account Account) bool   { return false }
func (f *FeeSettings) Affects(account Account) bool  { return false }
func (s *Escrow) Affects(account Account) bool {
	return s.Account.Equals(account) || s.Destination.Equals(account)
}
func (s *SignerList) Affects(account Account) bool {
	for _, entry := range s.SignerEntries {
		if entry.Account != nil && entry.Account.Equals(account) {
			return true
		}
	}
	return false
}
func (t *Ticket) Affects(account Account) bool { return t.Account != nil && t.Account.Equals(account) }
func (p *PayChannel) Affects(account Account) bool {
	return (p.Account != nil && p.Account.Equals(account)) || (p.Destination != nil && p.Destination.Equals(account))
}
func (p *Check) Affects(account Account) bool {
	return (p.Account != nil && p.Account.Equals(account)) || (p.Destination != nil && p.Destination.Equals(account))
}

func (d *DepositPreAuth) Affects(account Account) bool {
	return (d.Account != nil && d.Account.Equals(account)) || (d.Authorize != nil && d.Authorize.Equals(account))
}

func (le *leBase) GetType() string                     { return ledgerEntryNames[le.LedgerEntryType] }
func (le *leBase) GetLedgerEntryType() LedgerEntryType { return le.LedgerEntryType }
func (le *leBase) Prefix() HashPrefix                  { return HP_LEAF_NODE }
func (le *leBase) NodeType() NodeType                  { return NT_ACCOUNT_NODE }
func (le *leBase) Ledger() uint32                      { return 0 }
func (le *leBase) GetHash() *Hash256                   { return &le.Hash }
func (le *leBase) NodeId() *Hash256                    { return &le.Id }
func (le *leBase) GetLedgerIndex() *Hash256            { return le.LedgerIndex }
func (le *leBase) GetPreviousTxnId() *Hash256          { return le.PreviousTxnID }

func (o *Offer) Ratio() *Value {
	return o.TakerPays.Ratio(*o.TakerGets)
}
<commit_msg>Update ledgerentry.go<commit_after>package data

type LedgerEntrySlice []LedgerEntry

type leBase struct {
	LedgerEntryType   LedgerEntryType
	LedgerIndex       *Hash256 `json:"index,omitempty"`
	PreviousTxnID     *Hash256 `json:",omitempty"`
	PreviousTxnLgrSeq *uint32  `json:",omitempty"`
	Hash              Hash256  `json:"-"`
	Id                Hash256  `json:"-"`
}

type AccountRoot struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	Account       *Account         `json:",omitempty"`
	Sequence      *uint32          `json:",omitempty"`
	Balance       *Value           `json:",omitempty"`
	OwnerCount    *uint32          `json:",omitempty"`
	AccountTxnID  *Hash256         `json:",omitempty"`
	RegularKey    *RegularKey      `json:",omitempty"`
	EmailHash     *Hash128         `json:",omitempty"`
	WalletLocator *Hash256         `json:",omitempty"`
	WalletSize    *uint32          `json:",omitempty"`
	MessageKey    *VariableLength  `json:",omitempty"`
	TickSize      *uint8           `json:",omitempty"`
	TransferRate  *uint32          `json:",omitempty"`
	Domain        *VariableLength  `json:",omitempty"`
	Signers       *VariableLength  `json:",omitempty"`
}

type RippleState struct {
	leBase
	Flags          *LedgerEntryFlag `json:",omitempty"`
	LowLimit       *Amount          `json:",omitempty"`
	HighLimit      *Amount          `json:",omitempty"`
	Balance        *Amount          `json:",omitempty"`
	LowNode        *NodeIndex       `json:",omitempty"`
	HighNode       *NodeIndex       `json:",omitempty"`
	LowQualityIn   *uint32          `json:",omitempty"`
	LowQualityOut  *uint32          `json:",omitempty"`
	HighQualityIn  *uint32          `json:",omitempty"`
	HighQualityOut *uint32          `json:",omitempty"`
}

type Offer struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	Account       *Account         `json:",omitempty"`
	Sequence      *uint32          `json:",omitempty"`
	TakerPays     *Amount          `json:",omitempty"`
	TakerGets     *Amount          `json:",omitempty"`
	BookDirectory *Hash256         `json:",omitempty"`
	BookNode      *NodeIndex       `json:",omitempty"`
	OwnerNode     *NodeIndex       `json:",omitempty"`
	Expiration    *uint32          `json:",omitempty"`
}

type Directory struct {
	leBase
	Flags             *LedgerEntryFlag `json:",omitempty"`
	RootIndex         *Hash256         `json:",omitempty"`
	Indexes           *Vector256       `json:",omitempty"`
	Owner             *Account         `json:",omitempty"`
	TakerPaysCurrency *Hash160         `json:",omitempty"`
	TakerPaysIssuer   *Hash160         `json:",omitempty"`
	TakerGetsCurrency *Hash160         `json:",omitempty"`
	TakerGetsIssuer   *Hash160         `json:",omitempty"`
	ExchangeRate      *ExchangeRate    `json:",omitempty"`
	IndexNext         *NodeIndex       `json:",omitempty"`
	IndexPrevious     *NodeIndex       `json:",omitempty"`
}

type LedgerHashes struct {
	leBase
	Flags               *LedgerEntryFlag `json:",omitempty"`
	FirstLedgerSequence *uint32          `json:",omitempty"`
	LastLedgerSequence  *uint32          `json:",omitempty"`
	Hashes              *Vector256       `json:",omitempty"`
}

type Majority struct {
	Amendment *Hash256 `json:",omitempty"`
	CloseTime *uint32  `json:",omitempty"`
}

type Amendments struct {
	leBase
	Flags      *LedgerEntryFlag `json:",omitempty"`
	Amendments *Vector256       `json:",omitempty"`
	Majorities []Majority       `json:",omitempty"`
}

type FeeSettings struct {
	leBase
	Flags             *LedgerEntryFlag `json:",omitempty"`
	BaseFee           *Uint64Hex       `json:",omitempty"`
	ReferenceFeeUnits *uint32          `json:",omitempty"`
	ReserveBase       *uint32          `json:",omitempty"`
	ReserveIncrement  *uint32          `json:",omitempty"`
}

type Escrow struct {
	leBase
	Flags           *LedgerEntryFlag `json:",omitempty"`
	Account         Account          `json:",omitempty"`
	Destination     Account          `json:",omitempty"`
	Amount          Amount           `json:",omitempty"`
	Condition       *VariableLength  `json:",omitempty"`
	CancelAfter     *uint32          `json:",omitempty"`
	FinishAfter     *uint32          `json:",omitempty"`
	SourceTag       *uint32          `json:",omitempty"`
	DestinationTag  *uint32          `json:",omitempty"`
	OwnerNode       *NodeIndex       `json:",omitempty"`
	DestinationNode *NodeIndex       `json:",omitempty"`
}

type SignerEntry struct {
	Account      *Account `json:",omitempty"`
	SignerWeight *uint16  `json:",omitempty"`
}

type SignerList struct {
	leBase
	Flags         *LedgerEntryFlag `json:",omitempty"`
	OwnerNode     *NodeIndex       `json:",omitempty"`
	SignerQuorum  *uint32          `json:",omitempty"`
	SignerEntries []SignerEntry    `json:",omitempty"`
	SignerListID  *uint32          `json:",omitempty"`
}

type Ticket struct {
	leBase
	Flags      *LedgerEntryFlag `json:",omitempty"`
	Account    *Account         `json:",omitempty"`
	Sequence   *uint32          `json:",omitempty"`
	OwnerNode  *NodeIndex       `json:",omitempty"`
	Target     *Account         `json:",omitempty"`
	Expiration *uint32          `json:",omitempty"`
}

type PayChannel struct {
	leBase
	Flags           *LedgerEntryFlag `json:",omitempty"`
	Account         *Account         `json:",omitempty"`
	Destination     *Account         `json:",omitempty"`
	Amount          *Amount          `json:",omitempty"`
	Balance         *Amount          `json:",omitempty"`
	PublicKey       *PublicKey       `json:",omitempty"`
	SettleDelay     *uint32          `json:",omitempty"`
	OwnerNode       *NodeIndex       `json:",omitempty"`
	DestinationNode *NodeIndex       `json:",omitempty"`
	Expiration      *uint32          `json:",omitempty"`
	CancelAfter     *uint32          `json:",omitempty"`
	DestinationTag  *uint32          `json:",omitempty"`
	SourceTag       *uint32          `json:",omitempty"`
}

type Check struct {
	leBase
	Flags           *LedgerEntryFlag `json:",omitempty"`
	Account         *Account         `json:",omitempty"`
	Destination     *Account         `json:",omitempty"`
	OwnerNode       *NodeIndex       `json:",omitempty"`
	SendMax         *Amount          `json:",omitempty"`
	Sequence        *uint32          `json:",omitempty"`
	DestinationNode *NodeIndex       `json:",omitempty"`
	DestinationTag  *uint32          `json:",omitempty"`
	Expiration      *uint32          `json:",omitempty"`
	SourceTag       *uint32          `json:",omitempty"`
	InvoiceID       *Hash256         `json:",omitempty"`
}

type DepositPreAuth struct {
	leBase
	Account   *Account         `json:",omitempty"`
	Authorize *Account         `json:",omitempty"`
	Flags     *LedgerEntryFlag `json:",omitempty"`
	OwnerNode *NodeIndex       `json:",omitempty"`
}

func (a *AccountRoot) Affects(account Account) bool {
	return a.Account != nil && a.Account.Equals(account)
}
func (r *RippleState) Affects(account Account) bool {
	return r.LowLimit.Issuer.Equals(account) || r.HighLimit.Issuer.Equals(account)
}
func (o *Offer) Affects(account Account) bool        { return o.Account.Equals(account) }
func (d *Directory) Affects(account Account) bool    { return false }
func (l *LedgerHashes) Affects(account Account) bool { return false }
func (a *Amendments) Affects(account Account) bool   { return false }
func (f *FeeSettings) Affects(account Account) bool  { return false }
func (s *Escrow) Affects(account Account) bool {
	return s.Account.Equals(account) || s.Destination.Equals(account)
}
func (s *SignerList) Affects(account Account) bool {
	for _, entry := range s.SignerEntries {
		if entry.Account != nil && entry.Account.Equals(account) {
			return true
		}
	}
	return false
}
func (t *Ticket) Affects(account Account) bool { return t.Account != nil && t.Account.Equals(account) }
func (p *PayChannel) Affects(account Account) bool {
	return (p.Account != nil && p.Account.Equals(account)) || (p.Destination != nil && p.Destination.Equals(account))
}
func (p *Check) Affects(account Account) bool {
	return (p.Account != nil && p.Account.Equals(account)) || (p.Destination != nil && p.Destination.Equals(account))
}

func (d *DepositPreAuth) Affects(account Account) bool {
	return (d.Account != nil && d.Account.Equals(account)) || (d.Authorize != nil && d.Authorize.Equals(account))
}

func (le *leBase) GetType() string                     { return ledgerEntryNames[le.LedgerEntryType] }
func (le *leBase) GetLedgerEntryType() LedgerEntryType { return le.LedgerEntryType }
func (le *leBase) Prefix() HashPrefix                  { return HP_LEAF_NODE }
func (le *leBase) NodeType() NodeType                  { return NT_ACCOUNT_NODE }
func (le *leBase) Ledger() uint32                      { return 0 }
func (le *leBase) GetHash() *Hash256                   { return &le.Hash }
func (le *leBase) NodeId() *Hash256                    { return &le.Id }
func (le *leBase) GetLedgerIndex() *Hash256            { return le.LedgerIndex }
func (le *leBase) GetPreviousTxnId() *Hash256          { return le.PreviousTxnID }

func (o *Offer) Ratio() *Value {
	return o.TakerPays.Ratio(*o.TakerGets)
}
<|endoftext|>


<commit_before>// Copyright 2013 @atotto. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package clipboard

import (
	"os/exec"
)

const (
	xsel  = "xsel"
	xclip = "xclip"
)

var (
	pasteCmd, copyCmd *exec.Cmd

	xselPaste = exec.Command(xsel, "--output", "--clipboard")
	xselCopy  = exec.Command(xsel, "--input", "--clipboard")

	xclipPaste = exec.Command(xclip, "-out", "-sel", "clipboard")
	xclipCopy  = exec.Command(xclip, "-in", "-sel", "clipboard")
)

func init() {
	pasteCmd = xselPaste
	copyCmd = xselCopy

	_, err := exec.LookPath(xsel)
	if err == nil {
		return
	}

	pasteCmd = xclipPaste
	copyCmd = xclipCopy

	_, err = exec.LookPath(xclip)
	if err == nil {
		return
	}

	println("No clipboard utilities available. Please install xset or xclip.")
}

func readAll() (string, error) {
	out, err := pasteCmd.Output()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func writeAll(text string) error {
	in, err := copyCmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := copyCmd.Start(); err != nil {
		return err
	}
	if _, err := in.Write([]byte(text)); err != nil {
		return err
	}
	if err := in.Close(); err != nil {
		return err
	}
	return copyCmd.Wait()
}
<commit_msg>use full selection flag<commit_after>// Copyright 2013 @atotto. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package clipboard

import (
	"os/exec"
)

const (
	xsel  = "xsel"
	xclip = "xclip"
)

var (
	pasteCmd, copyCmd *exec.Cmd

	xselPaste = exec.Command(xsel, "--output", "--clipboard")
	xselCopy  = exec.Command(xsel, "--input", "--clipboard")

	xclipPaste = exec.Command(xclip, "-out", "-selection", "clipboard")
	xclipCopy  = exec.Command(xclip, "-in", "-selection", "clipboard")
)

func init() {
	pasteCmd = xselPaste
	copyCmd = xselCopy

	_, err := exec.LookPath(xsel)
	if err == nil {
		return
	}

	pasteCmd = xclipPaste
	copyCmd = xclipCopy

	_, err = exec.LookPath(xclip)
	if err == nil {
		return
	}

	println("No clipboard utilities available. Please install xset or xclip.")
}

func readAll() (string, error) {
	out, err := pasteCmd.Output()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func writeAll(text string) error {
	in, err := copyCmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := copyCmd.Start(); err != nil {
		return err
	}
	if _, err := in.Write([]byte(text)); err != nil {
		return err
	}
	if err := in.Close(); err != nil {
		return err
	}
	return copyCmd.Wait()
}
<|endoftext|>


<commit_before>package bagit

// Defaults are default tag values to use when creating bags.
// For example, the value Source-Organization tag will be the
// same for every bag an organization creates. Defaults allow
// us to set these values once.
type Defaults struct {
	// Path is the path to the file that contains the default tag data.
	// The file should be in JSON format, with a single object that
	// contains name-value pairs.
	Path string
	// Values are the default values parsed from the default.json file.
	Values map[string]string
}
<commit_msg>Removed unused defaults.go<commit_after><|endoftext|>