<commit_before>// Code for genetic algorithms
package genetic

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kevinburke/rct/Godeps/_workspace/src/github.com/pborman/uuid"

	"github.com/kevinburke/rct/tracks"
)

var directory *string

func init() {
	directory = flag.String("directory", "/usr/local/rct", "Path to the folder storing RCT experiment data")
}

// constants which may be altered to affect the ride runtime
const PARENTS = 2
const FIT_PERCENTAGE = 0.2
const MUTATION_RATE = 0.05

// crossover with a probability of 0.6 (taken from the book & De Jong 1975)
const CROSSOVER_PROBABILITY = 0.6
const POOL_SIZE = 5
const ITERATIONS = 6
const PRINT_RESULTS_EVERY = 1

type ExperimentMetadata struct {
	Hash                 string
	Date                 time.Time
	Notes                string
	Runtime              time.Duration
	PoolSize             int16
	Iterations           int32
	CrossoverProbability float32
	MutationRate         float32
}

func Run(packageRoot string) error {
	if directory == nil {
		return fmt.Errorf("invalid directory - need to specify it")
	}
	expDir := path.Join(*directory, "experiments")
	err := os.MkdirAll(expDir, 0755)
	if err != nil {
		return err
	}
	id := fmt.Sprintf("exp_%s", uuid.New())
	expIdDir := path.Join(expDir, id)
	err = os.MkdirAll(expIdDir, 0755)
	if err != nil {
		return err
	}
	iterationsDir := path.Join(expIdDir, "iterations")
	err = os.MkdirAll(iterationsDir, 0755)
	if err != nil {
		return err
	}
	cmd := exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = packageRoot
	hashb, err := cmd.Output()
	if err != nil {
		return err
	}
	mtd := ExperimentMetadata{
		Hash:                 strings.TrimSpace(string(hashb)),
		Date:                 time.Now().UTC(),
		Notes:                "(none)", // XXX
		CrossoverProbability: CROSSOVER_PROBABILITY,
		PoolSize:             POOL_SIZE,
		MutationRate:         MUTATION_RATE,
		Iterations:           ITERATIONS,
	}
	metadataPath := path.Join(expIdDir, "meta.json")
	err = encode(metadataPath, mtd)
	if err != nil {
		return err
	}
	fmt.Printf("Experiment %s\n======================================\n", id)
	pool := SeedPool(POOL_SIZE)
	pool.Id = id
	for i := 0; i < ITERATIONS; i++ {
		pool = pool.Crossover()
		pool.Mutate(MUTATION_RATE)
		pool.Evaluate()
		iterationDir := path.Join(iterationsDir, strconv.Itoa(i))
		err = os.MkdirAll(iterationDir, 0755)
		if err != nil {
			return err
		}
		pool.Statistics(i, *directory)
	}
	return nil
}

type Pool struct {
	Id      string
	Members []*Member
}

type Member struct {
	Id    string
	Score int64
	// Advantage or disadvantage in reproducing
	Fitness   float64
	Runtime   time.Duration
	Track     []tracks.Element
	ScoreData scoreData
}

type scoresArray [500]int64

func (a *scoresArray) Len() int {
	return len(a)
}

func (a *scoresArray) Swap(i int, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a *scoresArray) Less(i, j int) bool {
	return a[i] < a[j]
}

func (p *Pool) Statistics(iteration int, outputDirectory string) {
	if iteration%PRINT_RESULTS_EVERY == 0 {
		var scores scoresArray
		var highestScore int64 = -1
		var worstScore int64 = 100 * 1000 * 1000
		bestMember := new(Member)
		for i := 0; i < len(p.Members); i++ {
			scores[i] = p.Members[i].Score
			if p.Members[i].Score > highestScore {
				highestScore = p.Members[i].Score
				bestMember = p.Members[i]
			}
			if p.Members[i].Score < worstScore {
				worstScore = p.Members[i].Score
			}
		}
		middle := len(scores) / 2
		median := (scores[middle] + scores[middle-1]) / 2
		sort.Sort(&scores)
		bestScorer := fmt.Sprintf("\t(collisions: %d, to completion: %d, negative speed points: %d)\n\n",
			bestMember.ScoreData.Collisions, bestMember.ScoreData.Distance,
			bestMember.ScoreData.NegativeSpeed)
		fmt.Printf("Iteration %d: %d members, best member %s has score %d, "+
			"median %d, worst has score %d\n%s",
			iteration, len(p.Members), bestMember.Id, bestMember.Score, median,
			worstScore, bestScorer)
		//if iteration%20 == 0 && iteration > 0 {
		//for _, member := range p.Members {
		//for _, elem := range member.Track {
		//fmt.Println(elem.Segment.String())
		//}
		//fmt.Println("==================")
		//}
		//}
	}

	var wg sync.WaitGroup
	for _, member := range p.Members {
		wg.Add(1)
		go func(member *Member) {
			pth := path.Join(outputDirectory, "experiments", p.Id,
				"iterations", strconv.Itoa(iteration), fmt.Sprintf("%s.json", member.Id))
			err := encode(pth, member)
			if err != nil {
				log.Print(err)
			}
			wg.Done()
		}(member)
	}
	wg.Wait()
}

// Create an initial pool
func SeedPool(size int) *Pool {
	// 1. Create a station of length 10
	// 2. For the start station piece, generate a list of possible pieces.
	// 3. Choose one at random. Advance a pointer one forward.
	// 4. Repeat for 50 pieces (Woodchip is length 108. Mischief is 123)
	members := make([]*Member, POOL_SIZE)
	for i := 0; i < POOL_SIZE; i++ {
		track := CreateStation()
		for j := STATION_LENGTH - 1; j < INITIAL_TRACK_LENGTH-STATION_LENGTH; j++ {
			poss := track[j].Possibilities()
			track = append(track, poss[rand.Intn(len(poss))])
		}
		score, d := GetScore(track)
		members[i] = &Member{
			Id:        fmt.Sprintf("iter_%s", uuid.New()),
			Track:     track,
			Score:     score,
			ScoreData: d,
		}
		//if i%100 == 0 {
		//for _, elem := range track {
		//fmt.Println(elem.Segment.String())
		//}
		//fmt.Printf("========\n")
		//}
	}
	return &Pool{Members: members}
}

func (p *Pool) Mutate(rate float64) {
	// for each ride:
	// for each position in the ride:
	// add in a possibility of mutation - addition
	// if addition:
	//	find a piece that is compatible with both ends.
	//  if no piece:
	//	  advance to the next track piece and try again
}

// Assign scores for every member of the pool
func (p *Pool) Evaluate() {
	var wg sync.WaitGroup
	for i := 0; i < POOL_SIZE; i++ {
		wg.Add(1)
		go func(i int, track []tracks.Element) {
			p.Members[i].Score, p.Members[i].ScoreData = GetScore(track)
			wg.Done()
		}(i, p.Members[i].Track)
	}
	wg.Wait()

	// Assign fitness for every member. In the future, consider a smarter
	// algorithm higher score members a better chance of reproducing.
	for i := 0; i < POOL_SIZE; i++ {
		fmt.Println(p.Members[i].Score)
		p.Members[i].Fitness = float64(p.Members[i].Score)
	}
}

// Select chooses a member of the population at random
func (p *Pool) Select() (int, *Member) {
	// Stupid dumb version of this taken from here:
	// http://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python
	// If it's a bottleneck, rewrite it.
	var weightedTotal float64 = 0
	totals := make([]float64, len(p.Members))
	for i := 0; i < len(p.Members); i++ {
		weightedTotal += p.Members[i].Fitness
		totals[i] = weightedTotal
	}
	rnd := rand.Float64() * weightedTotal
	for index, element := range totals {
		if rnd < element {
			return index, p.Members[index]
		}
	}
	return -1, &Member{}
}

func min(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

// crossPoint1 = splice point in parent1.
func crossoverAtPoint(parent1 *Member, parent2 *Member, crossPoint1 int) (*Member, *Member) {
	// crossPoint2 = splice point in parent2.
	crossPoint2 := crossPoint1 + 1
	foundMatch := false
	for {
		// say cross point 1 = 3, track length = 6
		// parent 1 == 0-3, parent 2 = 4-5
		if tracks.Compatible(parent1.Track[crossPoint1], parent2.Track[crossPoint2]) {
			// Increment so we splice *after* the cross point in parent 1
			crossPoint1++
			foundMatch = true
			break
		}
		crossPoint1++
		if crossPoint1 >= len(parent1.Track) {
			break
		}
		if tracks.Compatible(parent1.Track[crossPoint1], parent2.Track[crossPoint2]) {
			foundMatch = true
			crossPoint2++
			break
		}
		crossPoint2++
		if crossPoint2 >= len(parent2.Track) {
			break
		}
	}
	//	swap the track pieces at the chosen point on track A and track B
	if foundMatch {
		c1, c2 := Swap(parent1, parent2, crossPoint1, crossPoint2)
		if crossPoint1 > 4 {
			fmt.Println("swapped at", crossPoint1, crossPoint2)
			fmt.Println("parent1")
			printTrack(parent1.Track)
			fmt.Println("parent2")
			printTrack(parent2.Track)
			fmt.Println("child1")
			printTrack(c1.Track)
			fmt.Println("child2")
			printTrack(c2.Track)
		}
		return c1, c2
	}
	return parent1, parent2
}

// crossoverOne crosses over the parent tracks. if point is -1, a random point
// is chosen. (point is used for testing)
func crossoverOne(parent1 *Member, parent2 *Member, point int) (*Member, *Member) {
	//	choose a random point between the beginning and the end
	if point == -1 {
		minval := min(len(parent1.Track), len(parent2.Track))
		point = rand.Intn(minval)
	}
	return crossoverAtPoint(parent1, parent2, point)
}

func printTrack(t []tracks.Element) {
	for i, elem := range t {
		fmt.Printf("%d %s\n", i, elem.Segment.String())
	}
}

// Crossover chooses two members of a pool and joins them at random.
func (p *Pool) Crossover() *Pool {
	halfLen := len(p.Members) / 2
	for i := 0; i < halfLen; i++ {
		// select 2 parents at random
		idx1, parent1 := p.Select()
		idx2, parent2 := p.Select()
		if idx1 == -1 || idx2 == -1 {
			continue
		}
		if idx1 == idx2 {
			// No point in crossing over with ourself
			continue
		}
		if rand.Float64() < CROSSOVER_PROBABILITY {
			// XXX delete parents
			child1, child2 := crossoverOne(parent1, parent2, -1)
			p.Members[idx1] = child1
			p.Members[idx2] = child2
		}
	}
	return p
}

// Swap creates two children out of the parents, by crossing over the tracks at
// the given cross points. The sum of the two track lengths may be the same,
// but the tracks themselves will change.
func Swap(parent1 *Member, parent2 *Member, crossPoint1 int, crossPoint2 int) (*Member, *Member) {
	child1len := crossPoint1 + (len(parent2.Track) - crossPoint2)
	child2len := crossPoint2 + (len(parent1.Track) - crossPoint1)

	child1track := make([]tracks.Element, child1len)
	child2track := make([]tracks.Element, child2len)

	copy(child1track[:crossPoint1], parent1.Track[:crossPoint1])
	// if we filled in to child1 from [crossPoint2:] we might have gaps in the
	// track. fill from the end of cross point 1, with the contents of cross
	// point 2.
	copy(child1track[crossPoint1:], parent2.Track[crossPoint2:])

	copy(child2track[:crossPoint2], parent2.Track[:crossPoint2])
	copy(child2track[crossPoint2:], parent1.Track[crossPoint1:])
	child1 := &Member{
		Id:    fmt.Sprintf("iter_%s", uuid.New()),
		Track: child1track,
	}
	child2 := &Member{
		Id:    fmt.Sprintf("iter_%s", uuid.New()),
		Track: child2track,
	}
	return child1, child2
}

func (p *Pool) Spawn(numParents int) *Pool {
	return nil
}
<commit_msg>Add comment<commit_after>// Code for genetic algorithms
package genetic

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kevinburke/rct/Godeps/_workspace/src/github.com/pborman/uuid"

	"github.com/kevinburke/rct/tracks"
)

var directory *string

func init() {
	directory = flag.String("directory", "/usr/local/rct", "Path to the folder storing RCT experiment data")
}

// constants which may be altered to affect the ride runtime
const PARENTS = 2
const FIT_PERCENTAGE = 0.2
const MUTATION_RATE = 0.05

// crossover with a probability of 0.6 (taken from the book & De Jong 1975)
const CROSSOVER_PROBABILITY = 0.6
const POOL_SIZE = 5
const ITERATIONS = 6
const PRINT_RESULTS_EVERY = 1

type ExperimentMetadata struct {
	Hash                 string
	Date                 time.Time
	Notes                string
	Runtime              time.Duration
	PoolSize             int16
	Iterations           int32
	CrossoverProbability float32
	MutationRate         float32
}

func Run(packageRoot string) error {
	if directory == nil {
		return fmt.Errorf("invalid directory - need to specify it")
	}
	expDir := path.Join(*directory, "experiments")
	err := os.MkdirAll(expDir, 0755)
	if err != nil {
		return err
	}
	id := fmt.Sprintf("exp_%s", uuid.New())
	expIdDir := path.Join(expDir, id)
	err = os.MkdirAll(expIdDir, 0755)
	if err != nil {
		return err
	}
	iterationsDir := path.Join(expIdDir, "iterations")
	err = os.MkdirAll(iterationsDir, 0755)
	if err != nil {
		return err
	}
	cmd := exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = packageRoot
	hashb, err := cmd.Output()
	if err != nil {
		return err
	}
	mtd := ExperimentMetadata{
		Hash:                 strings.TrimSpace(string(hashb)),
		Date:                 time.Now().UTC(),
		Notes:                "(none)", // XXX
		CrossoverProbability: CROSSOVER_PROBABILITY,
		PoolSize:             POOL_SIZE,
		MutationRate:         MUTATION_RATE,
		Iterations:           ITERATIONS,
	}
	metadataPath := path.Join(expIdDir, "meta.json")
	err = encode(metadataPath, mtd)
	if err != nil {
		return err
	}
	fmt.Printf("Experiment %s\n======================================\n", id)
	pool := SeedPool(POOL_SIZE)
	pool.Id = id
	for i := 0; i < ITERATIONS; i++ {
		pool = pool.Crossover()
		pool.Mutate(MUTATION_RATE)
		pool.Evaluate()
		iterationDir := path.Join(iterationsDir, strconv.Itoa(i))
		err = os.MkdirAll(iterationDir, 0755)
		if err != nil {
			return err
		}
		pool.Statistics(i, *directory)
	}
	return nil
}

type Pool struct {
	Id      string
	Members []*Member
}

type Member struct {
	Id    string
	Score int64
	// Advantage or disadvantage in reproducing
	Fitness   float64
	Runtime   time.Duration
	Track     []tracks.Element
	ScoreData scoreData
}

type scoresArray [500]int64

func (a *scoresArray) Len() int {
	return len(a)
}

func (a *scoresArray) Swap(i int, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a *scoresArray) Less(i, j int) bool {
	return a[i] < a[j]
}

func (p *Pool) Statistics(iteration int, outputDirectory string) {
	if iteration%PRINT_RESULTS_EVERY == 0 {
		var scores scoresArray
		var highestScore int64 = -1
		var worstScore int64 = 100 * 1000 * 1000
		bestMember := new(Member)
		for i := 0; i < len(p.Members); i++ {
			scores[i] = p.Members[i].Score
			if p.Members[i].Score > highestScore {
				highestScore = p.Members[i].Score
				bestMember = p.Members[i]
			}
			if p.Members[i].Score < worstScore {
				worstScore = p.Members[i].Score
			}
		}
		middle := len(scores) / 2
		median := (scores[middle] + scores[middle-1]) / 2
		sort.Sort(&scores)
		bestScorer := fmt.Sprintf("\t(collisions: %d, to completion: %d, negative speed points: %d)\n\n",
			bestMember.ScoreData.Collisions, bestMember.ScoreData.Distance,
			bestMember.ScoreData.NegativeSpeed)
		fmt.Printf("Iteration %d: %d members, best member %s has score %d, "+
			"median %d, worst has score %d\n%s",
			iteration, len(p.Members), bestMember.Id, bestMember.Score, median,
			worstScore, bestScorer)
		//if iteration%20 == 0 && iteration > 0 {
		//for _, member := range p.Members {
		//for _, elem := range member.Track {
		//fmt.Println(elem.Segment.String())
		//}
		//fmt.Println("==================")
		//}
		//}
	}

	var wg sync.WaitGroup
	for _, member := range p.Members {
		wg.Add(1)
		go func(member *Member) {
			pth := path.Join(outputDirectory, "experiments", p.Id,
				"iterations", strconv.Itoa(iteration), fmt.Sprintf("%s.json", member.Id))
			err := encode(pth, member)
			if err != nil {
				log.Print(err)
			}
			wg.Done()
		}(member)
	}
	wg.Wait()
}

// Create an initial pool
func SeedPool(size int) *Pool {
	// 1. Create a station of length 10
	// 2. For the start station piece, generate a list of possible pieces.
	// 3. Choose one at random. Advance a pointer one forward.
	// 4. Repeat for 50 pieces (Woodchip is length 108. Mischief is 123)
	members := make([]*Member, POOL_SIZE)
	for i := 0; i < POOL_SIZE; i++ {
		track := CreateStation()
		for j := STATION_LENGTH - 1; j < INITIAL_TRACK_LENGTH-STATION_LENGTH; j++ {
			poss := track[j].Possibilities()
			track = append(track, poss[rand.Intn(len(poss))])
		}
		score, d := GetScore(track)
		members[i] = &Member{
			Id:        fmt.Sprintf("iter_%s", uuid.New()),
			Track:     track,
			Score:     score,
			ScoreData: d,
		}
		//if i%100 == 0 {
		//for _, elem := range track {
		//fmt.Println(elem.Segment.String())
		//}
		//fmt.Printf("========\n")
		//}
	}
	return &Pool{Members: members}
}

func (p *Pool) Mutate(rate float64) {
	// for each ride:
	// for each position in the ride:
	// add in a possibility of mutation - addition
	// if addition:
	//	find a piece that is compatible with both ends.
	//  if no piece:
	//	  advance to the next track piece and try again
}

// Assign scores for every member of the pool
func (p *Pool) Evaluate() {
	var wg sync.WaitGroup
	for i := 0; i < POOL_SIZE; i++ {
		wg.Add(1)
		go func(i int, track []tracks.Element) {
			p.Members[i].Score, p.Members[i].ScoreData = GetScore(track)
			wg.Done()
		}(i, p.Members[i].Track)
	}
	wg.Wait()

	// Assign fitness for every member. In the future, consider a smarter
	// algorithm higher score members a better chance of reproducing.
	for i := 0; i < POOL_SIZE; i++ {
		fmt.Println(p.Members[i].Score)
		p.Members[i].Fitness = float64(p.Members[i].Score)
	}
}

// Select chooses a member of the population at random
func (p *Pool) Select() (int, *Member) {
	// Stupid dumb version of this taken from here:
	// http://eli.thegreenplace.net/2010/01/22/weighted-random-generation-in-python
	// If it's a bottleneck, rewrite it.
	var weightedTotal float64 = 0
	totals := make([]float64, len(p.Members))
	for i := 0; i < len(p.Members); i++ {
		weightedTotal += p.Members[i].Fitness
		totals[i] = weightedTotal
	}
	rnd := rand.Float64() * weightedTotal
	for index, element := range totals {
		if rnd < element {
			return index, p.Members[index]
		}
	}
	return -1, &Member{}
}

func min(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

// crossPoint1 = splice point in parent1.
func crossoverAtPoint(parent1 *Member, parent2 *Member, crossPoint1 int) (*Member, *Member) {
	// crossPoint2 = splice point in parent2.
	crossPoint2 := crossPoint1 + 1
	foundMatch := false
	for {
		// say cross point 1 = 3, track length = 6
		// parent 1 == 0-3, parent 2 = 4-5
		if tracks.Compatible(parent1.Track[crossPoint1], parent2.Track[crossPoint2]) {
			// Increment so we splice *after* the cross point in parent 1
			crossPoint1++
			foundMatch = true
			break
		}
		crossPoint1++
		if crossPoint1 >= len(parent1.Track) {
			break
		}
		if tracks.Compatible(parent1.Track[crossPoint1], parent2.Track[crossPoint2]) {
			foundMatch = true
			// Increment so we splice *after* the cross point in parent 1
			crossPoint1++
			break
		}
		crossPoint2++
		if crossPoint2 >= len(parent2.Track) {
			break
		}
	}
	//	swap the track pieces at the chosen point on track A and track B
	if foundMatch {
		c1, c2 := Swap(parent1, parent2, crossPoint1, crossPoint2)
		if crossPoint1 > 4 {
			fmt.Println("swapped at", crossPoint1, crossPoint2)
			fmt.Println("parent1")
			printTrack(parent1.Track)
			fmt.Println("parent2")
			printTrack(parent2.Track)
			fmt.Println("child1")
			printTrack(c1.Track)
			fmt.Println("child2")
			printTrack(c2.Track)
		}
		return c1, c2
	}
	return parent1, parent2
}

// crossoverOne crosses over the parent tracks. if point is -1, a random point
// is chosen. (point is used for testing)
func crossoverOne(parent1 *Member, parent2 *Member, point int) (*Member, *Member) {
	//	choose a random point between the beginning and the end
	if point == -1 {
		minval := min(len(parent1.Track), len(parent2.Track))
		point = rand.Intn(minval)
	}
	return crossoverAtPoint(parent1, parent2, point)
}

func printTrack(t []tracks.Element) {
	for i, elem := range t {
		fmt.Printf("%d %s\n", i, elem.Segment.String())
	}
}

// Crossover chooses two members of a pool and joins them at random.
func (p *Pool) Crossover() *Pool {
	halfLen := len(p.Members) / 2
	for i := 0; i < halfLen; i++ {
		// select 2 parents at random
		idx1, parent1 := p.Select()
		idx2, parent2 := p.Select()
		if idx1 == -1 || idx2 == -1 {
			continue
		}
		if idx1 == idx2 {
			// No point in crossing over with ourself
			continue
		}
		if rand.Float64() < CROSSOVER_PROBABILITY {
			// XXX delete parents
			child1, child2 := crossoverOne(parent1, parent2, -1)
			p.Members[idx1] = child1
			p.Members[idx2] = child2
		}
	}
	return p
}

// Swap creates two children out of the parents, by crossing over the tracks at
// the given cross points. The sum of the two track lengths may be the same,
// but the tracks themselves will change.
func Swap(parent1 *Member, parent2 *Member, crossPoint1 int, crossPoint2 int) (*Member, *Member) {
	child1len := crossPoint1 + (len(parent2.Track) - crossPoint2)
	child2len := crossPoint2 + (len(parent1.Track) - crossPoint1)

	child1track := make([]tracks.Element, child1len)
	child2track := make([]tracks.Element, child2len)

	copy(child1track[:crossPoint1], parent1.Track[:crossPoint1])
	// if we filled in to child1 from [crossPoint2:] we might have gaps in the
	// track. fill from the end of cross point 1, with the contents of cross
	// point 2.
	copy(child1track[crossPoint1:], parent2.Track[crossPoint2:])

	copy(child2track[:crossPoint2], parent2.Track[:crossPoint2])
	copy(child2track[crossPoint2:], parent1.Track[crossPoint1:])
	child1 := &Member{
		Id:    fmt.Sprintf("iter_%s", uuid.New()),
		Track: child1track,
	}
	child2 := &Member{
		Id:    fmt.Sprintf("iter_%s", uuid.New()),
		Track: child2track,
	}
	return child1, child2
}

func (p *Pool) Spawn(numParents int) *Pool {
	return nil
}
<|endoftext|>


<commit_before>package task

import (
    "log"
    "../parser"
    "../writer"
    "../reader"
)

var (
    channelSize = 100
    logInterval = 1000
    countPoolLowerLimit = 300

    rowChannel chan string
    textChannel chan string
    countChannel chan map[string]int
    countPoolChannel chan map[string]int
    finished chan bool
)

func Count(r reader.Reader, w writer.Writer) {

    log.Println("start")

    initialize()

    go read(r)
    go parse()
    go count()
    go pool()
    go write(w)

    <- finished

    log.Println("finished")
}

func initialize() {
    rowChannel = make(chan string, channelSize)
    textChannel = make(chan string, channelSize)
    countChannel = make(chan map[string]int, channelSize)
    countPoolChannel = make(chan map[string]int, channelSize)
    finished = make(chan bool)
}

func read(r reader.Reader) {
    progress, end := logger("read")
    defer end()
    defer close(rowChannel)

    var text string
    eof := false
    for !eof {
        text, eof = r.ReadLine()
        rowChannel <- text
        progress()
    }
}

func parse() {
    progress, end := logger("parse")
    defer end()
    defer close(textChannel)

    for row := range rowChannel {
        title := parser.ExtractTitleFromXml(row)
        if title != "" {
            textChannel <- title
        }

        body := parser.ExtractTextFromHtml(parser.ExtractBodyFromXml(row))
        if body != "" {
            textChannel <- body
        }

        progress()
    }
}

func count() {
    progress, end := logger("count")
    defer end()
    defer close(countChannel)

    for text := range textChannel {
        countChannel <- parser.CountWords(text)
        progress()
    }
}

func pool() {
    progress, end := logger("pool")
    defer end()
    defer close(countPoolChannel)

    var pool map[string]int = nil

    for count := range countChannel {
        pool = parser.MergeCountedWords(pool, count)
        if len(pool) > countPoolLowerLimit {
            countPoolChannel <- pool
            pool = nil
        }
        progress()
    }

    if pool != nil {
        countPoolChannel <- pool
    }
}

func write(w writer.Writer) {
    progress, end := logger("write")
    defer end()
    defer func () { finished <- true }()

    for count := range countPoolChannel {
        w.WriteCount(count)
        progress()
    }
}

func logger(msg string) (func(), func()) {
    i := 0
    return func () {
        i++
        if i % logInterval  == 0 {
            log.Print(msg + " : ", i)
        }
    }, func () {
        log.Print(msg + " : ", i, " (finished)")
    }
}
<commit_msg>Mod to use task object<commit_after>package task

import (
    "log"
    "../parser"
    "../writer"
    "../reader"
)

var (
    channelSize = 100
    logInterval = 1000
    countPoolLowerLimit = 300
)

type countTask struct {
    rowChannel chan string
    textChannel chan string
    countChannel chan map[string]int
    countPoolChannel chan map[string]int
    finished chan bool
}

func Count(r reader.Reader, w writer.Writer) {

    log.Println("start")

    t := newCountTask()

    go t.read(r)
    go t.parse()
    go t.count()
    go t.pool()
    go t.write(w)

    t.waitToFinish()

    log.Println("finished")
}

func newCountTask() countTask {
    return countTask{
        rowChannel: make(chan string, channelSize),
        textChannel: make(chan string, channelSize),
        countChannel: make(chan map[string]int, channelSize),
        countPoolChannel: make(chan map[string]int, channelSize),
        finished: make(chan bool),
    }
}

func (t *countTask) read(r reader.Reader) {
    progress, end := logger("read")
    defer end()
    defer close(t.rowChannel)

    var text string
    eof := false
    for !eof {
        text, eof = r.ReadLine()
        t.rowChannel <- text
        progress()
    }
}

func (t *countTask) parse() {
    progress, end := logger("parse")
    defer end()
    defer close(t.textChannel)

    for row := range t.rowChannel {
        title := parser.ExtractTitleFromXml(row)
        if title != "" {
            t.textChannel <- title
        }

        body := parser.ExtractTextFromHtml(parser.ExtractBodyFromXml(row))
        if body != "" {
            t.textChannel <- body
        }

        progress()
    }
}

func (t *countTask) count() {
    progress, end := logger("count")
    defer end()
    defer close(t.countChannel)

    for text := range t.textChannel {
        t.countChannel <- parser.CountWords(text)
        progress()
    }
}

func (t *countTask) pool() {
    progress, end := logger("pool")
    defer end()
    defer close(t.countPoolChannel)

    var pool map[string]int = nil

    for count := range t.countChannel {
        pool = parser.MergeCountedWords(pool, count)
        if len(pool) > countPoolLowerLimit {
            t.countPoolChannel <- pool
            pool = nil
        }
        progress()
    }

    if pool != nil {
        t.countPoolChannel <- pool
    }
}

func (t *countTask) write(w writer.Writer) {
    progress, end := logger("write")
    defer end()
    defer func () { t.finished <- true }()

    for count := range t.countPoolChannel {
        w.WriteCount(count)
        progress()
    }
}

func (t *countTask) waitToFinish() {
    <- t.finished
}

func logger(msg string) (func(), func()) {
    i := 0
    return func () {
        i++
        if i % logInterval  == 0 {
            log.Print(msg + " : ", i)
        }
    }, func () {
        log.Print(msg + " : ", i, " (finished)")
    }
}
<|endoftext|>


<commit_before>// Intended to be used with "bind -x" on bash.
// Replace the current token in the command line with a given string.
package main

import (
	"fmt"
	"github.com/omakoto/go-common/src/common"
	"github.com/omakoto/go-common/src/shell"
	"github.com/omakoto/go-common/src/textio"
	"github.com/pborman/getopt/v2"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

var (
	index   = getopt.BoolLong("index", 'i', "Show indexes")
	null    = getopt.BoolLong("null", '0', "Use \\0 to terminate elements")
	useArgs = getopt.BoolLong("args", 'a', "Get command line from os.Args")
	stdin   = getopt.BoolLong("stdin", 0, "Read from stdin")
)

func main() {
	common.RunAndExit(realMain)
}

func realMain() int {
	getopt.Parse()

	eol := byte('\n')
	if *null {
		eol = 0
	}
	printWords(textio.BufferedStdout, eol, *index)

	return 0
}

func printWords(out io.Writer, eol byte, showIndex bool) {
	sh := shell.MustGetSupportedProxy()

	commandLine := ""

	if *stdin {
		in, err := ioutil.ReadAll(os.Stdin)
		common.Checke(err)
		commandLine = string(in)
	} else if *useArgs {
		commandLine = strings.Join(getopt.Args(), " ")
	} else {
		commandLine, _ = sh.GetCommandLine()
	}
	tokens := sh.Split(commandLine)

	eolBytes := append(make([]byte, 0), eol)

	for _, t := range tokens {
		if showIndex {
			fmt.Fprintf(out, "%d ", t.Index)
		}
		out.Write([]byte(t.Word))
		out.Write(eolBytes)
	}
}
<commit_msg>latest<commit_after>// Intended to be used with "bind -x" on bash.
// Replace the current token in the command line with a given string.
package main

import (
	"fmt"
	"github.com/omakoto/go-common/src/common"
	"github.com/omakoto/go-common/src/shell"
	"github.com/omakoto/go-common/src/textio"
	"github.com/pborman/getopt/v2"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

var (
	index   = getopt.BoolLong("index", 'i', "Show indexes")
	null    = getopt.BoolLong("null", '0', "Use \\0 to terminate elements")
	useArgs = getopt.BoolLong("args", 'a', "Get command line from os.Args")
	stdin   = getopt.BoolLong("stdin", 0, "Read from stdin")
)

func main() {
	common.RunAndExit(realMain)
}

func realMain() int {
	getopt.Parse()

	eol := byte('\n')
	if *null {
		eol = 0
	}
	printWords(textio.BufferedStdout, eol, *index)

	return 0
}

func printWords(out io.Writer, eol byte, showIndex bool) {
	sh := shell.MustGetSupportedProxy()

	commandLine := ""

	if *stdin {
		in, err := ioutil.ReadAll(os.Stdin)
		common.Checke(err)
		commandLine = string(in)
	} else if *useArgs {
		commandLine = strings.Join(getopt.Args(), " ")
	} else {
		commandLine, _ = sh.GetCommandL	ine()
	}
	tokens := sh.Split(commandLine)

	eolBytes := append(make([]byte, 0), eol)

	for _, t := range tokens {
		if showIndex {
			fmt.Fprintf(out, "%d ", t.Index)
		}
		out.Write([]byte(t.Word))
		out.Write(eolBytes)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/service"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

// Keep this around to simplify things
var G = libkb.G

var cmd libcmdline.Command

type Canceler interface {
	Cancel() error
}

func main() {

	g := G
	g.Init()

	go HandleSignals()
	err := mainInner(g)
	e2 := g.Shutdown()
	if err == nil {
		err = e2
	}
	if err != nil {
		g.Log.Error(err.Error())
		os.Exit(2)
	}
}

func mainInner(g *libkb.GlobalContext) error {

	cl := libcmdline.NewCommandLine(true, client.GetExtraFlags())
	cl.AddCommands(client.GetCommands(cl))
	cl.AddCommands(service.GetCommands(cl))

	var err error
	cmd, err = cl.Parse(os.Args)
	if err != nil {
		err = fmt.Errorf("Error parsing command line arguments: %s\n", err)
		return err
	}

	if cmd == nil {
		err = fmt.Errorf("No command selected")
		return err
	}

	if !cl.IsService() {
		client.InitUI()
	}

	if err = g.ConfigureAll(cl, cmd); err != nil {
		return err
	}

	if cl.IsService() {
		return cmd.Run()
	}

	// Start the server on the other end, possibly.
	// There are two cases in which we do this: (1) we want
	// a local loopback server in standalone mode; (2) we
	// need to "autofork" it. Do at most one of these
	// operations.
	if g.Env.GetStandalone() {
		if cl.IsNoStandalone() {
			return fmt.Errorf("Can't run command in standalone mode")
		}
		service.NewService(false /* isDaemon */).StartLoopbackServer(g)
	} else {
		// If this command warrants an autofork, do it now.
		if fc := cl.GetForkCmd(); fc == libcmdline.ForceFork || (g.Env.GetAutoFork() && fc != libcmdline.NoFork) {
			if err = client.ForkServerNix(cl); err != nil {
				return err
			}
		}
		// Whether or not we autoforked, we're now running in client-server
		// mode (as opposed to standalone). Register a global LogUI so that
		// calls to G.Log() in the daemon can be copied to us. This is
		// something of a hack on the daemon side.
		err = registerGlobalLogUI(g)
		if err != nil {
			return err
		}
	}

	return cmd.Run()
}

func registerGlobalLogUI(g *libkb.GlobalContext) error {
	protocols := []rpc2.Protocol{client.NewLogUIProtocol()}
	if err := client.RegisterProtocols(protocols); err != nil {
		return err
	}
	// Send our current debugging state, so that the server can avoid
	// sending us verbose logs when we don't want to read them.
	logLevel := keybase1.LogLevel_INFO
	if g.Env.GetDebug() {
		logLevel = keybase1.LogLevel_DEBUG
	}
	ctlClient, err := client.GetCtlClient()
	if err != nil {
		return err
	}
	ctlClient.SetLogLevel(logLevel)
	return nil
}

func HandleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	for {
		s := <-c
		if s != nil {
			G.Log.Debug("trapped signal %v", s)

			// if the current command has a Cancel function, then call it:
			if canc, ok := cmd.(Canceler); ok {
				G.Log.Debug("canceling running command")
				if err := canc.Cancel(); err != nil {
					G.Log.Warning("error canceling command: %s", err)
				}
			}

			G.Log.Debug("calling shutdown")
			G.Shutdown()
			G.Log.Error("interrupted")
			os.Exit(3)
		}
	}
}
<commit_msg>Fix command required (it's not)<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/service"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

// Keep this around to simplify things
var G = libkb.G

var cmd libcmdline.Command

type Canceler interface {
	Cancel() error
}

func main() {

	g := G
	g.Init()

	go HandleSignals()
	err := mainInner(g)
	e2 := g.Shutdown()
	if err == nil {
		err = e2
	}
	if err != nil {
		g.Log.Error(err.Error())
		os.Exit(2)
	}
}

func mainInner(g *libkb.GlobalContext) error {

	cl := libcmdline.NewCommandLine(true, client.GetExtraFlags())
	cl.AddCommands(client.GetCommands(cl))
	cl.AddCommands(service.GetCommands(cl))

	var err error
	cmd, err = cl.Parse(os.Args)
	if err != nil {
		err = fmt.Errorf("Error parsing command line arguments: %s\n", err)
		return err
	}

	if cmd == nil {
		return nil
	}

	if !cl.IsService() {
		client.InitUI()
	}

	if err = g.ConfigureAll(cl, cmd); err != nil {
		return err
	}

	if cl.IsService() {
		return cmd.Run()
	}

	// Start the server on the other end, possibly.
	// There are two cases in which we do this: (1) we want
	// a local loopback server in standalone mode; (2) we
	// need to "autofork" it. Do at most one of these
	// operations.
	if g.Env.GetStandalone() {
		if cl.IsNoStandalone() {
			return fmt.Errorf("Can't run command in standalone mode")
		}
		service.NewService(false /* isDaemon */).StartLoopbackServer(g)
	} else {
		// If this command warrants an autofork, do it now.
		if fc := cl.GetForkCmd(); fc == libcmdline.ForceFork || (g.Env.GetAutoFork() && fc != libcmdline.NoFork) {
			if err = client.ForkServerNix(cl); err != nil {
				return err
			}
		}
		// Whether or not we autoforked, we're now running in client-server
		// mode (as opposed to standalone). Register a global LogUI so that
		// calls to G.Log() in the daemon can be copied to us. This is
		// something of a hack on the daemon side.
		err = registerGlobalLogUI(g)
		if err != nil {
			return err
		}
	}

	return cmd.Run()
}

func registerGlobalLogUI(g *libkb.GlobalContext) error {
	protocols := []rpc2.Protocol{client.NewLogUIProtocol()}
	if err := client.RegisterProtocols(protocols); err != nil {
		return err
	}
	// Send our current debugging state, so that the server can avoid
	// sending us verbose logs when we don't want to read them.
	logLevel := keybase1.LogLevel_INFO
	if g.Env.GetDebug() {
		logLevel = keybase1.LogLevel_DEBUG
	}
	ctlClient, err := client.GetCtlClient()
	if err != nil {
		return err
	}
	ctlClient.SetLogLevel(logLevel)
	return nil
}

func HandleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	for {
		s := <-c
		if s != nil {
			G.Log.Debug("trapped signal %v", s)

			// if the current command has a Cancel function, then call it:
			if canc, ok := cmd.(Canceler); ok {
				G.Log.Debug("canceling running command")
				if err := canc.Cancel(); err != nil {
					G.Log.Warning("error canceling command: %s", err)
				}
			}

			G.Log.Debug("calling shutdown")
			G.Shutdown()
			G.Log.Error("interrupted")
			os.Exit(3)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"

	"golang.org/x/crypto/nacl/secretbox"
	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
	"github.com/keybase/gregor/protocol/chat1"
)

// chatLocalHandler implements keybase1.chatLocal.
type chatLocalHandler struct {
	*BaseHandler
	libkb.Contextified
	gh   *gregorHandler
	tlfh *tlfHandler
}

// newChatLocalHandler creates a chatLocalHandler.
func newChatLocalHandler(xp rpc.Transporter, g *libkb.GlobalContext, gh *gregorHandler) *chatLocalHandler {
	return &chatLocalHandler{
		BaseHandler:  NewBaseHandler(xp),
		Contextified: libkb.NewContextified(g),
		gh:           gh,
		tlfh:         newTlfHandler(xp, g),
	}
}

// GetInboxLocal implements keybase.chatLocal.getInboxLocal protocol.
func (h *chatLocalHandler) GetInboxLocal(ctx context.Context, p *chat1.Pagination) (chat1.InboxView, error) {
	return h.remoteClient().GetInboxRemote(ctx, p)
}

// GetThreadLocal implements keybase.chatLocal.getThreadLocal protocol.
func (h *chatLocalHandler) GetThreadLocal(ctx context.Context, arg keybase1.GetThreadLocalArg) (keybase1.ThreadView, error) {
	rarg := chat1.GetThreadRemoteArg{
		ConversationID: arg.ConversationID,
		Pagination:     arg.Pagination,
	}
	boxed, err := h.remoteClient().GetThreadRemote(ctx, rarg)
	if err != nil {
		return keybase1.ThreadView{}, err
	}

	return h.unboxThread(ctx, boxed)
}

// NewConversationLocal implements keybase.chatLocal.newConversationLocal protocol.
func (h *chatLocalHandler) NewConversationLocal(context.Context, chat1.ConversationIDTriple) error {
	return nil
}

// PostLocal implements keybase.chatLocal.postLocal protocol.
func (h *chatLocalHandler) PostLocal(ctx context.Context, arg keybase1.PostLocalArg) error {
	// encrypt the message
	boxed, err := h.boxMessage(ctx, arg.MessagePlaintext)
	if err != nil {
		return err
	}

	// get device signing key for this user
	signingKey, err := engine.GetMySecretKey(h.G(), h.getSecretUI, libkb.DeviceSigningKeyType, "sign chat message")
	if err != nil {
		return err
	}
	kp, ok := signingKey.(libkb.NaclSigningKeyPair)
	if !ok || kp.Private == nil {
		return libkb.KeyCannotSignError{}
	}

	// sign the header, encrypted body
	if err := h.signMessageBoxed(&boxed, kp); err != nil {
		return err
	}

	// post to remote gregord
	rarg := chat1.PostRemoteArg{
		ConversationID: arg.ConversationID,
		MessageBoxed:   boxed,
	}

	return h.remoteClient().PostRemote(ctx, rarg)
}

// getSecretUI returns a SecretUI, preferring a delegated SecretUI if
// possible.
func (h *chatLocalHandler) getSecretUI() libkb.SecretUI {
	ui, err := h.G().UIRouter.GetSecretUI(0)
	if err == nil && ui != nil {
		h.G().Log.Debug("chatLocalHandler: using delegated SecretUI")
		return ui
	}
	h.G().Log.Debug("chatLocalHandler: using local SecretUI")
	return h.BaseHandler.getSecretUI(0, h.G())
}

// remoteClient returns a client connection to gregord.
func (h *chatLocalHandler) remoteClient() *chat1.RemoteClient {
	return &chat1.RemoteClient{Cli: h.gh.cli}
}

// unboxThread transforms a chat1.ThreadViewBoxed to a keybase1.ThreadView.
func (h *chatLocalHandler) unboxThread(ctx context.Context, boxed chat1.ThreadViewBoxed) (keybase1.ThreadView, error) {
	thread := keybase1.ThreadView{
		Pagination: boxed.Pagination,
	}

	finder := newKeyFinder()
	for _, msg := range boxed.Messages {
		unboxed, err := h.unboxMessage(ctx, finder, msg)
		if err != nil {
			return keybase1.ThreadView{}, err
		}
		thread.Messages = append(thread.Messages, unboxed)
	}

	return thread, nil
}

// unboxMessage unboxes a chat1.MessageBoxed into a keybase1.Message.  It finds
// the appropriate keybase1.CryptKey.
func (h *chatLocalHandler) unboxMessage(ctx context.Context, finder *keyFinder, msg chat1.MessageBoxed) (keybase1.Message, error) {
	tlfName := msg.ClientHeader.TlfName
	keys, err := finder.find(ctx, h.tlfh, tlfName)
	if err != nil {
		return keybase1.Message{}, err
	}

	var matchKey *keybase1.CryptKey
	for _, key := range keys.CryptKeys {
		if key.KeyGeneration == msg.KeyGeneration {
			matchKey = &key
			break
		}
	}

	if matchKey == nil {
		return keybase1.Message{}, fmt.Errorf("no key found for generation %d", msg.KeyGeneration)
	}

	return h.unboxMessageWithKey(msg, matchKey)
}

// unboxMessageWithKey unboxes a chat1.MessageBoxed into a keybase1.Message given
// a keybase1.CryptKey.
func (h *chatLocalHandler) unboxMessageWithKey(msg chat1.MessageBoxed, key *keybase1.CryptKey) (keybase1.Message, error) {
	if msg.ServerHeader == nil {
		return keybase1.Message{}, errors.New("nil ServerHeader in MessageBoxed")
	}

	unboxed := keybase1.Message{
		ServerHeader: *msg.ServerHeader,
	}

	if len(msg.BodyCiphertext.N) != libkb.NaclDHNonceSize {
		return keybase1.Message{}, libkb.DecryptBadNonceError{}
	}
	var nonce [libkb.NaclDHNonceSize]byte
	copy(nonce[:], msg.BodyCiphertext.N)

	plaintext, ok := secretbox.Open(nil, msg.BodyCiphertext.E, &nonce, ((*[32]byte)(&key.Key)))
	if !ok {
		return keybase1.Message{}, libkb.DecryptOpenError{}
	}

	if err := json.Unmarshal(plaintext, &unboxed.MessagePlaintext); err != nil {
		return keybase1.Message{}, err
	}

	return unboxed, nil

}

// boxMessage encrypts a keybase1.MessagePlaintext into a chat1.MessageBoxed.  It
// finds the most recent key for the TLF.
func (h *chatLocalHandler) boxMessage(ctx context.Context, msg keybase1.MessagePlaintext) (chat1.MessageBoxed, error) {
	tlfName := msg.ClientHeader.TlfName
	keys, err := h.tlfh.CryptKeys(ctx, tlfName)
	if err != nil {
		return chat1.MessageBoxed{}, err
	}

	var recentKey *keybase1.CryptKey
	for _, key := range keys.CryptKeys {
		if recentKey == nil || key.KeyGeneration > recentKey.KeyGeneration {
			recentKey = &key
		}
	}

	if recentKey == nil {
		return chat1.MessageBoxed{}, fmt.Errorf("no key found for tlf %q", tlfName)
	}

	return h.boxMessageWithKey(msg, recentKey)
}

// boxMessageWithKey encrypts a keybase1.MessagePlaintext into a chat1.MessageBoxed
// given a keybase1.CryptKey.
func (h *chatLocalHandler) boxMessageWithKey(msg keybase1.MessagePlaintext, key *keybase1.CryptKey) (chat1.MessageBoxed, error) {
	s, err := json.Marshal(msg)
	if err != nil {
		return chat1.MessageBoxed{}, err
	}

	var nonce [libkb.NaclDHNonceSize]byte
	if _, err := rand.Read(nonce[:]); err != nil {
		return chat1.MessageBoxed{}, err
	}

	sealed := secretbox.Seal(nil, []byte(s), &nonce, ((*[32]byte)(&key.Key)))

	boxed := chat1.MessageBoxed{
		ClientHeader: msg.ClientHeader,
		BodyCiphertext: chat1.EncryptedData{
			V: 1,
			E: sealed,
			N: nonce[:],
		},
		KeyGeneration: key.KeyGeneration,
	}

	return boxed, nil
}

// signMessageBoxed signs the header and encrypted body of a chat1.MessageBoxed
// with the NaclSigningKeyPair.
func (h *chatLocalHandler) signMessageBoxed(msg *chat1.MessageBoxed, kp libkb.NaclSigningKeyPair) error {
	header, err := h.sign(msg.ClientHeader, kp)
	if err != nil {
		return err
	}
	msg.HeaderSignature = header

	body, err := h.sign(msg.BodyCiphertext, kp)
	if err != nil {
		return err
	}
	msg.BodySignature = body

	return nil
}

// sign signs data with a NaclSigningKeyPair, returning a chat1.SignatureInfo.
func (h *chatLocalHandler) sign(data interface{}, kp libkb.NaclSigningKeyPair) (chat1.SignatureInfo, error) {
	encoded, err := json.Marshal(data)
	if err != nil {
		return chat1.SignatureInfo{}, err
	}

	sig := *kp.Private.Sign(encoded)

	info := chat1.SignatureInfo{
		V: 1,
		S: sig[:],
		K: kp.Public[:],
	}

	return info, nil
}

// keyFinder remembers results from previous calls to CryptKeys().
// It is not intended to be used by multiple concurrent goroutines
// or held onto for very long, just to remember the keys while
// unboxing a thread of messages.
type keyFinder struct {
	keys map[string]keybase1.TLFCryptKeys
}

// newKeyFinder creates a keyFinder.
func newKeyFinder() *keyFinder {
	return &keyFinder{keys: make(map[string]keybase1.TLFCryptKeys)}
}

// find finds keybase1.TLFCryptKeys for tlfName, checking for existing
// results.
func (k *keyFinder) find(ctx context.Context, handler *tlfHandler, tlfName string) (keybase1.TLFCryptKeys, error) {
	existing, ok := k.keys[tlfName]
	if ok {
		return existing, nil
	}

	keys, err := handler.CryptKeys(ctx, tlfName)
	if err != nil {
		return keybase1.TLFCryptKeys{}, err
	}

	k.keys[tlfName] = keys

	return keys, nil
}
<commit_msg>Add remote call for NewConversation<commit_after>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"

	"golang.org/x/crypto/nacl/secretbox"
	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
	"github.com/keybase/gregor/protocol/chat1"
)

// chatLocalHandler implements keybase1.chatLocal.
type chatLocalHandler struct {
	*BaseHandler
	libkb.Contextified
	gh   *gregorHandler
	tlfh *tlfHandler
}

// newChatLocalHandler creates a chatLocalHandler.
func newChatLocalHandler(xp rpc.Transporter, g *libkb.GlobalContext, gh *gregorHandler) *chatLocalHandler {
	return &chatLocalHandler{
		BaseHandler:  NewBaseHandler(xp),
		Contextified: libkb.NewContextified(g),
		gh:           gh,
		tlfh:         newTlfHandler(xp, g),
	}
}

// GetInboxLocal implements keybase.chatLocal.getInboxLocal protocol.
func (h *chatLocalHandler) GetInboxLocal(ctx context.Context, p *chat1.Pagination) (chat1.InboxView, error) {
	return h.remoteClient().GetInboxRemote(ctx, p)
}

// GetThreadLocal implements keybase.chatLocal.getThreadLocal protocol.
func (h *chatLocalHandler) GetThreadLocal(ctx context.Context, arg keybase1.GetThreadLocalArg) (keybase1.ThreadView, error) {
	rarg := chat1.GetThreadRemoteArg{
		ConversationID: arg.ConversationID,
		Pagination:     arg.Pagination,
	}
	boxed, err := h.remoteClient().GetThreadRemote(ctx, rarg)
	if err != nil {
		return keybase1.ThreadView{}, err
	}

	return h.unboxThread(ctx, boxed)
}

// NewConversationLocal implements keybase.chatLocal.newConversationLocal protocol.
func (h *chatLocalHandler) NewConversationLocal(ctx context.Context, trip chat1.ConversationIDTriple) error {
	md := chat1.ConversationMetadata{
		IdTriple: trip,
	}
	return h.remoteClient().NewConversationRemote(ctx, md)
}

// PostLocal implements keybase.chatLocal.postLocal protocol.
func (h *chatLocalHandler) PostLocal(ctx context.Context, arg keybase1.PostLocalArg) error {
	// encrypt the message
	boxed, err := h.boxMessage(ctx, arg.MessagePlaintext)
	if err != nil {
		return err
	}

	// get device signing key for this user
	signingKey, err := engine.GetMySecretKey(h.G(), h.getSecretUI, libkb.DeviceSigningKeyType, "sign chat message")
	if err != nil {
		return err
	}
	kp, ok := signingKey.(libkb.NaclSigningKeyPair)
	if !ok || kp.Private == nil {
		return libkb.KeyCannotSignError{}
	}

	// sign the header, encrypted body
	if err := h.signMessageBoxed(&boxed, kp); err != nil {
		return err
	}

	// post to remote gregord
	rarg := chat1.PostRemoteArg{
		ConversationID: arg.ConversationID,
		MessageBoxed:   boxed,
	}

	return h.remoteClient().PostRemote(ctx, rarg)
}

// getSecretUI returns a SecretUI, preferring a delegated SecretUI if
// possible.
func (h *chatLocalHandler) getSecretUI() libkb.SecretUI {
	ui, err := h.G().UIRouter.GetSecretUI(0)
	if err == nil && ui != nil {
		h.G().Log.Debug("chatLocalHandler: using delegated SecretUI")
		return ui
	}
	h.G().Log.Debug("chatLocalHandler: using local SecretUI")
	return h.BaseHandler.getSecretUI(0, h.G())
}

// remoteClient returns a client connection to gregord.
func (h *chatLocalHandler) remoteClient() *chat1.RemoteClient {
	return &chat1.RemoteClient{Cli: h.gh.cli}
}

// unboxThread transforms a chat1.ThreadViewBoxed to a keybase1.ThreadView.
func (h *chatLocalHandler) unboxThread(ctx context.Context, boxed chat1.ThreadViewBoxed) (keybase1.ThreadView, error) {
	thread := keybase1.ThreadView{
		Pagination: boxed.Pagination,
	}

	finder := newKeyFinder()
	for _, msg := range boxed.Messages {
		unboxed, err := h.unboxMessage(ctx, finder, msg)
		if err != nil {
			return keybase1.ThreadView{}, err
		}
		thread.Messages = append(thread.Messages, unboxed)
	}

	return thread, nil
}

// unboxMessage unboxes a chat1.MessageBoxed into a keybase1.Message.  It finds
// the appropriate keybase1.CryptKey.
func (h *chatLocalHandler) unboxMessage(ctx context.Context, finder *keyFinder, msg chat1.MessageBoxed) (keybase1.Message, error) {
	tlfName := msg.ClientHeader.TlfName
	keys, err := finder.find(ctx, h.tlfh, tlfName)
	if err != nil {
		return keybase1.Message{}, err
	}

	var matchKey *keybase1.CryptKey
	for _, key := range keys.CryptKeys {
		if key.KeyGeneration == msg.KeyGeneration {
			matchKey = &key
			break
		}
	}

	if matchKey == nil {
		return keybase1.Message{}, fmt.Errorf("no key found for generation %d", msg.KeyGeneration)
	}

	return h.unboxMessageWithKey(msg, matchKey)
}

// unboxMessageWithKey unboxes a chat1.MessageBoxed into a keybase1.Message given
// a keybase1.CryptKey.
func (h *chatLocalHandler) unboxMessageWithKey(msg chat1.MessageBoxed, key *keybase1.CryptKey) (keybase1.Message, error) {
	if msg.ServerHeader == nil {
		return keybase1.Message{}, errors.New("nil ServerHeader in MessageBoxed")
	}

	unboxed := keybase1.Message{
		ServerHeader: *msg.ServerHeader,
	}

	if len(msg.BodyCiphertext.N) != libkb.NaclDHNonceSize {
		return keybase1.Message{}, libkb.DecryptBadNonceError{}
	}
	var nonce [libkb.NaclDHNonceSize]byte
	copy(nonce[:], msg.BodyCiphertext.N)

	plaintext, ok := secretbox.Open(nil, msg.BodyCiphertext.E, &nonce, ((*[32]byte)(&key.Key)))
	if !ok {
		return keybase1.Message{}, libkb.DecryptOpenError{}
	}

	if err := json.Unmarshal(plaintext, &unboxed.MessagePlaintext); err != nil {
		return keybase1.Message{}, err
	}

	return unboxed, nil

}

// boxMessage encrypts a keybase1.MessagePlaintext into a chat1.MessageBoxed.  It
// finds the most recent key for the TLF.
func (h *chatLocalHandler) boxMessage(ctx context.Context, msg keybase1.MessagePlaintext) (chat1.MessageBoxed, error) {
	tlfName := msg.ClientHeader.TlfName
	keys, err := h.tlfh.CryptKeys(ctx, tlfName)
	if err != nil {
		return chat1.MessageBoxed{}, err
	}

	var recentKey *keybase1.CryptKey
	for _, key := range keys.CryptKeys {
		if recentKey == nil || key.KeyGeneration > recentKey.KeyGeneration {
			recentKey = &key
		}
	}

	if recentKey == nil {
		return chat1.MessageBoxed{}, fmt.Errorf("no key found for tlf %q", tlfName)
	}

	return h.boxMessageWithKey(msg, recentKey)
}

// boxMessageWithKey encrypts a keybase1.MessagePlaintext into a chat1.MessageBoxed
// given a keybase1.CryptKey.
func (h *chatLocalHandler) boxMessageWithKey(msg keybase1.MessagePlaintext, key *keybase1.CryptKey) (chat1.MessageBoxed, error) {
	s, err := json.Marshal(msg)
	if err != nil {
		return chat1.MessageBoxed{}, err
	}

	var nonce [libkb.NaclDHNonceSize]byte
	if _, err := rand.Read(nonce[:]); err != nil {
		return chat1.MessageBoxed{}, err
	}

	sealed := secretbox.Seal(nil, []byte(s), &nonce, ((*[32]byte)(&key.Key)))

	boxed := chat1.MessageBoxed{
		ClientHeader: msg.ClientHeader,
		BodyCiphertext: chat1.EncryptedData{
			V: 1,
			E: sealed,
			N: nonce[:],
		},
		KeyGeneration: key.KeyGeneration,
	}

	return boxed, nil
}

// signMessageBoxed signs the header and encrypted body of a chat1.MessageBoxed
// with the NaclSigningKeyPair.
func (h *chatLocalHandler) signMessageBoxed(msg *chat1.MessageBoxed, kp libkb.NaclSigningKeyPair) error {
	header, err := h.sign(msg.ClientHeader, kp)
	if err != nil {
		return err
	}
	msg.HeaderSignature = header

	body, err := h.sign(msg.BodyCiphertext, kp)
	if err != nil {
		return err
	}
	msg.BodySignature = body

	return nil
}

// sign signs data with a NaclSigningKeyPair, returning a chat1.SignatureInfo.
func (h *chatLocalHandler) sign(data interface{}, kp libkb.NaclSigningKeyPair) (chat1.SignatureInfo, error) {
	encoded, err := json.Marshal(data)
	if err != nil {
		return chat1.SignatureInfo{}, err
	}

	sig := *kp.Private.Sign(encoded)

	info := chat1.SignatureInfo{
		V: 1,
		S: sig[:],
		K: kp.Public[:],
	}

	return info, nil
}

// keyFinder remembers results from previous calls to CryptKeys().
// It is not intended to be used by multiple concurrent goroutines
// or held onto for very long, just to remember the keys while
// unboxing a thread of messages.
type keyFinder struct {
	keys map[string]keybase1.TLFCryptKeys
}

// newKeyFinder creates a keyFinder.
func newKeyFinder() *keyFinder {
	return &keyFinder{keys: make(map[string]keybase1.TLFCryptKeys)}
}

// find finds keybase1.TLFCryptKeys for tlfName, checking for existing
// results.
func (k *keyFinder) find(ctx context.Context, handler *tlfHandler, tlfName string) (keybase1.TLFCryptKeys, error) {
	existing, ok := k.keys[tlfName]
	if ok {
		return existing, nil
	}

	keys, err := handler.CryptKeys(ctx, tlfName)
	if err != nil {
		return keybase1.TLFCryptKeys{}, err
	}

	k.keys[tlfName] = keys

	return keys, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	lg "github.com/hiromaily/golibs/log"
	"os"
	"strconv"
	"strings"
)

type TestData struct {
	id          int
	q1          string
	q2          string
	q3          string
	q4          string
	q5          string
	q6          string
	q7          string
	q8          string
	q9          string
	q10         string
	q11         string
	startedAt   string
	completedAt string
	platform    string
	deviceType  string
	deviceBrand string
	browser     string
	ip          string
	email       string
	hiddenEmail string
	hiddenName  string
}

var (
	fileName = flag.String("f", "", "Filename")
	lineNum  = flag.Int("l", 10, "Max Line number")
	//testData = []TestData{}
	testData = TestData{}
	//initID     = 2500
	//mailFormat = "gogo%d@gmail.com"
	//maxLines   = 50001
)

var usage = `Usage: %s [options...]
Options:
  -f  File name.
  -t  File type.
e.g.:
  gobulkdata -f ${HOME}/work/go/src/github.com/hiromaily/gotools/text.txt -l 20
`

//TODO
//add file type
//file can not be output

//text.txt
//2501,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo1@gmail.com,,test name
//2502,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo2@gmail.com,,test name
//2503,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo3@gmail.com,,test name

func init() {
	lg.InitializeLog(lg.DebugStatus, lg.LogOff, 99, "[GOTOOLS GOTEST]", "/var/log/go/gotool.log")

	flag.Usage = func() {
		fmt.Fprint(os.Stderr, fmt.Sprintf(usage, os.Args[0]))
	}

	flag.Parse()

	if *fileName == "" {
		flag.Usage()

		os.Exit(1)
		return
	}

	//initial value
	//testData = append(testData, TestData{
	testData = TestData{
		id:          2500,
		q1:          "",
		q2:          "",
		q3:          "",
		q4:          "",
		q5:          "",
		q6:          "",
		q7:          "",
		q8:          "",
		q9:          "",
		q10:         "Newsletter",
		q11:         "Artist c",
		startedAt:   "2016-06-15 13:18:43",
		completedAt: "2016-06-15 13:19:13",
		platform:    "Windows",
		deviceType:  "Desktop",
		deviceBrand: "unknown",
		browser:     "Chrome",
		ip:          "92.111.79.210",
		email:       "gogo%d@gmail.com",
		hiddenEmail: "",
		hiddenName:  "test name",
	}
}

func makeData() {
	var writer *bufio.Writer

	fmt.Println(*fileName)

	file, _ := os.OpenFile(*fileName, os.O_WRONLY, 0644)
	writer = bufio.NewWriter(file)

	for i := 1; i < *lineNum; i++ {
		tmpData := []string{
			strconv.Itoa(testData.id + i),
			testData.q1,
			testData.q2,
			testData.q3,
			testData.q4,
			testData.q5,
			testData.q6,
			testData.q7,
			testData.q8,
			testData.q9,
			testData.q10,
			testData.q11,
			testData.startedAt,
			testData.completedAt,
			testData.platform,
			testData.deviceType,
			testData.deviceBrand,
			testData.browser,
			testData.ip,
			fmt.Sprintf(testData.email, i),
			testData.hiddenEmail,
			testData.hiddenName,
		}
		fmt.Println(strings.Join(tmpData[:], ","))
		writer.WriteString(strings.Join(tmpData[:], ",") + "\n")
	}

	writer.Flush()

}

func main() {
	makeData()
}
<commit_msg>fixed bugs<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	lg "github.com/hiromaily/golibs/log"
	"os"
	"strconv"
	"strings"
)

type TestData struct {
	id          int
	q1          string
	q2          string
	q3          string
	q4          string
	q5          string
	q6          string
	q7          string
	q8          string
	q9          string
	q10         string
	q11         string
	startedAt   string
	completedAt string
	platform    string
	deviceType  string
	deviceBrand string
	browser     string
	ip          string
	email       string
	hiddenEmail string
	hiddenName  string
}

var (
	fileName = flag.String("f", "", "Filename")
	lineNum  = flag.Int("l", 10, "Max Line number")
	//testData = []TestData{}
	testData = TestData{}
	//initID     = 2500
	//mailFormat = "gogo%d@gmail.com"
	//maxLines   = 50001
)

var usage = `Usage: %s [options...]
Options:
  -f  File name.
  -t  File type.
e.g.:
  gobulkdata -f ${HOME}/work/go/src/github.com/hiromaily/gotools/text.txt -l 20
`

//TODO
//add file type
//file can not be output

//text.txt
//2501,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo1@gmail.com,,test name
//2502,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo2@gmail.com,,test name
//2503,,,,,,,,,,Newsletter,Artist c,2016-06-15 13:18:43,2016-06-15 13:19:13,Windows,Desktop,unknown,Chrome,92.111.79.210,gogo3@gmail.com,,test name

func init() {
	lg.InitializeLog(lg.DebugStatus, lg.LogOff, 99, "[GOTOOLS GOTEST]", "/var/log/go/gotool.log")

	flag.Usage = func() {
		fmt.Fprint(os.Stderr, fmt.Sprintf(usage, os.Args[0]))
	}

	flag.Parse()

	if *fileName == "" {
		flag.Usage()

		os.Exit(1)
		return
	}

	//initial value
	//testData = append(testData, TestData{
	testData = TestData{
		id:          2500,
		q1:          "",
		q2:          "",
		q3:          "",
		q4:          "",
		q5:          "",
		q6:          "",
		q7:          "",
		q8:          "",
		q9:          "",
		q10:         "Newsletter",
		q11:         "Artist c",
		startedAt:   "2016-06-15 13:18:43",
		completedAt: "2016-06-15 13:19:13",
		platform:    "Windows",
		deviceType:  "Desktop",
		deviceBrand: "unknown",
		browser:     "Chrome",
		ip:          "92.111.79.210",
		email:       "gogo%d@gmail.com",
		hiddenEmail: "",
		hiddenName:  "test name",
	}
}

func makeData() {
	var writer *bufio.Writer
	var err error

	fmt.Printf("file path is [%s]\n", *fileName)

	file, err := os.Create(*fileName)
	if err != nil {
		lg.Error(err)
	}
	//file, _ := os.OpenFile(*fileName, os.O_WRONLY | os.O_APPEND, 0644)
	defer file.Close()

	writer = bufio.NewWriter(file)

	for i := 1; i < *lineNum; i++ {
		tmpData := []string{
			strconv.Itoa(testData.id + i),
			testData.q1,
			testData.q2,
			testData.q3,
			testData.q4,
			testData.q5,
			testData.q6,
			testData.q7,
			testData.q8,
			testData.q9,
			testData.q10,
			testData.q11,
			testData.startedAt,
			testData.completedAt,
			testData.platform,
			testData.deviceType,
			testData.deviceBrand,
			testData.browser,
			testData.ip,
			fmt.Sprintf(testData.email, i),
			testData.hiddenEmail,
			testData.hiddenName,
		}
		fmt.Println(strings.Join(tmpData[:], ","))
		//
		_, err = writer.WriteString(strings.Join(tmpData[:], ",") + "\n")
		if err != nil {
			lg.Fatalf("[ERROR]WriteString: %v", err)
			break
		}
	}

	//fp, _ := os.OpenFile("/path/to/file", os.O_WRONLY | os.O_APPEND, 0644)
	//defer fp.close()
	//writer := bufio.NewWriter(fp)
	//bw := bufio.NewWriter(writer)
	//bw.WriteString("Hello world!")
	//bw.Flush()

	err = writer.Flush()
	if err != nil {
		//invalid argument
		lg.Fatalf("[ERROR]Flush: %v", err)
	}
}

func main() {
	makeData()
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"os"
	"reflect"
	"sort"
	"time"

	"tux21b.org/v1/gocql"
	"tux21b.org/v1/gocql/uuid"
)

var session *gocql.Session

func init() {
	cluster := gocql.NewCluster("127.0.0.1")
	cluster.Compressor = gocql.SnappyCompressor{}
	session = cluster.CreateSession()
}

type Page struct {
	Title       string
	RevId       uuid.UUID
	Body        string
	Views       int64
	Protected   bool
	Modified    time.Time
	Tags        []string
	Attachments map[string]Attachment
}

type Attachment []byte

func initSchema() error {
	if err := session.Query("DROP KEYSPACE gocql_test").Exec(); err != nil {
		log.Println("drop keyspace", err)
	}

	if err := session.Query(`CREATE KEYSPACE gocql_test
		WITH replication = {
			'class' : 'SimpleStrategy',
			'replication_factor' : 1
		}`).Exec(); err != nil {
		return err
	}

	if err := session.Query("USE gocql_test").Exec(); err != nil {
		return err
	}

	if err := session.Query(`CREATE TABLE page (
			title       varchar,
			revid       timeuuid,
			body        varchar,
			views       bigint,
			protected   boolean,
			modified    timestamp,
			tags        set<varchar>,
			attachments map<varchar, text>,
			PRIMARY KEY (title, revid)
		)`).Exec(); err != nil {
		return err
	}

	if err := session.Query(`CREATE TABLE page_stats (
			title varchar,
			views counter,
			PRIMARY KEY (title)
		)`).Exec(); err != nil {
		return err
	}

	return nil
}

var pageTestData = []*Page{
	&Page{
		Title:    "Frontpage",
		RevId:    uuid.TimeUUID(),
		Body:     "Welcome to this wiki page!",
		Modified: time.Date(2013, time.August, 13, 9, 52, 3, 0, time.UTC),
		Tags:     []string{"start", "important", "test"},
		Attachments: map[string]Attachment{
			"logo":    Attachment("\x00company logo\x00"),
			"favicon": Attachment("favicon.ico"),
		},
	},
	&Page{
		Title:    "Foobar",
		RevId:    uuid.TimeUUID(),
		Body:     "foo::Foo f = new foo::Foo(foo::Foo::INIT);",
		Modified: time.Date(2013, time.August, 13, 9, 52, 3, 0, time.UTC),
	},
}

func insertTestData() error {
	for _, page := range pageTestData {
		if err := session.Query(`INSERT INTO page
			(title, revid, body, views, protected, modified, tags, attachments)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
			page.Title, page.RevId, page.Body, page.Views, page.Protected,
			page.Modified, page.Tags, page.Attachments).Exec(); err != nil {
			return err
		}
	}
	return nil
}

func insertBatch() error {
	batch := gocql.NewBatch(gocql.LoggedBatch)
	for _, page := range pageTestData {
		batch.Query(`INSERT INTO page
			(title, revid, body, views, protected, modified, tags, attachments)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
			page.Title, page.RevId, page.Body, page.Views, page.Protected,
			page.Modified, page.Tags, page.Attachments)
	}
	if err := session.ExecuteBatch(batch); err != nil {
		return err
	}
	return nil
}

func getPage(title string, revid uuid.UUID) (*Page, error) {
	p := new(Page)
	err := session.Query(`SELECT title, revid, body, views, protected, modified,
			tags, attachments
			FROM page WHERE title = ? AND revid = ? LIMIT 1`, title, revid).Scan(
		&p.Title, &p.RevId, &p.Body, &p.Views, &p.Protected, &p.Modified,
		&p.Tags, &p.Attachments)
	return p, err
}

func main() {
	if err := initSchema(); err != nil {
		log.Fatal("initSchema: ", err)
	}

	if err := insertTestData(); err != nil {
		log.Fatal("insertTestData: ", err)
	}

	var count int
	if err := session.Query("SELECT COUNT(*) FROM page").Scan(&count); err != nil {
		log.Fatal("getCount: ", err)
	}
	if count != len(pageTestData) {
		log.Printf("count: expected %d, got %d", len(pageTestData), count)
	}

	for _, original := range pageTestData {
		page, err := getPage(original.Title, original.RevId)
		if err != nil {
			log.Print("getPage: ", err)
			continue
		}
		sort.Sort(sort.StringSlice(page.Tags))
		sort.Sort(sort.StringSlice(original.Tags))
		if !reflect.DeepEqual(page, original) {
			log.Printf("page: expected %#v, got %#v\n", original, page)
		}
	}

	trace := gocql.NewTraceWriter(session, os.Stdout)
	if err := session.Query("SELECT COUNT(*) FROM page").Trace(trace).Scan(&count); err != nil {
		log.Fatal("trace: ", err)
	}

	if err := session.Query("CREATE TABLE large (id int primary key)").Exec(); err != nil {
		log.Fatal("create table: ", err)
	}
	for i := 0; i < 100; i++ {
		if err := session.Query("INSERT INTO large (id) VALUES (?)", i).Exec(); err != nil {
			log.Fatal("insert: ", err)
		}
	}
	iter := session.Query("SELECT id FROM large").PageSize(10).Iter()
	var id int
	count = 0
	for iter.Scan(&id) {
		count++
	}
	if err := iter.Close(); err != nil {
		log.Fatal("large iter:", err)
	}
	if count != 100 {
		log.Fatalf("expected %d, got %d", 100, count)
	}

	for _, original := range pageTestData {
		if err := session.Query("DELETE FROM page WHERE title = ? AND revid = ?",
			original.Title, original.RevId).Exec(); err != nil {
			log.Println("delete:", err)
		}
	}
	if err := session.Query("SELECT COUNT(*) FROM page").Scan(&count); err != nil {
		log.Fatal("getCount: ", err)
	}
	if count != 0 {
		log.Printf("count: expected %d, got %d", len(pageTestData), count)
	}

	if err := insertBatch(); err != nil {
		log.Fatal("insertBatch: ", err)
	}

}
<commit_msg>document the usage of C* 1.2 better and do not run any non-supported examples.<commit_after>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"os"
	"reflect"
	"sort"
	"time"

	"tux21b.org/v1/gocql"
	"tux21b.org/v1/gocql/uuid"
)

var cluster *gocql.ClusterConfig
var session *gocql.Session

func init() {
	cluster = gocql.NewCluster("127.0.0.1")
	// uncomment the following two lines if you want to use Cassandra 1.2
	// cluster.ProtoVersion = 1
	// cluster.CQLVersion = "3.0.0"
	session = cluster.CreateSession()
}

type Page struct {
	Title       string
	RevId       uuid.UUID
	Body        string
	Views       int64
	Protected   bool
	Modified    time.Time
	Tags        []string
	Attachments map[string]Attachment
}

type Attachment []byte

func initSchema() error {
	if err := session.Query("DROP KEYSPACE gocql_test").Exec(); err != nil {
		log.Println("drop keyspace", err)
	}

	if err := session.Query(`CREATE KEYSPACE gocql_test
		WITH replication = {
			'class' : 'SimpleStrategy',
			'replication_factor' : 1
		}`).Exec(); err != nil {
		return err
	}

	if err := session.Query("USE gocql_test").Exec(); err != nil {
		return err
	}

	if err := session.Query(`CREATE TABLE page (
			title       varchar,
			revid       timeuuid,
			body        varchar,
			views       bigint,
			protected   boolean,
			modified    timestamp,
			tags        set<varchar>,
			attachments map<varchar, text>,
			PRIMARY KEY (title, revid)
		)`).Exec(); err != nil {
		return err
	}

	if err := session.Query(`CREATE TABLE page_stats (
			title varchar,
			views counter,
			PRIMARY KEY (title)
		)`).Exec(); err != nil {
		return err
	}

	return nil
}

var pageTestData = []*Page{
	&Page{
		Title:    "Frontpage",
		RevId:    uuid.TimeUUID(),
		Body:     "Welcome to this wiki page!",
		Modified: time.Date(2013, time.August, 13, 9, 52, 3, 0, time.UTC),
		Tags:     []string{"start", "important", "test"},
		Attachments: map[string]Attachment{
			"logo":    Attachment("\x00company logo\x00"),
			"favicon": Attachment("favicon.ico"),
		},
	},
	&Page{
		Title:    "Foobar",
		RevId:    uuid.TimeUUID(),
		Body:     "foo::Foo f = new foo::Foo(foo::Foo::INIT);",
		Modified: time.Date(2013, time.August, 13, 9, 52, 3, 0, time.UTC),
	},
}

func insertTestData() error {
	for _, page := range pageTestData {
		if err := session.Query(`INSERT INTO page
			(title, revid, body, views, protected, modified, tags, attachments)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
			page.Title, page.RevId, page.Body, page.Views, page.Protected,
			page.Modified, page.Tags, page.Attachments).Exec(); err != nil {
			return err
		}
	}
	return nil
}

func insertBatch() error {
	batch := gocql.NewBatch(gocql.LoggedBatch)
	for _, page := range pageTestData {
		batch.Query(`INSERT INTO page
			(title, revid, body, views, protected, modified, tags, attachments)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
			page.Title, page.RevId, page.Body, page.Views, page.Protected,
			page.Modified, page.Tags, page.Attachments)
	}
	if err := session.ExecuteBatch(batch); err != nil {
		return err
	}
	return nil
}

func getPage(title string, revid uuid.UUID) (*Page, error) {
	p := new(Page)
	err := session.Query(`SELECT title, revid, body, views, protected, modified,
			tags, attachments
			FROM page WHERE title = ? AND revid = ? LIMIT 1`, title, revid).Scan(
		&p.Title, &p.RevId, &p.Body, &p.Views, &p.Protected, &p.Modified,
		&p.Tags, &p.Attachments)
	return p, err
}

func main() {
	if err := initSchema(); err != nil {
		log.Fatal("initSchema: ", err)
	}

	if err := insertTestData(); err != nil {
		log.Fatal("insertTestData: ", err)
	}

	var count int
	if err := session.Query("SELECT COUNT(*) FROM page").Scan(&count); err != nil {
		log.Fatal("getCount: ", err)
	}
	if count != len(pageTestData) {
		log.Printf("count: expected %d, got %d", len(pageTestData), count)
	}

	for _, original := range pageTestData {
		page, err := getPage(original.Title, original.RevId)
		if err != nil {
			log.Print("getPage: ", err)
			continue
		}
		sort.Sort(sort.StringSlice(page.Tags))
		sort.Sort(sort.StringSlice(original.Tags))
		if !reflect.DeepEqual(page, original) {
			log.Printf("page: expected %#v, got %#v\n", original, page)
		}
	}

	// Query Tracing
	trace := gocql.NewTraceWriter(session, os.Stdout)
	if err := session.Query("SELECT COUNT(*) FROM page").Trace(trace).Scan(&count); err != nil {
		log.Fatal("trace: ", err)
	}

	if err := session.Query("CREATE TABLE large (id int primary key)").Exec(); err != nil {
		log.Fatal("create table: ", err)
	}
	for i := 0; i < 100; i++ {
		if err := session.Query("INSERT INTO large (id) VALUES (?)", i).Exec(); err != nil {
			log.Fatal("insert: ", err)
		}
	}

	if cluster.ProtoVersion >= 2 {
		// Result Paging
		iter := session.Query("SELECT id FROM large").PageSize(10).Iter()
		var id int
		count = 0
		for iter.Scan(&id) {
			count++
		}
		if err := iter.Close(); err != nil {
			log.Fatal("large iter:", err)
		}
		if count != 100 {
			log.Fatalf("expected %d, got %d", 100, count)
		}

		// Atomic Batches
		for _, original := range pageTestData {
			if err := session.Query("DELETE FROM page WHERE title = ? AND revid = ?",
				original.Title, original.RevId).Exec(); err != nil {
				log.Println("delete:", err)
			}
		}
		if err := session.Query("SELECT COUNT(*) FROM page").Scan(&count); err != nil {
			log.Fatal("getCount: ", err)
		}
		if count != 0 {
			log.Printf("count: expected %d, got %d", len(pageTestData), count)
		}

		if err := insertBatch(); err != nil {
			log.Fatal("insertBatch: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package tddbc

import (
    "testing"
)

func Test_Xが正しく取得できる(t *testing.T) {
    gridPoint := GridPoint{ X: 1, Y: 2 }

    if 1 != gridPoint.X {
        t.Errorf("expected: 1, actual: %v", gridPoint.X)
    }
}

func Test_Yが正しく取得できる(t *testing.T) {
    gridPoint := GridPoint{ X: 1, Y: 2 }

    if 2 != gridPoint.Y {
        t.Errorf("expected: 2, actual: %v", gridPoint.Y)
    }
}

func Test_格子点の文字列表現を取得できる(t *testing.T) {
    gridPoint := GridPoint{ X: 1, Y: 2 }

    if "(1,2)" != gridPoint.getNotation() {
        t.Errorf("expected: \"(1,2)\", actual: \"%v\"", gridPoint.getNotation())
    }
}

func Test_格子点の文字列表現を取得できる2(t *testing.T) {
    gridPoint := GridPoint{ X: 4, Y: 7 }

    if "(4,7)" != gridPoint.getNotation() {
        t.Errorf("expected: \"(4,7)\", actual: \"%v\"", gridPoint.getNotation())
    }
}
<commit_msg>Do refactoring of test methods to parameterize.<commit_after>package tddbc

import (
    "testing"
)

func Test_Xが正しく取得できる(t *testing.T) {
    gridPoint := GridPoint{ X: 1, Y: 2 }

    if 1 != gridPoint.X {
        t.Errorf("expected: 1, actual: %v", gridPoint.X)
    }
}

func Test_Yが正しく取得できる(t *testing.T) {
    gridPoint := GridPoint{ X: 1, Y: 2 }

    if 2 != gridPoint.Y {
        t.Errorf("expected: 2, actual: %v", gridPoint.Y)
    }
}

type gridPointTestParameter struct {
    gridPoint GridPoint
    notation string
}

func Test_格子点の文字列表現を取得できる(t *testing.T) {
    gridPointTestParameters := []gridPointTestParameter{
        gridPointTestParameter{ gridPoint: GridPoint{ X: 1, Y: 2 }, notation: "(1,2)" },
        gridPointTestParameter{ gridPoint: GridPoint{ X: 4, Y: 7 }, notation: "(4,7)" } }

    for _, parameter := range gridPointTestParameters {
        if parameter.notation != parameter.gridPoint.getNotation() {
            t.Errorf("expected: \"%v\", actual: \"%v\"", parameter.notation, parameter.gridPoint.getNotation())
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

package plot

import (
	"code.google.com/p/plotinum/vg"
	"math/rand"
	"fmt"
)

// An example of making and saving a plot.
func Example() *Plot {
	// Get some data to plot.
	pts := MakeXYLabelErrors(10)
	for i := range pts.XYs {
		if i == 0 {
			pts.XYs[i].X = rand.Float64()
		} else {
			pts.XYs[i].X = pts.XYs[i-1].X + rand.Float64()
		}
		pts.XYs[i].Y = rand.Float64()
		pts.Labels[i] = fmt.Sprintf("%05d", i)
		pts.XErrors[i].Low = -rand.Float64()/2
		pts.XErrors[i].High = rand.Float64()/2
		pts.YErrors[i].Low = -rand.Float64()/2
		pts.YErrors[i].High = rand.Float64()/2
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.X.Label.Text = "X Values"
	p.Y.Label.Text = "Y Values"
	line := Line{pts, DefaultLineStyle}
	scatter := Scatter{pts, DefaultGlyphStyle}
	errbars, err := MakeErrorBars(pts)
	if err != nil {
		panic(err)
	}
	labels, err := MakeLabels(pts)
	if err != nil {
		panic(err)
	}
	p.Add(line, scatter, errbars, labels)
	p.Legend.Add("line", line, scatter)
	p.Legend.Top = true
	return p
}

// Draw the plotinum logo.
func Example_logo() *Plot {
	p, err := New()
	if err != nil {
		panic(err)
	}

	DefaultLineStyle.Width = vg.Points(1)
	DefaultGlyphStyle.Radius = vg.Points(3)

	p.Y.Tick.Marker = ConstantTicks([]Tick{{0, "0"}, {0.25, ""}, {0.5, "0.5"}, {0.75, ""}, {1, "1"}})
	p.X.Tick.Marker = ConstantTicks([]Tick{{0, "0"}, {0.25, ""}, {0.5, "0.5"}, {0.75, ""}, {1, "1"}})

	pts := XYs{{0, 0}, {0, 1}, {0.5, 1}, {0.5, 0.6}, {0, 0.6}}
	line := Line{pts, DefaultLineStyle}
	scatter := Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	pts = XYs{{1, 0}, {0.75, 0}, {0.75, 0.75}}
	line = Line{pts, DefaultLineStyle}
	scatter = Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	pts = XYs{{0.5, 0.5}, {1, 0.5}}
	line = Line{pts, DefaultLineStyle}
	scatter = Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	return p
}

// An example of making a box plot.
func Example_boxPlot() *Plot {
	// Get some data to plot.
	n := 10
	uniform := make(Ys, n)
	normal := make(Ys, n)
	expon := make(Ys, n)
	for i := 0; i < n; i++ {
		uniform[i] = rand.Float64()
		normal[i] = rand.NormFloat64()
		expon[i] = rand.ExpFloat64()
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.Y.Label.Text = "Values"

	// Make boxes for our data and add them to the plot.
	p.Add(NewBox(vg.Points(20), 0, uniform),
		NewBox(vg.Points(20), 1, normal),
		NewBox(vg.Points(20), 2, expon))

	// Set the X axis of the plot to nominal with
	// the given names for x=0, x=1 and x=2.
	p.NominalX("Uniform\nDistribution", "Normal\nDistribution",
		"Exponential\nDistribution")
	return p
}

// An example of making a horizontal box plot.
func Example_horizontalBoxes() *Plot {
	// Get some data to plot.
	n := 10
	uniform := make(Ys, n)
	normal := make(Ys, n)
	expon := make(Ys, n)
	for i := 0; i < n; i++ {
		uniform[i] = rand.Float64()
		normal[i] = rand.NormFloat64()
		expon[i] = rand.ExpFloat64()
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.X.Label.Text = "Values"

	// Make horizontal boxes for our data and add
	// them to the plot.
	p.Add(MakeHorizBox(vg.Points(20), 0, uniform),
		MakeHorizBox(vg.Points(20), 1, normal),
		MakeHorizBox(vg.Points(20), 2, expon))

	// Set the Y axis of the plot to nominal with
	// the given names for y=0, y=1 and y=2.
	p.NominalY("Uniform\nDistribution", "Normal\nDistribution",
		"Exponential\nDistribution")
	return p
}
<commit_msg>Move the example plot around so things look nicer.<commit_after>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

package plot

import (
	"code.google.com/p/plotinum/vg"
	"math/rand"
	"fmt"
)

// An example of making and saving a plot.
func Example() *Plot {
	// Get some data to plot.
	pts := MakeXYLabelErrors(10)
	for i := range pts.XYs {
		if i == 0 {
			pts.XYs[i].X = rand.Float64()
		} else {
			pts.XYs[i].X = pts.XYs[i-1].X + rand.Float64()
		}
		pts.XYs[i].Y = rand.Float64()
		pts.Labels[i] = fmt.Sprintf("%d", i)
		pts.XErrors[i].Low = -rand.Float64()/2
		pts.XErrors[i].High = rand.Float64()/2
		pts.YErrors[i].Low = -rand.Float64()/2
		pts.YErrors[i].High = rand.Float64()/2
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.X.Label.Text = "X Values"
	p.Y.Label.Text = "Y Values"
	line := Line{pts, DefaultLineStyle}
	scatter := Scatter{pts, DefaultGlyphStyle}
	errbars, err := MakeErrorBars(pts)
	if err != nil {
		panic(err)
	}
	labels, err := MakeLabels(pts)
	if err != nil {
		panic(err)
	}
	labels.XOffs = scatter.Radius
	labels.YOffs = scatter.Radius
	p.Add(line, scatter, errbars, labels)
	p.Legend.Add("line", line, scatter)
	p.Legend.Left = true
	return p
}

// Draw the plotinum logo.
func Example_logo() *Plot {
	p, err := New()
	if err != nil {
		panic(err)
	}

	DefaultLineStyle.Width = vg.Points(1)
	DefaultGlyphStyle.Radius = vg.Points(3)

	p.Y.Tick.Marker = ConstantTicks([]Tick{{0, "0"}, {0.25, ""}, {0.5, "0.5"}, {0.75, ""}, {1, "1"}})
	p.X.Tick.Marker = ConstantTicks([]Tick{{0, "0"}, {0.25, ""}, {0.5, "0.5"}, {0.75, ""}, {1, "1"}})

	pts := XYs{{0, 0}, {0, 1}, {0.5, 1}, {0.5, 0.6}, {0, 0.6}}
	line := Line{pts, DefaultLineStyle}
	scatter := Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	pts = XYs{{1, 0}, {0.75, 0}, {0.75, 0.75}}
	line = Line{pts, DefaultLineStyle}
	scatter = Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	pts = XYs{{0.5, 0.5}, {1, 0.5}}
	line = Line{pts, DefaultLineStyle}
	scatter = Scatter{pts, DefaultGlyphStyle}
	p.Add(line, scatter)

	return p
}

// An example of making a box plot.
func Example_boxPlot() *Plot {
	// Get some data to plot.
	n := 10
	uniform := make(Ys, n)
	normal := make(Ys, n)
	expon := make(Ys, n)
	for i := 0; i < n; i++ {
		uniform[i] = rand.Float64()
		normal[i] = rand.NormFloat64()
		expon[i] = rand.ExpFloat64()
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.Y.Label.Text = "Values"

	// Make boxes for our data and add them to the plot.
	p.Add(NewBox(vg.Points(20), 0, uniform),
		NewBox(vg.Points(20), 1, normal),
		NewBox(vg.Points(20), 2, expon))

	// Set the X axis of the plot to nominal with
	// the given names for x=0, x=1 and x=2.
	p.NominalX("Uniform\nDistribution", "Normal\nDistribution",
		"Exponential\nDistribution")
	return p
}

// An example of making a horizontal box plot.
func Example_horizontalBoxes() *Plot {
	// Get some data to plot.
	n := 10
	uniform := make(Ys, n)
	normal := make(Ys, n)
	expon := make(Ys, n)
	for i := 0; i < n; i++ {
		uniform[i] = rand.Float64()
		normal[i] = rand.NormFloat64()
		expon[i] = rand.ExpFloat64()
	}

	// Make our plot and set some labels.
	p, err := New()
	if err != nil {
		panic(err)
	}
	p.Title.Text = "Plot Title"
	p.X.Label.Text = "Values"

	// Make horizontal boxes for our data and add
	// them to the plot.
	p.Add(MakeHorizBox(vg.Points(20), 0, uniform),
		MakeHorizBox(vg.Points(20), 1, normal),
		MakeHorizBox(vg.Points(20), 2, expon))

	// Set the Y axis of the plot to nominal with
	// the given names for y=0, y=1 and y=2.
	p.NominalY("Uniform\nDistribution", "Normal\nDistribution",
		"Exponential\nDistribution")
	return p
}
<|endoftext|>


<commit_before>package ruby

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/markelog/release"

	"github.com/markelog/eclectica/variables"
)

var (
	VersionsLink   = "https://rvm.io/binaries"
	versionPattern = "\\d+\\.\\d+\\.\\d"

	bins = []string{"erb", "gem", "irb", "rake", "rdoc", "ri", "ruby"}
)

type Ruby struct {
	Version string
}

func (ruby Ruby) Install() error {
	return removeRVMArtefacts(variables.Path("ruby", ruby.Version))
}

func (ruby Ruby) PostInstall() error {
	return dealWithShell()
}

// Removes RVM artefacts (ignore errors)
func removeRVMArtefacts(base string) error {
	gems := filepath.Join(base, "lib/ruby/gems")

	// Remove `cache` folder since it supposed to work with RVM cache
	folders, _ := ioutil.ReadDir(gems)
	for _, folder := range folders {
		err := os.Remove(filepath.Join(gems, folder.Name(), "cache"))
		if err != nil {
			return err
		}
	}

	return nil
}

func (ruby Ruby) Environment() (string, error) {
	return "", nil
}

func (ruby Ruby) Info() (map[string]string, error) {
	result := make(map[string]string)

	result["filename"] = fmt.Sprintf("ruby-%s", ruby.Version)
	result["extension"] = "tar.bz2"
	result["url"] = fmt.Sprintf("%s/%s.%s", getUrl(), result["filename"], result["extension"])

	return result, nil
}

func (ruby Ruby) Bins() []string {
	return bins
}

func (ruby Ruby) Current() string {
	bin := variables.GetBin("ruby")
	out, _ := exec.Command(bin, "--version").Output()

	if len(out) == 0 {
		return ""
	}

	version := strings.TrimSpace(string(out))
	rVersion := regexp.MustCompile(versionPattern)
	testVersion := rVersion.FindAllStringSubmatch(version, 1)

	if len(testVersion) == 0 {
		return ""
	}

	return testVersion[0][0]
}

func (ruby Ruby) ListRemote() ([]string, error) {
	url := getUrl()
	doc, err := goquery.NewDocument(url)

	if err != nil {
		if _, ok := err.(net.Error); ok {
			return nil, errors.New("Can't establish connection")
		}

		return nil, err
	}

	version := regexp.MustCompile("\\d+\\.\\d+\\.\\d+")
	result := []string{}
	links := doc.Find("a")

	for i := range links.Nodes {
		href, _ := links.Eq(i).Attr("href")

		href = strings.Replace(href, "ruby-", "", 1)
		href = strings.Replace(href, ".tar.bz2", "", 1)

		if version.MatchString(href) {
			result = append(result, href)
		}
	}

	return result, nil
}

func getUrl() string {
	typa, _, version := release.All()
	arch := "x86_64"
	versions := strings.Split(version, ".")
	version = versions[0] + "." + versions[1]

	return fmt.Sprintf("%s/%s/%s/%s", VersionsLink, typa, version, arch)
}
<commit_msg>Move removal of artefacts for ruby plugin to PostInstall<commit_after>package ruby

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/markelog/release"

	"github.com/markelog/eclectica/variables"
)

var (
	VersionsLink   = "https://rvm.io/binaries"
	versionPattern = "\\d+\\.\\d+\\.\\d"

	bins = []string{"erb", "gem", "irb", "rake", "rdoc", "ri", "ruby"}
)

type Ruby struct {
	Version string
}

func (ruby Ruby) Install() error {
	return nil
}

func (ruby Ruby) PostInstall() error {
	err := removeRVMArtefacts(variables.Path("ruby", ruby.Version))
	if err != nil {
		return err
	}

	return dealWithShell()
}

// Removes RVM artefacts (ignore errors)
func removeRVMArtefacts(base string) error {
	gems := filepath.Join(base, "lib/ruby/gems")

	// Remove `cache` folder since it supposed to work with RVM cache
	folders, _ := ioutil.ReadDir(gems)
	for _, folder := range folders {
		fmt.Println(filepath.Join(gems, folder.Name(), "cache"))
		err := os.RemoveAll(filepath.Join(gems, folder.Name(), "cache"))
		if err != nil {
			return err
		}
	}

	return nil
}

func (ruby Ruby) Environment() (string, error) {
	return "", nil
}

func (ruby Ruby) Info() (map[string]string, error) {
	result := make(map[string]string)

	result["filename"] = fmt.Sprintf("ruby-%s", ruby.Version)
	result["extension"] = "tar.bz2"
	result["url"] = fmt.Sprintf("%s/%s.%s", getUrl(), result["filename"], result["extension"])

	return result, nil
}

func (ruby Ruby) Bins() []string {
	return bins
}

func (ruby Ruby) Current() string {
	bin := variables.GetBin("ruby")
	out, _ := exec.Command(bin, "--version").Output()

	if len(out) == 0 {
		return ""
	}

	version := strings.TrimSpace(string(out))
	rVersion := regexp.MustCompile(versionPattern)
	testVersion := rVersion.FindAllStringSubmatch(version, 1)

	if len(testVersion) == 0 {
		return ""
	}

	return testVersion[0][0]
}

func (ruby Ruby) ListRemote() ([]string, error) {
	url := getUrl()
	doc, err := goquery.NewDocument(url)

	if err != nil {
		if _, ok := err.(net.Error); ok {
			return nil, errors.New("Can't establish connection")
		}

		return nil, err
	}

	version := regexp.MustCompile("\\d+\\.\\d+\\.\\d+")
	result := []string{}
	links := doc.Find("a")

	for i := range links.Nodes {
		href, _ := links.Eq(i).Attr("href")

		href = strings.Replace(href, "ruby-", "", 1)
		href = strings.Replace(href, ".tar.bz2", "", 1)

		if version.MatchString(href) {
			result = append(result, href)
		}
	}

	return result, nil
}

func getUrl() string {
	typa, _, version := release.All()
	arch := "x86_64"
	versions := strings.Split(version, ".")
	version = versions[0] + "." + versions[1]

	return fmt.Sprintf("%s/%s/%s/%s", VersionsLink, typa, version, arch)
}
<|endoftext|>