<commit_before>/*
package stm provides Software Transactional Memory operations for Go. This is
an alternative to the standard way of writing concurrent code (channels and
mutexes). STM makes it easy to perform arbitrarily complex operations in an
atomic fashion. One of its primary advantages over traditional locking is that
STM transactions are composable, whereas locking functions are not -- the
composition will either deadlock or release the lock between functions (making
it non-atomic).

To begin, create an STM object that wraps the data you want to access
concurrently.

	x := stm.NewVar(3)

You can then use the Atomically method to atomically read and/or write the the
data. This code atomically decrements x:

	stm.Atomically(func(tx *stm.Tx) {
		cur := tx.Get(x).(int)
		tx.Set(x, cur-1)
	})

An important part of STM transactions is retrying. At any point during the
transaction, you can call tx.Retry(), which will abort the transaction, but
not cancel it entirely. The call to Atomically will block until another call
to Atomically finishes, at which point the transaction will be rerun.
Specifically, one of the values read by the transaction (via tx.Get) must be
updated before the transaction will be rerun. As an example, this code will
try to decrement x, but will block as long as x is zero:

	stm.Atomically(func(tx *stm.Tx) {
		cur := tx.Get(x).(int)
		if cur == 0 {
			tx.Retry()
		}
		tx.Set(x, cur-1)
	})

Internally, tx.Retry simply calls panic(stm.Retry). Panicking with any other
value will cancel the transaction; no values will be changed. However, it is
the responsibility of the caller to catch such panics.

Multiple transactions can be composed using Select. If the first transaction
calls Retry, the next transaction will be run, and so on. If all of the
transactions call Retry, the call will block and the entire selection will be
retried. For example, this code implements the "decrement-if-nonzero"
transaction above, but for two values. It will first try to decrement x, then
y, and block if both values are zero.

	func dec(v *stm.Var) {
		return func(tx *stm.Tx) {
			cur := tx.Get(v).(int)
			if cur == 0 {
				tx.Retry()
			}
			tx.Set(v, cur-1)
		}
	}

	// Note that Select does not perform any work itself, but merely
	// returns a transaction function.
	stm.Atomically(stm.Select(dec(x), dec(y)))

An important caveat: transactions must not have side effects! This is because a
transaction may be restarted several times before completing, meaning the side
effects may execute more than once. This will almost certainly cause incorrect
behavior. One common way to get around this is to build up a list of operations
to perform inside the transaction, and then perform them after the transaction
completes.

The stm API tries to mimic that of Haskell's Control.Concurrent.STM, but this
is not entirely possible due to Go's type system; we are forced to use
interface{} and type assertions. Furthermore, Haskell can enforce at compile
time that STM variables are not modified outside the STM monad. This is not
possible in Go, so be especially careful when using pointers in your STM code.

It remains to be seen whether this style of concurrency has practical
applications in Go. If you find this package useful, please tell me about it!
*/
package stm

import (
	"sync"
)

// Retry is a sentinel value. When thrown via panic, it indicates that a
// transaction should be retried.
const Retry = "retry"

// The globalLock serializes transaction verification/committal.
var globalLock sync.Mutex

// A Var holds an STM variable.
type Var struct {
	val interface{}
}

// NewVar returns a new STM variable.
func NewVar(val interface{}) *Var {
	return &Var{val}
}

// A Tx represents an atomic transaction.
type Tx struct {
	reads  map[*Var]interface{}
	writes map[*Var]interface{}
}

// verify checks that none of the logged values have changed since the
// transaction began
func (tx *Tx) verify() bool {
	for v, val := range tx.reads {
		if v.val != val {
			return false
		}
	}
	return true
}

// commit writes the values in the transaction log to their respective Vars.
func (tx *Tx) commit() {
	for v, val := range tx.writes {
		v.val = val
	}
}

// Get returns the value of v as of the start of the transaction.
func (tx *Tx) Get(v *Var) interface{} {
	// If we previously wrote to v, it will be in the write log.
	if val, ok := tx.writes[v]; ok {
		return val
	}
	// If we previously read v, it will be in the read log.
	if val, ok := tx.reads[v]; ok {
		return val
	}
	// Otherwise, record and return its current value.
	tx.reads[v] = v.val
	return v.val
}

// Set sets the value of a Var for the lifetime of the transaction.
func (tx *Tx) Set(v *Var, val interface{}) {
	tx.writes[v] = val
}

// Retry aborts the transaction and retries it when a Var changes.
func (tx *Tx) Retry() {
	panic(Retry)
}

// Assert is a helper function that retries a transaction if the condition is
// not satisfied.
func (tx *Tx) Assert(p bool) {
	if !p {
		tx.Retry()
	}
}

// catchRetry returns true if fn calls tx.Retry.
func catchRetry(fn func(*Tx), tx *Tx) (retry bool) {
	defer func() {
		if r := recover(); r == Retry {
			retry = true
		} else if r != nil {
			panic(r)
		}
	}()
	fn(tx)
	return
}

// Select runs the supplied functions in order. Execution stops when a
// function succeeds without calling Retry. If no functions succeed, the
// entire selection will be retried.
func Select(fns ...func(*Tx)) func(*Tx) {
	return func(tx *Tx) {
		switch len(fns) {
		case 0:
			// empty Select blocks forever
			tx.Retry()
		case 1:
			fns[0](tx)
		default:
			if catchRetry(fns[0], tx) {
				Select(fns[1:]...)(tx)
			}
		}
	}
}

// Atomically executes the atomic function fn.
func Atomically(fn func(*Tx)) {
retry:
	// run the transaction
	tx := &Tx{
		reads:  make(map[*Var]interface{}),
		writes: make(map[*Var]interface{}),
	}
	if catchRetry(fn, tx) {
		goto retry
	}
	// verify the read log
	globalLock.Lock()
	if !tx.verify() {
		globalLock.Unlock()
		goto retry
	}
	// commit the write log
	if len(tx.writes) > 0 {
		tx.commit()
	}
	globalLock.Unlock()
}

// AtomicGet is a helper function that atomically reads a value.
func AtomicGet(v *Var) interface{} {
	var i interface{}
	Atomically(func(tx *Tx) { i = tx.Get(v) })
	return i
}

// AtomicSet is a helper function that atomically writes a value.
func AtomicSet(v *Var, val interface{}) {
	Atomically(func(tx *Tx) { tx.Set(v, val) })
}

// Compose is a helper function that composes multiple transactions into a
// single transaction.
func Compose(fns ...func(*Tx)) func(*Tx) {
	return func(tx *Tx) {
		for _, f := range fns {
			f(tx)
		}
	}
}
<commit_msg>fix race condition in tx.Get<commit_after>/*
package stm provides Software Transactional Memory operations for Go. This is
an alternative to the standard way of writing concurrent code (channels and
mutexes). STM makes it easy to perform arbitrarily complex operations in an
atomic fashion. One of its primary advantages over traditional locking is that
STM transactions are composable, whereas locking functions are not -- the
composition will either deadlock or release the lock between functions (making
it non-atomic).

To begin, create an STM object that wraps the data you want to access
concurrently.

	x := stm.NewVar(3)

You can then use the Atomically method to atomically read and/or write the the
data. This code atomically decrements x:

	stm.Atomically(func(tx *stm.Tx) {
		cur := tx.Get(x).(int)
		tx.Set(x, cur-1)
	})

An important part of STM transactions is retrying. At any point during the
transaction, you can call tx.Retry(), which will abort the transaction, but
not cancel it entirely. The call to Atomically will block until another call
to Atomically finishes, at which point the transaction will be rerun.
Specifically, one of the values read by the transaction (via tx.Get) must be
updated before the transaction will be rerun. As an example, this code will
try to decrement x, but will block as long as x is zero:

	stm.Atomically(func(tx *stm.Tx) {
		cur := tx.Get(x).(int)
		if cur == 0 {
			tx.Retry()
		}
		tx.Set(x, cur-1)
	})

Internally, tx.Retry simply calls panic(stm.Retry). Panicking with any other
value will cancel the transaction; no values will be changed. However, it is
the responsibility of the caller to catch such panics.

Multiple transactions can be composed using Select. If the first transaction
calls Retry, the next transaction will be run, and so on. If all of the
transactions call Retry, the call will block and the entire selection will be
retried. For example, this code implements the "decrement-if-nonzero"
transaction above, but for two values. It will first try to decrement x, then
y, and block if both values are zero.

	func dec(v *stm.Var) {
		return func(tx *stm.Tx) {
			cur := tx.Get(v).(int)
			if cur == 0 {
				tx.Retry()
			}
			tx.Set(v, cur-1)
		}
	}

	// Note that Select does not perform any work itself, but merely
	// returns a transaction function.
	stm.Atomically(stm.Select(dec(x), dec(y)))

An important caveat: transactions must not have side effects! This is because a
transaction may be restarted several times before completing, meaning the side
effects may execute more than once. This will almost certainly cause incorrect
behavior. One common way to get around this is to build up a list of operations
to perform inside the transaction, and then perform them after the transaction
completes.

The stm API tries to mimic that of Haskell's Control.Concurrent.STM, but this
is not entirely possible due to Go's type system; we are forced to use
interface{} and type assertions. Furthermore, Haskell can enforce at compile
time that STM variables are not modified outside the STM monad. This is not
possible in Go, so be especially careful when using pointers in your STM code.

It remains to be seen whether this style of concurrency has practical
applications in Go. If you find this package useful, please tell me about it!
*/
package stm

import (
	"sync"
)

// Retry is a sentinel value. When thrown via panic, it indicates that a
// transaction should be retried.
const Retry = "retry"

// The globalLock serializes transaction verification/committal.
var globalLock sync.Mutex

// A Var holds an STM variable.
type Var struct {
	val interface{}
}

// NewVar returns a new STM variable.
func NewVar(val interface{}) *Var {
	return &Var{val}
}

// A Tx represents an atomic transaction.
type Tx struct {
	reads  map[*Var]interface{}
	writes map[*Var]interface{}
}

// verify checks that none of the logged values have changed since the
// transaction began
func (tx *Tx) verify() bool {
	for v, val := range tx.reads {
		if v.val != val {
			return false
		}
	}
	return true
}

// commit writes the values in the transaction log to their respective Vars.
func (tx *Tx) commit() {
	for v, val := range tx.writes {
		v.val = val
	}
}

// Get returns the value of v as of the start of the transaction.
func (tx *Tx) Get(v *Var) interface{} {
	// If we previously wrote to v, it will be in the write log.
	if val, ok := tx.writes[v]; ok {
		return val
	}
	// If we previously read v, it will be in the read log.
	if val, ok := tx.reads[v]; ok {
		return val
	}
	// Otherwise, record and return its current value.
	globalLock.Lock()
	defer globalLock.Unlock()
	tx.reads[v] = v.val
	return v.val
}

// Set sets the value of a Var for the lifetime of the transaction.
func (tx *Tx) Set(v *Var, val interface{}) {
	tx.writes[v] = val
}

// Retry aborts the transaction and retries it when a Var changes.
func (tx *Tx) Retry() {
	panic(Retry)
}

// Assert is a helper function that retries a transaction if the condition is
// not satisfied.
func (tx *Tx) Assert(p bool) {
	if !p {
		tx.Retry()
	}
}

// catchRetry returns true if fn calls tx.Retry.
func catchRetry(fn func(*Tx), tx *Tx) (retry bool) {
	defer func() {
		if r := recover(); r == Retry {
			retry = true
		} else if r != nil {
			panic(r)
		}
	}()
	fn(tx)
	return
}

// Select runs the supplied functions in order. Execution stops when a
// function succeeds without calling Retry. If no functions succeed, the
// entire selection will be retried.
func Select(fns ...func(*Tx)) func(*Tx) {
	return func(tx *Tx) {
		switch len(fns) {
		case 0:
			// empty Select blocks forever
			tx.Retry()
		case 1:
			fns[0](tx)
		default:
			if catchRetry(fns[0], tx) {
				Select(fns[1:]...)(tx)
			}
		}
	}
}

// Atomically executes the atomic function fn.
func Atomically(fn func(*Tx)) {
retry:
	// run the transaction
	tx := &Tx{
		reads:  make(map[*Var]interface{}),
		writes: make(map[*Var]interface{}),
	}
	if catchRetry(fn, tx) {
		goto retry
	}
	// verify the read log
	globalLock.Lock()
	if !tx.verify() {
		globalLock.Unlock()
		goto retry
	}
	// commit the write log
	if len(tx.writes) > 0 {
		tx.commit()
	}
	globalLock.Unlock()
}

// AtomicGet is a helper function that atomically reads a value.
func AtomicGet(v *Var) interface{} {
	var i interface{}
	Atomically(func(tx *Tx) { i = tx.Get(v) })
	return i
}

// AtomicSet is a helper function that atomically writes a value.
func AtomicSet(v *Var, val interface{}) {
	Atomically(func(tx *Tx) { tx.Set(v, val) })
}

// Compose is a helper function that composes multiple transactions into a
// single transaction.
func Compose(fns ...func(*Tx)) func(*Tx) {
	return func(tx *Tx) {
		for _, f := range fns {
			f(tx)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package tag provides MP3 (ID3: v1, 2.2, 2.3 and 2.4), MP4, FLAC and OGG metadata detection,
// parsing and artwork extraction.

// Detect and parse tag metadata from an io.ReadSeeker (i.e. an *os.File):
// 	m, err := tag.ReadFrom(f)
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	log.Print(m.Format()) // The detected format.
// 	log.Print(m.Title())  // The title of the track (see Metadata interface for more details).
package tag

import (
	"errors"
	"fmt"
	"io"
	"os"
)

// ErrNoTagsFound is the error returned by ReadFrom when the metadata format
// cannot be identified.
var ErrNoTagsFound = errors.New("no tags found")

// ReadFrom detects and parses audio file metadata tags (currently supports ID3v1,2.{2,3,4}, MP4, FLAC/OGG).
// Returns non-nil error if the format of the given data could not be determined, or if there was a problem
// parsing the data.
func ReadFrom(r io.ReadSeeker) (Metadata, error) {
	b, err := readBytes(r, 11)
	if err != nil {
		return nil, err
	}

	_, err = r.Seek(-11, os.SEEK_CUR)
	if err != nil {
		return nil, fmt.Errorf("could not seek back to original position: %v", err)
	}

	switch {
	case string(b[0:4]) == "fLaC":
		return ReadFLACTags(r)

	case string(b[0:4]) == "OggS":
		return ReadOGGTags(r)

	case string(b[4:11]) == "ftypM4A":
		return ReadAtoms(r)

	case string(b[0:3]) == "ID3":
		return ReadID3v2Tags(r)
	}

	m, err := ReadID3v1Tags(r)
	if err != nil {
		if err == ErrNotID3v1 {
			err = ErrNoTagsFound
		}
		return nil, err
	}
	return m, nil
}

// Format is an enumeration of metadata types supported by this package.
type Format string

// Supported tag formats.
const (
	ID3v1   Format = "ID3v1"   // ID3v1 tag format.
	ID3v2_2        = "ID3v2.2" // ID3v2.2 tag format.
	ID3v2_3        = "ID3v2.3" // ID3v2.3 tag format (most common).
	ID3v2_4        = "ID3v2.4" // ID3v2.4 tag format.
	MP4            = "MP4"     // MP4 tag (atom) format.
	VORBIS         = "VORBIS"  // Vorbis Comment tag format.
)

// FileType is an enumeration of the audio file types supported by this package, in particular
// there are audio file types which share metadata formats, and this type is used to distinguish
// between them.
type FileType string

// Supported file types.
const (
	MP3  FileType = "MP3"  // MP3 file
	AAC           = "AAC"  // M4A file (MP4)
	ALAC          = "ALAC" // Apple Lossless file FIXME: actually detect this
	FLAC          = "FLAC" // FLAC file
	OGG           = "OGG"  // OGG file
)

// Metadata is an interface which is used to describe metadata retrieved by this package.
type Metadata interface {
	// Format returns the metadata Format used to encode the data.
	Format() Format

	// FileType returns the file type of the audio file.
	FileType() FileType

	// Title returns the title of the track.
	Title() string

	// Album returns the album name of the track.
	Album() string

	// Artist returns the artist name of the track.
	Artist() string

	// AlbumArtist returns the album artist name of the track.
	AlbumArtist() string

	// Composer returns the composer of the track.
	Composer() string

	// Year returns the year of the track.
	Year() int

	// Genre returns the genre of the track.
	Genre() string

	// Track returns the track number and total tracks, or zero values if unavailable.
	Track() (int, int)

	// Disc returns the disc number and total discs, or zero values if unavailable.
	Disc() (int, int)

	// Picture returns a picture, or nil if not available.
	Picture() *Picture

	// Lyrics returns the lyrics, or an empty string if unavailable.
	Lyrics() string

	// Raw returns the raw mapping of retrieved tag names and associated values.
	// NB: tag/atom names are not standardised between formats.
	Raw() map[string]interface{}
}
<commit_msg>Fixed package doc comment<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package tag provides MP3 (ID3: v1, 2.2, 2.3 and 2.4), MP4, FLAC and OGG metadata detection,
// parsing and artwork extraction.
//
// Detect and parse tag metadata from an io.ReadSeeker (i.e. an *os.File):
// 	m, err := tag.ReadFrom(f)
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	log.Print(m.Format()) // The detected format.
// 	log.Print(m.Title())  // The title of the track (see Metadata interface for more details).
package tag

import (
	"errors"
	"fmt"
	"io"
	"os"
)

// ErrNoTagsFound is the error returned by ReadFrom when the metadata format
// cannot be identified.
var ErrNoTagsFound = errors.New("no tags found")

// ReadFrom detects and parses audio file metadata tags (currently supports ID3v1,2.{2,3,4}, MP4, FLAC/OGG).
// Returns non-nil error if the format of the given data could not be determined, or if there was a problem
// parsing the data.
func ReadFrom(r io.ReadSeeker) (Metadata, error) {
	b, err := readBytes(r, 11)
	if err != nil {
		return nil, err
	}

	_, err = r.Seek(-11, os.SEEK_CUR)
	if err != nil {
		return nil, fmt.Errorf("could not seek back to original position: %v", err)
	}

	switch {
	case string(b[0:4]) == "fLaC":
		return ReadFLACTags(r)

	case string(b[0:4]) == "OggS":
		return ReadOGGTags(r)

	case string(b[4:11]) == "ftypM4A":
		return ReadAtoms(r)

	case string(b[0:3]) == "ID3":
		return ReadID3v2Tags(r)
	}

	m, err := ReadID3v1Tags(r)
	if err != nil {
		if err == ErrNotID3v1 {
			err = ErrNoTagsFound
		}
		return nil, err
	}
	return m, nil
}

// Format is an enumeration of metadata types supported by this package.
type Format string

// Supported tag formats.
const (
	ID3v1   Format = "ID3v1"   // ID3v1 tag format.
	ID3v2_2        = "ID3v2.2" // ID3v2.2 tag format.
	ID3v2_3        = "ID3v2.3" // ID3v2.3 tag format (most common).
	ID3v2_4        = "ID3v2.4" // ID3v2.4 tag format.
	MP4            = "MP4"     // MP4 tag (atom) format.
	VORBIS         = "VORBIS"  // Vorbis Comment tag format.
)

// FileType is an enumeration of the audio file types supported by this package, in particular
// there are audio file types which share metadata formats, and this type is used to distinguish
// between them.
type FileType string

// Supported file types.
const (
	MP3  FileType = "MP3"  // MP3 file
	AAC           = "AAC"  // M4A file (MP4)
	ALAC          = "ALAC" // Apple Lossless file FIXME: actually detect this
	FLAC          = "FLAC" // FLAC file
	OGG           = "OGG"  // OGG file
)

// Metadata is an interface which is used to describe metadata retrieved by this package.
type Metadata interface {
	// Format returns the metadata Format used to encode the data.
	Format() Format

	// FileType returns the file type of the audio file.
	FileType() FileType

	// Title returns the title of the track.
	Title() string

	// Album returns the album name of the track.
	Album() string

	// Artist returns the artist name of the track.
	Artist() string

	// AlbumArtist returns the album artist name of the track.
	AlbumArtist() string

	// Composer returns the composer of the track.
	Composer() string

	// Year returns the year of the track.
	Year() int

	// Genre returns the genre of the track.
	Genre() string

	// Track returns the track number and total tracks, or zero values if unavailable.
	Track() (int, int)

	// Disc returns the disc number and total discs, or zero values if unavailable.
	Disc() (int, int)

	// Picture returns a picture, or nil if not available.
	Picture() *Picture

	// Lyrics returns the lyrics, or an empty string if unavailable.
	Lyrics() string

	// Raw returns the raw mapping of retrieved tag names and associated values.
	// NB: tag/atom names are not standardised between formats.
	Raw() map[string]interface{}
}
<|endoftext|>


<commit_before><commit_msg>remove missing format var<commit_after><|endoftext|>


<commit_before><commit_msg>changed server port to 9331<commit_after><|endoftext|>


<commit_before><commit_msg>:green_heart: Fix invalid JSON for test<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
)

type domainSet map[string]bool

// Basically a concurrent map. I don't want to use channels to implement
// concurrent access to this as I'm comfortable to use locks for simple tasks
// like this
type paraDomainSet struct {
	sync.RWMutex
	domainSet
}

func newDomainSet() domainSet {
	return make(map[string]bool)
}

func (ds domainSet) loadDomainList(fpath string) (lst []string, err error) {
	lst, err = loadDomainList(fpath)
	if err != nil {
		return
	}
	// This executes in single goroutine, so no need to use lock
	for _, v := range lst {
		// debug.Println("loaded domain:", v)
		ds[v] = true
	}
	return
}

func (ds domainSet) toArray() []string {
	l := len(ds)
	lst := make([]string, l, l)

	i := 0
	for k, _ := range ds {
		lst[i] = k
		i++
	}
	return lst
}

func newParaDomainSet() *paraDomainSet {
	return &paraDomainSet{domainSet: newDomainSet()}
}

func (ds *paraDomainSet) add(dm string) {
	ds.Lock()
	ds.domainSet[dm] = true
	ds.Unlock()
}

func (ds *paraDomainSet) has(dm string) bool {
	ds.RLock()
	_, ok := ds.domainSet[dm]
	ds.RUnlock()
	return ok
}

func (ds *paraDomainSet) del(dm string) {
	ds.Lock()
	delete(ds.domainSet, dm)
	ds.Unlock()
}

var blockedDs = newParaDomainSet()
var directDs = newParaDomainSet()

var blockedDomainChanged = false
var directDomainChanged = false

var alwaysBlockedDs = newDomainSet()
var alwaysDirectDs = newDomainSet()
var chouDs = newDomainSet()

func requestDomain(r *Request) string {
	host, _ := splitHostPort(r.URL.Host)
	return host2Domain(host)
}

func inAlwaysDs(dm string) bool {
	return alwaysBlockedDs[dm] || alwaysDirectDs[dm]
}

func hostInAlwaysDirectDs(host string) bool {
	h, _ := splitHostPort(host)
	return alwaysDirectDs[host2Domain(h)]
}

func hostInAlwaysBlockedDs(host string) bool {
	h, _ := splitHostPort(host)
	return alwaysBlockedDs[host2Domain(h)]
}

func isRequestBlocked(r *Request) bool {
	dm := requestDomain(r)
	if alwaysDirectDs[dm] {
		return false
	}
	if alwaysBlockedDs[dm] {
		return true
	}
	return blockedDs.has(dm)
}

func isRequestInChouDs(r *Request) bool {
	dm := requestDomain(r)
	return chouDs[dm]
}

func addBlockedRequest(r *Request) bool {
	host, _ := splitHostPort(r.URL.Host)
	if hostIsIP(host) {
		return false
	}
	dm := host2Domain(host)
	// For chou domain, we should add it to the blocked list in order to use
	// parent proxy, but don't write it back to auto-block file.
	if inAlwaysDs(dm) {
		return false
	}
	if !blockedDs.has(dm) {
		blockedDs.add(dm)
		blockedDomainChanged = true
		debug.Printf("%v added to blocked list\n", dm)
		return true
	}
	// Delete this request from direct domain set
	delDirectRequest(r)
	return false
}

func delBlockedRequest(r *Request) {
	dm := requestDomain(r)
	if blockedDs.has(dm) {
		blockedDs.del(dm)
		blockedDomainChanged = true
		debug.Printf("%v deleted from blocked list\n", dm)
	}
}

func addDirectRequest(r *Request) {
	host, _ := splitHostPort(r.URL.Host)
	if hostIsIP(host) {
		return
	}
	dm := host2Domain(host)
	if inAlwaysDs(dm) || chouDs[dm] {
		return
	}
	if !directDs.has(dm) {
		directDs.add(dm)
		directDomainChanged = true
	}
	// Delete this request from blocked domain set
	delBlockedRequest(r)
}

func delDirectRequest(r *Request) {
	dm := requestDomain(r)
	if directDs.has(dm) {
		directDs.del(dm)
		directDomainChanged = true
	}
}

func writeBlockedDs() {
	if !config.updateBlocked {
		return
	}
	if !blockedDomainChanged {
		return
	}
	writeDomainList(config.blockedFile, blockedDs.toArray())
}

func writeDirectDs() {
	if !config.updateDirect {
		return
	}
	if !directDomainChanged {
		return
	}
	writeDomainList(config.directFile, directDs.toArray())
}

func filterOutChouDs() {
	for k, _ := range chouDs {
		if _, ok := blockedDs.domainSet[k]; ok {
			delete(blockedDs.domainSet, k)
			blockedDomainChanged = true
		}
		if _, ok := directDs.domainSet[k]; ok {
			delete(directDs.domainSet, k)
			directDomainChanged = true
		}
	}
}

func writeDomainSet() {
	filterOutChouDs()

	writeBlockedDs()
	writeDirectDs()
}

func loadDomainList(fpath string) (lst []string, err error) {
	f, err := openFile(fpath)
	if f == nil || err != nil {
		return
	}
	defer f.Close()

	fr := bufio.NewReader(f)
	lst = make([]string, 0)
	var domain string
	for {
		domain, err = ReadLine(fr)
		if err == io.EOF {
			return lst, nil
		} else if err != nil {
			errl.Println("Error reading domain list from:", fpath, err)
			return
		}
		if domain == "" {
			continue
		}
		lst = append(lst, strings.TrimSpace(domain))
	}
	return
}

func writeDomainList(fpath string, lst []string) (err error) {
	tmpPath := path.Join(config.dir, "tmp-domain")
	f, err := os.Create(tmpPath)
	if err != nil {
		errl.Println("Error creating tmp domain list file:", err)
		return
	}

	sort.Sort(sort.StringSlice(lst))

	all := strings.Join(lst, "\n")
	f.WriteString(all)
	f.Close()

	if err = os.Rename(tmpPath, fpath); err != nil {
		errl.Printf("Error moving tmp domain list file to %s: %v\n", fpath, err)
	}
	return
}

var topLevelDomain = map[string]bool{
	"co":  true,
	"org": true,
	"com": true,
	"net": true,
	"edu": true,
}

func host2Domain(host string) (domain string) {
	lastDot := strings.LastIndex(host, ".")
	if lastDot == -1 {
		return host // simple host name, we should not hanlde this
	}
	// Find the 2nd last dot
	dot2ndLast := strings.LastIndex(host[:lastDot], ".")
	if dot2ndLast == -1 {
		return host
	}

	part := host[dot2ndLast+1 : lastDot]
	// If the 2nd last part of a domain name equals to a top level
	// domain, search for the 3rd part in the host name.
	// So domains like bbc.co.uk will not be recorded as co.uk
	if topLevelDomain[part] {
		dot3rdLast := strings.LastIndex(host[:dot2ndLast], ".")
		if dot3rdLast == -1 {
			return host
		}
		return host[dot3rdLast+1:]
	}
	return host[dot2ndLast+1:]
}

func loadDomainSet() {
	blockedDs.loadDomainList(config.blockedFile)
	directDs.loadDomainList(config.directFile)
	alwaysBlockedDs.loadDomainList(config.alwaysBlockedFile)
	alwaysDirectDs.loadDomainList(config.alwaysDirectFile)
	chouDs.loadDomainList(config.chouFile)

	filterOutChouDs()
}
<commit_msg>Sanity check for domain set.<commit_after>package main

import (
	"bufio"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
)

type domainSet map[string]bool

// Basically a concurrent map. I don't want to use channels to implement
// concurrent access to this as I'm comfortable to use locks for simple tasks
// like this
type paraDomainSet struct {
	sync.RWMutex
	domainSet
}

func newDomainSet() domainSet {
	return make(map[string]bool)
}

func (ds domainSet) loadDomainList(fpath string) (lst []string, err error) {
	lst, err = loadDomainList(fpath)
	if err != nil {
		return
	}
	// This executes in single goroutine, so no need to use lock
	for _, v := range lst {
		// debug.Println("loaded domain:", v)
		ds[v] = true
	}
	return
}

func (ds domainSet) toArray() []string {
	l := len(ds)
	lst := make([]string, l, l)

	i := 0
	for k, _ := range ds {
		lst[i] = k
		i++
	}
	return lst
}

func newParaDomainSet() *paraDomainSet {
	return &paraDomainSet{domainSet: newDomainSet()}
}

func (ds *paraDomainSet) add(dm string) {
	ds.Lock()
	ds.domainSet[dm] = true
	ds.Unlock()
}

func (ds *paraDomainSet) has(dm string) bool {
	ds.RLock()
	_, ok := ds.domainSet[dm]
	ds.RUnlock()
	return ok
}

func (ds *paraDomainSet) del(dm string) {
	ds.Lock()
	delete(ds.domainSet, dm)
	ds.Unlock()
}

var blockedDs = newParaDomainSet()
var directDs = newParaDomainSet()

var blockedDomainChanged = false
var directDomainChanged = false

var alwaysBlockedDs = newDomainSet()
var alwaysDirectDs = newDomainSet()
var chouDs = newDomainSet()

func requestDomain(r *Request) string {
	host, _ := splitHostPort(r.URL.Host)
	return host2Domain(host)
}

func inAlwaysDs(dm string) bool {
	return alwaysBlockedDs[dm] || alwaysDirectDs[dm]
}

func hostInAlwaysDirectDs(host string) bool {
	h, _ := splitHostPort(host)
	return alwaysDirectDs[host2Domain(h)]
}

func hostInAlwaysBlockedDs(host string) bool {
	h, _ := splitHostPort(host)
	return alwaysBlockedDs[host2Domain(h)]
}

func isRequestBlocked(r *Request) bool {
	dm := requestDomain(r)
	if alwaysDirectDs[dm] {
		return false
	}
	if alwaysBlockedDs[dm] {
		return true
	}
	return blockedDs.has(dm)
}

func isRequestInChouDs(r *Request) bool {
	dm := requestDomain(r)
	return chouDs[dm]
}

func addBlockedRequest(r *Request) bool {
	host, _ := splitHostPort(r.URL.Host)
	if hostIsIP(host) {
		return false
	}
	dm := host2Domain(host)
	// For chou domain, we should add it to the blocked list in order to use
	// parent proxy, but don't write it back to auto-block file.
	if inAlwaysDs(dm) {
		return false
	}
	if !blockedDs.has(dm) {
		blockedDs.add(dm)
		blockedDomainChanged = true
		debug.Printf("%v added to blocked list\n", dm)
		return true
	}
	// Delete this request from direct domain set
	delDirectRequest(r)
	return false
}

func delBlockedRequest(r *Request) {
	dm := requestDomain(r)
	if blockedDs.has(dm) {
		blockedDs.del(dm)
		blockedDomainChanged = true
		debug.Printf("%v deleted from blocked list\n", dm)
	}
}

func addDirectRequest(r *Request) {
	host, _ := splitHostPort(r.URL.Host)
	if hostIsIP(host) {
		return
	}
	dm := host2Domain(host)
	if inAlwaysDs(dm) || chouDs[dm] {
		return
	}
	if !directDs.has(dm) {
		directDs.add(dm)
		directDomainChanged = true
	}
	// Delete this request from blocked domain set
	delBlockedRequest(r)
}

func delDirectRequest(r *Request) {
	dm := requestDomain(r)
	if directDs.has(dm) {
		directDs.del(dm)
		directDomainChanged = true
	}
}

func writeBlockedDs() {
	if !config.updateBlocked {
		return
	}
	if !blockedDomainChanged {
		return
	}
	writeDomainList(config.blockedFile, blockedDs.toArray())
}

func writeDirectDs() {
	if !config.updateDirect {
		return
	}
	if !directDomainChanged {
		return
	}
	writeDomainList(config.directFile, directDs.toArray())
}

// filter out domain in blocked and direct domain set.
func filterOutDs(ds domainSet) {
	for k, _ := range ds {
		if blockedDs.domainSet[k] {
			delete(blockedDs.domainSet, k)
			blockedDomainChanged = true
		}
		if directDs.domainSet[k] {
			delete(directDs.domainSet, k)
			directDomainChanged = true
		}
	}
}

// If a domain name appears in both blocked and direct domain set, only keep
// it in the blocked set.
func filterOutBlockedDsInDirectDs() {
	for k, _ := range blockedDs.domainSet {
		if directDs.domainSet[k] {
			delete(directDs.domainSet, k)
			directDomainChanged = true
		}
	}
	for k, _ := range alwaysBlockedDs {
		if alwaysDirectDs[k] {
			errl.Printf("%s in both always blocked and direct domain lists, taken as blocked.\n", k)
			delete(alwaysDirectDs, k)
		}
	}
}

func writeDomainSet() {
	// chou domain set maybe added to blocked site during execution,
	// filter them out before writing back to disk.
	filterOutDs(chouDs)

	writeBlockedDs()
	writeDirectDs()
}

func loadDomainList(fpath string) (lst []string, err error) {
	f, err := openFile(fpath)
	if f == nil || err != nil {
		return
	}
	defer f.Close()

	fr := bufio.NewReader(f)
	lst = make([]string, 0)
	var domain string
	for {
		domain, err = ReadLine(fr)
		if err == io.EOF {
			return lst, nil
		} else if err != nil {
			errl.Println("Error reading domain list from:", fpath, err)
			return
		}
		if domain == "" {
			continue
		}
		lst = append(lst, strings.TrimSpace(domain))
	}
	return
}

func writeDomainList(fpath string, lst []string) (err error) {
	tmpPath := path.Join(config.dir, "tmp-domain")
	f, err := os.Create(tmpPath)
	if err != nil {
		errl.Println("Error creating tmp domain list file:", err)
		return
	}

	sort.Sort(sort.StringSlice(lst))

	all := strings.Join(lst, "\n")
	f.WriteString(all)
	f.Close()

	if err = os.Rename(tmpPath, fpath); err != nil {
		errl.Printf("Error moving tmp domain list file to %s: %v\n", fpath, err)
	}
	return
}

var topLevelDomain = map[string]bool{
	"co":  true,
	"org": true,
	"com": true,
	"net": true,
	"edu": true,
}

func host2Domain(host string) (domain string) {
	lastDot := strings.LastIndex(host, ".")
	if lastDot == -1 {
		return host // simple host name, we should not hanlde this
	}
	// Find the 2nd last dot
	dot2ndLast := strings.LastIndex(host[:lastDot], ".")
	if dot2ndLast == -1 {
		return host
	}

	part := host[dot2ndLast+1 : lastDot]
	// If the 2nd last part of a domain name equals to a top level
	// domain, search for the 3rd part in the host name.
	// So domains like bbc.co.uk will not be recorded as co.uk
	if topLevelDomain[part] {
		dot3rdLast := strings.LastIndex(host[:dot2ndLast], ".")
		if dot3rdLast == -1 {
			return host
		}
		return host[dot3rdLast+1:]
	}
	return host[dot2ndLast+1:]
}

func loadDomainSet() {
	blockedDs.loadDomainList(config.blockedFile)
	directDs.loadDomainList(config.directFile)
	alwaysBlockedDs.loadDomainList(config.alwaysBlockedFile)
	alwaysDirectDs.loadDomainList(config.alwaysDirectFile)
	chouDs.loadDomainList(config.chouFile)

	filterOutDs(chouDs)
	filterOutDs(alwaysDirectDs)
	filterOutDs(alwaysBlockedDs)
	filterOutBlockedDsInDirectDs()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"

	"github.com/gabstv/sandpiper/route"
	"github.com/gabstv/sandpiper/server"
	"github.com/gabstv/sandpiper/util"
	"github.com/mattn/go-colorable"
	"github.com/mgutz/ansi"
	"gopkg.in/yaml.v2"
)

var (
	configfile string
	stdout     io.Writer
	stderr     io.Writer
)

func main() {
	stdout = colorable.NewColorableStdout()
	stderr = colorable.NewColorableStderr()
	fmt.Fprintln(stdout, ansi.Color("\nSANDPIPER 1.1.0\n", "green"))
	// init flags
	flag.Parse()
	// print help if needed
	printHelp()
	// find config
	findConfigFile()

	bs, err := ioutil.ReadFile(configfile)
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: Unable to load the config file at %v!\n%v\n",
			ansi.Color(configfile, "red"),
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}

	cfg := &Config{}
	err = yaml.Unmarshal(bs, cfg)
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: Unable to unmarshal the config file at %v!\n%v\n",
			ansi.Color(configfile, "red"),
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}

	// unpack Config
	svCfg := &server.Config{}
	svCfg.Debug = cfg.Debug
	if svCfg.Debug {
		util.DEBUG = true
	}
	svCfg.ListenAddr = cfg.ListenAddr
	svCfg.ListenAddrTLS = cfg.ListenAddrTLS
	svCfg.NumCPU = cfg.NumCPU
	svCfg.FallbackDomain = cfg.FallbackDomain
	svCfg.Graceful = cfg.Graceful
	svCfg.CachePath = cfg.CachePath
	svCfg.APIListen = cfg.APIListen
	svCfg.APIKey = cfg.APIKey
	svCfg.APIDomain = cfg.APIDomain
	svCfg.APIDomainAutocert = cfg.APIDomainAutocert

	// ENV VARS
	if vv := os.Getenv("API_LISTEN"); vv != "" {
		svCfg.APIListen = vv
	}
	if vv := os.Getenv("API_KEY"); vv != "" {
		svCfg.APIKey = vv
	}
	if vv := os.Getenv("API_DOMAIN"); vv != "" {
		svCfg.APIDomain = vv
	}
	if vv := os.Getenv("API_DOMAIN_AUTOCERT"); vv != "" {
		svCfg.APIDomainAutocert = (vv == "1")
	}

	s := server.Default(svCfg)

	for _, v := range cfg.Routes {
		r := route.Route{}
		// apply Websockets config
		r.WsCFG = v.Websockets
		if ct, ok := unpackConnType(v.OutgoingServerConnType); ok {
			r.Server.OutConnType = ct
		} else {
			fmt.Fprintf(stderr, "\n CONFIGURATION ERROR\nDomain: %v\n ERR: %v\n",
				ansi.Color(v.Domain, "yellow"),
				ansi.Color("Invalid conn type "+v.OutgoingServerConnType, "red"))
			os.Exit(1)
		}
		r.Domain = v.Domain
		r.Server.OutAddress = v.OutgoingServerAddress
		r.Certificate.CertFile = v.TLSCertFile
		r.Certificate.KeyFile = v.TLSKeyFile
		r.Autocert = v.Autocert
		r.AuthMode = v.AuthMode
		r.AuthKey = v.AuthKey
		r.AuthValue = v.AuthValue
		err = s.Add(r)
		if err != nil {
			fmt.Fprintf(stderr, "\nERROR: Could not add route %v\n%v\n",
				ansi.Color(v.Domain, "yellow"),
				ansi.Color(err.Error(), "red"))
			os.Exit(1)
		}
		fmt.Fprintf(stdout, "%v: %v\n",
			ansi.Color("Domain added", "green"),
			v.Domain)
	}
	//
	if cfg.Debug {
		fmt.Fprintf(stdout, "%v DEBUG MODE IS %v\n",
			ansi.Color("WARNING:", "yellow"),
			ansi.Color("ON", "green"))
	}
	//
	// Close if received signal
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, os.Interrupt, os.Kill)
		<-sigchan
		s.Close()
	}()
	//
	err = s.Run()
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: %v\n",
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}
}

func printHelp() {
	if len(flag.Args()) < 1 {
		return
	}
	if flag.Arg(0) != "help" {
		return
	}
	fmt.Fprintln(stdout, ansi.Color("Usage:", "blue"))
	fmt.Fprintf(stdout, "sandpiper %v\n", ansi.Color("config.yml", "green"))
	fmt.Fprintf(stdout, "sandpiper %v\n", ansi.Color("help", "green"))
	os.Exit(0)
}

func findConfigFile() {
	if len(flag.Args()) > 0 {
		configfile = flag.Arg(0)
		return
	}
	if env := os.Getenv("SANDPIPER_CONFIG"); env != "" {
		configfile = env
		return
	}
	if _, err := os.Stat("config.yml"); err == nil {
		configfile = "config.yml"
		return
	}
	// no config file found!
	fmt.Fprint(stderr, ansi.Color("No configuration file found!\n\n", "red"))
	fmt.Println(`There are three different ways to fix this:
  - Pass it as a parameter:
     sandpiper /path/to/config.yml
  - Set the env var SANDPIPER_CONFIG
  - Have a config.yml in the current working directory`)
	fmt.Fprintf(stdout, "\nrun %v fo more info.\n", ansi.Color("sandpiper help", "blue"))
	os.Exit(1)
}

func unpackConnType(input string) (route.ConnType, bool) {
	if input == "HTTP" {
		return route.HTTP, true
	}
	if input == "HTTPS" || input == "HTTPS_VERIFY" {
		return route.HTTPS_VERIFY, true
	}
	if input == "HTTPS_SKIP_VERIFY" {
		return route.HTTPS_SKIP_VERIFY, true
	}
	return route.HTTP, false
}

type Config struct {
	Debug             bool          `yaml:"debug"`
	NumCPU            int           `yaml:"num_cpu"`
	ListenAddr        string        `yaml:"listen_addr"`
	ListenAddrTLS     string        `yaml:"listen_addr_tls"`
	Routes            []ConfigRoute `yaml:"routes"`
	FallbackDomain    string        `yaml:"fallback_domain"`
	Graceful          bool          `yaml:"graceful"`
	CachePath         string        `yaml:"cache_path"`
	APIListen         string        `yaml:"api_listen"`
	APIKey            string        `yaml:"api_key"`
	APIDomain         string        `yaml:"api_domain"`
	APIDomainAutocert bool          `yaml:"api_domain_autocert"`
}

type ConfigRoute struct {
	Domain                 string        `yaml:"domain"`
	OutgoingServerConnType string        `yaml:"out_conn_type"`
	OutgoingServerAddress  string        `yaml:"out_addr"`
	TLSCertFile            string        `yaml:"tls_cert_file"`
	TLSKeyFile             string        `yaml:"tls_key_file"`
	Autocert               bool          `yaml:"autocert"`
	Websockets             util.WsConfig `yaml:"websockets"`
	AuthMode               string        `yaml:"auth_mode"`
	AuthKey                string        `yaml:"auth_key"`
	AuthValue              string        `yaml:"auth_value"`
}
<commit_msg>fallback domain env<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"

	"github.com/gabstv/sandpiper/route"
	"github.com/gabstv/sandpiper/server"
	"github.com/gabstv/sandpiper/util"
	"github.com/mattn/go-colorable"
	"github.com/mgutz/ansi"
	"gopkg.in/yaml.v2"
)

var (
	configfile string
	stdout     io.Writer
	stderr     io.Writer
)

func main() {
	stdout = colorable.NewColorableStdout()
	stderr = colorable.NewColorableStderr()
	fmt.Fprintln(stdout, ansi.Color("\nSANDPIPER 1.1.0\n", "green"))
	// init flags
	flag.Parse()
	// print help if needed
	printHelp()
	// find config
	findConfigFile()

	bs, err := ioutil.ReadFile(configfile)
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: Unable to load the config file at %v!\n%v\n",
			ansi.Color(configfile, "red"),
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}

	cfg := &Config{}
	err = yaml.Unmarshal(bs, cfg)
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: Unable to unmarshal the config file at %v!\n%v\n",
			ansi.Color(configfile, "red"),
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}

	// unpack Config
	svCfg := &server.Config{}
	svCfg.Debug = cfg.Debug
	if svCfg.Debug {
		util.DEBUG = true
	}
	svCfg.ListenAddr = cfg.ListenAddr
	svCfg.ListenAddrTLS = cfg.ListenAddrTLS
	svCfg.NumCPU = cfg.NumCPU
	svCfg.FallbackDomain = cfg.FallbackDomain
	svCfg.Graceful = cfg.Graceful
	svCfg.CachePath = cfg.CachePath
	svCfg.APIListen = cfg.APIListen
	svCfg.APIKey = cfg.APIKey
	svCfg.APIDomain = cfg.APIDomain
	svCfg.APIDomainAutocert = cfg.APIDomainAutocert

	// ENV VARS
	if vv := os.Getenv("FALLBACK_DOMAIN"); vv != "" {
		svCfg.FallbackDomain = vv
	}

	if vv := os.Getenv("API_LISTEN"); vv != "" {
		svCfg.APIListen = vv
	}
	if vv := os.Getenv("API_KEY"); vv != "" {
		svCfg.APIKey = vv
	}
	if vv := os.Getenv("API_DOMAIN"); vv != "" {
		svCfg.APIDomain = vv
	}
	if vv := os.Getenv("API_DOMAIN_AUTOCERT"); vv != "" {
		svCfg.APIDomainAutocert = (vv == "1")
	}

	s := server.Default(svCfg)

	for _, v := range cfg.Routes {
		r := route.Route{}
		// apply Websockets config
		r.WsCFG = v.Websockets
		if ct, ok := unpackConnType(v.OutgoingServerConnType); ok {
			r.Server.OutConnType = ct
		} else {
			fmt.Fprintf(stderr, "\n CONFIGURATION ERROR\nDomain: %v\n ERR: %v\n",
				ansi.Color(v.Domain, "yellow"),
				ansi.Color("Invalid conn type "+v.OutgoingServerConnType, "red"))
			os.Exit(1)
		}
		r.Domain = v.Domain
		r.Server.OutAddress = v.OutgoingServerAddress
		r.Certificate.CertFile = v.TLSCertFile
		r.Certificate.KeyFile = v.TLSKeyFile
		r.Autocert = v.Autocert
		r.AuthMode = v.AuthMode
		r.AuthKey = v.AuthKey
		r.AuthValue = v.AuthValue
		err = s.Add(r)
		if err != nil {
			fmt.Fprintf(stderr, "\nERROR: Could not add route %v\n%v\n",
				ansi.Color(v.Domain, "yellow"),
				ansi.Color(err.Error(), "red"))
			os.Exit(1)
		}
		fmt.Fprintf(stdout, "%v: %v\n",
			ansi.Color("Domain added", "green"),
			v.Domain)
	}
	//
	if cfg.Debug {
		fmt.Fprintf(stdout, "%v DEBUG MODE IS %v\n",
			ansi.Color("WARNING:", "yellow"),
			ansi.Color("ON", "green"))
	}
	//
	// Close if received signal
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, os.Interrupt, os.Kill)
		<-sigchan
		s.Close()
	}()
	//
	err = s.Run()
	if err != nil {
		fmt.Fprintf(stderr, "ERROR: %v\n",
			ansi.Color(err.Error(), "red"))
		os.Exit(1)
	}
}

func printHelp() {
	if len(flag.Args()) < 1 {
		return
	}
	if flag.Arg(0) != "help" {
		return
	}
	fmt.Fprintln(stdout, ansi.Color("Usage:", "blue"))
	fmt.Fprintf(stdout, "sandpiper %v\n", ansi.Color("config.yml", "green"))
	fmt.Fprintf(stdout, "sandpiper %v\n", ansi.Color("help", "green"))
	os.Exit(0)
}

func findConfigFile() {
	if len(flag.Args()) > 0 {
		configfile = flag.Arg(0)
		return
	}
	if env := os.Getenv("SANDPIPER_CONFIG"); env != "" {
		configfile = env
		return
	}
	if _, err := os.Stat("config.yml"); err == nil {
		configfile = "config.yml"
		return
	}
	// no config file found!
	fmt.Fprint(stderr, ansi.Color("No configuration file found!\n\n", "red"))
	fmt.Println(`There are three different ways to fix this:
  - Pass it as a parameter:
     sandpiper /path/to/config.yml
  - Set the env var SANDPIPER_CONFIG
  - Have a config.yml in the current working directory`)
	fmt.Fprintf(stdout, "\nrun %v fo more info.\n", ansi.Color("sandpiper help", "blue"))
	os.Exit(1)
}

func unpackConnType(input string) (route.ConnType, bool) {
	if input == "HTTP" {
		return route.HTTP, true
	}
	if input == "HTTPS" || input == "HTTPS_VERIFY" {
		return route.HTTPS_VERIFY, true
	}
	if input == "HTTPS_SKIP_VERIFY" {
		return route.HTTPS_SKIP_VERIFY, true
	}
	return route.HTTP, false
}

type Config struct {
	Debug             bool          `yaml:"debug"`
	NumCPU            int           `yaml:"num_cpu"`
	ListenAddr        string        `yaml:"listen_addr"`
	ListenAddrTLS     string        `yaml:"listen_addr_tls"`
	Routes            []ConfigRoute `yaml:"routes"`
	FallbackDomain    string        `yaml:"fallback_domain"`
	Graceful          bool          `yaml:"graceful"`
	CachePath         string        `yaml:"cache_path"`
	APIListen         string        `yaml:"api_listen"`
	APIKey            string        `yaml:"api_key"`
	APIDomain         string        `yaml:"api_domain"`
	APIDomainAutocert bool          `yaml:"api_domain_autocert"`
}

type ConfigRoute struct {
	Domain                 string        `yaml:"domain"`
	OutgoingServerConnType string        `yaml:"out_conn_type"`
	OutgoingServerAddress  string        `yaml:"out_addr"`
	TLSCertFile            string        `yaml:"tls_cert_file"`
	TLSKeyFile             string        `yaml:"tls_key_file"`
	Autocert               bool          `yaml:"autocert"`
	Websockets             util.WsConfig `yaml:"websockets"`
	AuthMode               string        `yaml:"auth_mode"`
	AuthKey                string        `yaml:"auth_key"`
	AuthValue              string        `yaml:"auth_value"`
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

// Package simulator provides a go-tpm interface to the IBM TPM2 simulator.
package simulator

// CFLAGS and LDFLAGS modified from upstream Makefile to get the library
// building with cgo: -Wno-unused-variable is to work around a cgo bug and
// -Wno-strict-aliasing is due to unsafe pointer math in the TPM simulator.

// #cgo CFLAGS: -DTPM_POSIX -DVTPM=NO -DUSE_BIT_FIELD_STRUCTURES=NO -Wall -Wno-expansion-to-defined -Wno-self-assign -Wno-unused-variable -Wno-strict-aliasing -Wnested-externs -Wsign-compare
// #cgo LDFLAGS: -lcrypto -lpthread -lrt
//
// #include <stdlib.h>
// #include "Tpm.h"
import "C"
import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"unsafe"

	"github.com/google/go-tpm/tpm2"
)

// Simulator represents a go-tpm compatible interface to the IBM TPM2 simulator.
// Similar to the file-based (for linux) or syscall-based (for Windows) TPM
// handles, no synchronization is provided; the same simulator handle should not
// be used from multiple threads.
type Simulator struct {
	buf bytes.Buffer
}

// ErrSimulatorInUse indicates another open Simulator already exists.
var ErrSimulatorInUse = errors.New("simulator is being used by another caller")

// The simulator is a global resource, so we use the variables below to make
// sure we only ever have one open reference to the Simulator at a time.
var (
	lock  sync.Mutex
	inUse bool
)

// Get the pointer to an initialized, powered on, and started simulator. As only
// one simulator may be running at a time, a second call to Get() will return
// ErrSimulatorInUse until the first Simulator is Closed.
func Get() (*Simulator, error) {
	lock.Lock()
	defer lock.Unlock()
	if inUse {
		return nil, ErrSimulatorInUse
	}

	simulator := &Simulator{}
	if err := simulator.on(true); err != nil {
		return nil, err
	}
	inUse = true
	return simulator, nil
}

// Reset the TPM as if the host computer had rebooted.
func (s *Simulator) Reset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(false)
}

// ManufactureReset behaves like Reset() except that the TPM is complete wiped.
// All data (NVData, Hierarchy seeds, etc...) is cleared or reset.
func (s *Simulator) ManufactureReset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(true)
}

// Write executes the command specified by commandBuffer. The command response
// can be retrieved with a subsequent call to Read().
func (s *Simulator) Write(commandBuffer []byte) (int, error) {
	resp, err := runCommand(commandBuffer)
	if err != nil {
		return 0, err
	}
	return s.buf.Write(resp)
}

// Read gets the response of a command previously issued by calling Write().
func (s *Simulator) Read(responseBuffer []byte) (int, error) {
	return s.buf.Read(responseBuffer)
}

// Close cleans up and stops the simulator, Close() should always be called when
// the Simulator is no longer needed, freeing up other callers to use Get().
func (s *Simulator) Close() error {
	lock.Lock()
	defer lock.Unlock()
	inUse = false
	return s.off()
}

func (s *Simulator) on(manufactureReset bool) error {
	// Setup the simulator to receive commands
	C._plat__Signal_PowerOn()
	C._plat__Signal_Reset()
	C._plat__SetNvAvail()
	C._plat__Signal_PhysicalPresenceOn()
	if manufactureReset {
		if rc := C.TPM_Manufacture(1); rc != C.TPM_RC_SUCCESS {
			return fmt.Errorf("manufacture reset failed: code %x", rc)
		}
	}
	// TPM2_Setup must be the first command the TPM receives
	if err := tpm2.Startup(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("startup: %v", err)
	}
	return nil
}

func (s *Simulator) off() error {
	// TPM2_Shutdown must be the first command the TPM receives. We call
	// Shutdown with StartupClear to simulate a full reboot.
	if err := tpm2.Shutdown(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("shutdown: %v", err)
	}
	C._plat__Signal_PhysicalPresenceOff()
	C._plat__ClearNvAvail()
	C._plat__Signal_PowerOff()
	return nil
}

func runCommand(cmd []byte) ([]byte, error) {
	responseSize := C.uint32_t(C.MAX_RESPONSE_SIZE)
	// _plat__RunCommand takes the response buffer as a uint8_t** instead of as
	// a uint8_t*. As Cgo bans go pointers to go pointers, we must allocate the
	// response buffer with malloc().
	response := C.malloc(C.size_t(responseSize))
	defer C.free(response)
	// Make a copy of the response pointer, so we can be sure _plat__RunCommand
	// doesn't modify the pointer (it _is_ expected to modify the buffer).
	responsePtr := (*C.uint8_t)(response)

	C._plat__RunCommand(C.uint32_t(len(cmd)), (*C.uint8_t)(&cmd[0]),
		&responseSize, &responsePtr)
	// As long as NO_FAIL_TRACE is not defined, debug error information is
	// written to certain global variables on internal failure.
	if C.g_inFailureMode == C.TRUE {
		if C.s_failFunction == 0 {
			return nil, errors.New("internal failure (NO_FAIL_TRACE)")
		}
		// This terrible (and quite unsafe) pointer magic is mandated by the
		// TCG TPM2 Specification, Part 4 (code), section 9.17.4.2.
		functionName := (*C.char)(unsafe.Pointer(uintptr(C.s_failFunction)))
		return nil, fmt.Errorf("internal failure: %s, line %d, code %d",
			C.GoString(functionName), C.s_failLine, C.s_failCode)
	}
	if response != unsafe.Pointer(responsePtr) {
		panic("Response pointer shouldn't be modified on success")
	}
	return C.GoBytes(response, C.int(responseSize)), nil
}
<commit_msg>Prevent simulator from returning TPM_RC_RETRY<commit_after>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

// Package simulator provides a go-tpm interface to the IBM TPM2 simulator.
package simulator

// CFLAGS and LDFLAGS modified from upstream Makefile to get the library
// building with cgo: -Wno-unused-variable is to work around a cgo bug and
// -Wno-strict-aliasing is due to unsafe pointer math in the TPM simulator.

// #cgo CFLAGS: -DTPM_POSIX -DVTPM=NO -DUSE_BIT_FIELD_STRUCTURES=NO -DUSE_DA_USED=NO -Wall -Wno-expansion-to-defined -Wno-self-assign -Wno-unused-variable -Wno-strict-aliasing -Wnested-externs -Wsign-compare
// #cgo LDFLAGS: -lcrypto -lpthread -lrt
//
// #include <stdlib.h>
// #include "Tpm.h"
import "C"
import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"unsafe"

	"github.com/google/go-tpm/tpm2"
)

// Simulator represents a go-tpm compatible interface to the IBM TPM2 simulator.
// Similar to the file-based (for linux) or syscall-based (for Windows) TPM
// handles, no synchronization is provided; the same simulator handle should not
// be used from multiple threads.
type Simulator struct {
	buf bytes.Buffer
}

// ErrSimulatorInUse indicates another open Simulator already exists.
var ErrSimulatorInUse = errors.New("simulator is being used by another caller")

// The simulator is a global resource, so we use the variables below to make
// sure we only ever have one open reference to the Simulator at a time.
var (
	lock  sync.Mutex
	inUse bool
)

// Get the pointer to an initialized, powered on, and started simulator. As only
// one simulator may be running at a time, a second call to Get() will return
// ErrSimulatorInUse until the first Simulator is Closed.
func Get() (*Simulator, error) {
	lock.Lock()
	defer lock.Unlock()
	if inUse {
		return nil, ErrSimulatorInUse
	}

	simulator := &Simulator{}
	if err := simulator.on(true); err != nil {
		return nil, err
	}
	inUse = true
	return simulator, nil
}

// Reset the TPM as if the host computer had rebooted.
func (s *Simulator) Reset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(false)
}

// ManufactureReset behaves like Reset() except that the TPM is complete wiped.
// All data (NVData, Hierarchy seeds, etc...) is cleared or reset.
func (s *Simulator) ManufactureReset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(true)
}

// Write executes the command specified by commandBuffer. The command response
// can be retrieved with a subsequent call to Read().
func (s *Simulator) Write(commandBuffer []byte) (int, error) {
	resp, err := runCommand(commandBuffer)
	if err != nil {
		return 0, err
	}
	return s.buf.Write(resp)
}

// Read gets the response of a command previously issued by calling Write().
func (s *Simulator) Read(responseBuffer []byte) (int, error) {
	return s.buf.Read(responseBuffer)
}

// Close cleans up and stops the simulator, Close() should always be called when
// the Simulator is no longer needed, freeing up other callers to use Get().
func (s *Simulator) Close() error {
	lock.Lock()
	defer lock.Unlock()
	inUse = false
	return s.off()
}

func (s *Simulator) on(manufactureReset bool) error {
	// Setup the simulator to receive commands
	C._plat__Signal_PowerOn()
	C._plat__Signal_Reset()
	C._plat__SetNvAvail()
	C._plat__Signal_PhysicalPresenceOn()
	if manufactureReset {
		if rc := C.TPM_Manufacture(1); rc != C.TPM_RC_SUCCESS {
			return fmt.Errorf("manufacture reset failed: code %x", rc)
		}
	}
	// TPM2_Setup must be the first command the TPM receives
	if err := tpm2.Startup(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("startup: %v", err)
	}
	return nil
}

func (s *Simulator) off() error {
	// TPM2_Shutdown must be the first command the TPM receives. We call
	// Shutdown with StartupClear to simulate a full reboot.
	if err := tpm2.Shutdown(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("shutdown: %v", err)
	}
	C._plat__Signal_PhysicalPresenceOff()
	C._plat__ClearNvAvail()
	C._plat__Signal_PowerOff()
	return nil
}

func runCommand(cmd []byte) ([]byte, error) {
	responseSize := C.uint32_t(C.MAX_RESPONSE_SIZE)
	// _plat__RunCommand takes the response buffer as a uint8_t** instead of as
	// a uint8_t*. As Cgo bans go pointers to go pointers, we must allocate the
	// response buffer with malloc().
	response := C.malloc(C.size_t(responseSize))
	defer C.free(response)
	// Make a copy of the response pointer, so we can be sure _plat__RunCommand
	// doesn't modify the pointer (it _is_ expected to modify the buffer).
	responsePtr := (*C.uint8_t)(response)

	C._plat__RunCommand(C.uint32_t(len(cmd)), (*C.uint8_t)(&cmd[0]),
		&responseSize, &responsePtr)
	// As long as NO_FAIL_TRACE is not defined, debug error information is
	// written to certain global variables on internal failure.
	if C.g_inFailureMode == C.TRUE {
		if C.s_failFunction == 0 {
			return nil, errors.New("internal failure (NO_FAIL_TRACE)")
		}
		// This terrible (and quite unsafe) pointer magic is mandated by the
		// TCG TPM2 Specification, Part 4 (code), section 9.17.4.2.
		functionName := (*C.char)(unsafe.Pointer(uintptr(C.s_failFunction)))
		return nil, fmt.Errorf("internal failure: %s, line %d, code %d",
			C.GoString(functionName), C.s_failLine, C.s_failCode)
	}
	if response != unsafe.Pointer(responsePtr) {
		panic("Response pointer shouldn't be modified on success")
	}
	return C.GoBytes(response, C.int(responseSize)), nil
}
<|endoftext|>


<commit_before>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

const lastCompiled = "27 Apr 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")
	flag.Parse()

	fmt.Println(" dsrt will display a directory by date or size.  Written in Go.  LastCompiled ", lastCompiled)
	fmt.Println()

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := int(f.Size())
			sizestr := strconv.Itoa(sizeint)
			if sizeint > 100000 {
				sizestr = AddCommas(sizestr)
			}
			//			fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
			fmt.Printf("%10v %15s %s %s\n", f.Mode(), sizestr, s, f.Name())
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//-----------------------------------------------------------------------------------------------------------------------------

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<commit_msg>modified:   dsrt/dsrt.go -- intermed state of UID, GID<commit_after>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
)

const lastCompiled = "20 May 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
19 May 19 -- Will now show the uid:gid for linux.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var userptr *user.User
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")
	flag.Parse()

	fmt.Println(" dsrt will display a directory by date or size.  Written in Go.  LastCompiled ", lastCompiled)
	fmt.Println()

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	uid := os.Getuid() // int
	gid := os.Getgid() // int
	systemStr := ""
	linuxflag := runtime.GOOS == "linux"
	if linuxflag {
		systemStr = "Linux"
	} else if runtime.GOOS == "windows" {
		systemStr = "Windows"
	} else {
		systemStr = "Mac, maybe"
	}

	userptr, err = user.Current()
	if err != nil {
		fmt.Println(" user.Current error is ", err, "Exiting.")
		os.Exit(1)
	}

	fmt.Printf("uid = %d, gid = %d, on a computer running %s for %s:%s Username %s, Name %s, HomeDir %s \n",
		uid, gid, systemStr, userptr.Uid, userptr.Gid, userptr.Username, userptr.Name, userptr.HomeDir)
	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			sysUID := f.Sys().(*syscall.Stat_t).Uid
			sysGID := f.Sys().(*syscall.Stat_t).Gid
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := int(f.Size())
			sizestr := strconv.Itoa(sizeint)
			if sizeint > 100000 {
				sizestr = AddCommas(sizestr)
			}
			//			fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
			fmt.Printf("%10v %d %d %15s %s %s\n", f.Mode(), sysUID, sysGID, sizestr, s, f.Name())
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//-----------------------------------------------------------------------------------------------------------------------------

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<|endoftext|>


<commit_before>package echo_test

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"testing"

	"encoding/json"

	"github.com/stretchr/testify/assert"
	. "github.com/webx-top/echo"
	test "github.com/webx-top/echo/testing"
)

func request(method, path string, e *Echo, afters ...func(*http.Request)) (int, string) {
	rec := test.Request(method, path, e, afters...)
	return rec.Code, rec.Body.String()
}

func TestEchoMiddleware(t *testing.T) {
	e := New()
	buf := new(bytes.Buffer)

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-1")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("1")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("2")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("3")
			return next.Handle(c)
		}
	})

	// Route
	e.Get("/", func(c Context) error {
		return c.String("OK")
	})

	e.RebuildRouter()

	c, b := request(GET, "/", e)
	assert.Equal(t, "-1123", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
}

func TestEchoMiddlewareError(t *testing.T) {
	e := New()
	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return errors.New("error")
		}
	})
	e.Get("/", NotFoundHandler)

	e.RebuildRouter()

	c, _ := request(GET, "/", e)
	assert.Equal(t, http.StatusInternalServerError, c)
}

func TestGroupMiddleware(t *testing.T) {
	e := New()
	buf := new(bytes.Buffer)

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-1")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("0")
			return next.Handle(c)
		}
	})

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-3")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-2")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("1")
			return next.Handle(c)
		}
	})

	g := e.Group("/", func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("2")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("3")
			return next.Handle(c)
		}
	})

	// Route
	g.Get("", func(c Context) error {
		return c.String("OK")
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("4")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("5")
			return next.Handle(c)
		}
	})

	e.RebuildRouter()

	c, b := request(GET, "/", e)
	assert.Equal(t, "-3-2-1012345", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)

	buf = new(bytes.Buffer)
	e.RebuildRouter()
	c, b = request(GET, "/", e)
	assert.Equal(t, "-3-2-1012345", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
}

func TestEchoHandler(t *testing.T) {
	e := New()

	// HandlerFunc
	e.Get("/ok", func(c Context) error {
		return c.String("OK")
	})
	e.Get("/view/:id", func(c Context) error {
		return c.String(c.Param(`id`))
	}).SetName(`view`)
	e.Get("/file/*", func(c Context) error {
		return c.String(c.P(0))
	})
	g := e.Host(".admpub.com")
	g.Get("/host", func(c Context) error {
		return c.String(c.Host())
	})

	e.RebuildRouter()

	assert.Equal(t, `/view/8`, e.URI(`view`, 8))

	c, b := request(GET, "/ok", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
	c, b = request(GET, "/view/123", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "123", b)
	c, b = request(POST, "/view/0", e)
	assert.Equal(t, http.StatusMethodNotAllowed, c)
	assert.Equal(t, "Method Not Allowed", b)
	c, b = request(GET, "/file/path/to/file.js", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "path/to/file.js", b)
	c, b = request(GET, "/host", e, func(req *http.Request) {
		req.Host = "test.admpub.com"
	})
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "test.admpub.com", b)
}

func TestEchoRouter(t *testing.T) {
	e := New()

	e.Get("/router/:n/list", func(c Context) error {
		//Dump(c.Route())
		return c.String(c.Param(`n`))
	})
	e.RebuildRouter()

	c, b := request(GET, "/router/123/list", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "123", b)
}

func TestEchoMeta(t *testing.T) {
	e := New()

	g := e.Group("/root")

	g.Get("/", e.MetaHandler(
		H{"version": 1.0, "data": H{"by": "handler"}},
		func(c Context) error {
			return c.JSON(c.Route().Meta)
		},
	))

	e.RebuildRouter()

	var meta H

	for _, route := range e.Routes() {
		if route.Path == "/root/" {
			meta = route.Meta
		}
	}
	expected := H{
		"version": 1.0,
		"data": H{
			"by": "handler",
		},
	}
	assert.Equal(t, expected, meta)

	c, b := request(GET, "/root/", e)
	assert.Equal(t, http.StatusOK, c)
	expected2, _ := json.Marshal(expected)
	assert.Equal(t, string(expected2), b)
}

func TestEchoData(t *testing.T) {
	data := NewData(nil)
	data.SetCode(0)
	assert.Equal(t, 0, data.Code.Int())
	assert.Equal(t, `0`, fmt.Sprintf(`%d`, data.Code))
	assert.Equal(t, `Failure`, fmt.Sprintf(`%v`, data.Code))
	assert.Equal(t, `Failure`, fmt.Sprintf(`%s`, data.Code))
	assert.Equal(t, `Failure`, data.State)
}
<commit_msg>update<commit_after>package echo_test

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"testing"

	"encoding/json"

	"github.com/stretchr/testify/assert"
	. "github.com/webx-top/echo"
	test "github.com/webx-top/echo/testing"
)

func request(method, path string, e *Echo, afters ...func(*http.Request)) (int, string) {
	rec := test.Request(method, path, e, afters...)
	return rec.Code, rec.Body.String()
}

func TestEchoMiddleware(t *testing.T) {
	e := New()
	buf := new(bytes.Buffer)

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-1")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("1")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("2")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("3")
			return next.Handle(c)
		}
	})

	// Route
	e.Get("/", func(c Context) error {
		return c.String("OK")
	})

	e.RebuildRouter()

	c, b := request(GET, "/", e)
	assert.Equal(t, "-1123", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
}

func TestEchoMiddlewareError(t *testing.T) {
	e := New()
	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return errors.New("error")
		}
	})
	e.Get("/", NotFoundHandler)

	e.RebuildRouter()

	c, _ := request(GET, "/", e)
	assert.Equal(t, http.StatusInternalServerError, c)
}

func TestGroupMiddleware(t *testing.T) {
	e := New()
	buf := new(bytes.Buffer)

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-1")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("0")
			return next.Handle(c)
		}
	})

	e.Pre(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-3")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			assert.Empty(t, c.Path())
			buf.WriteString("-2")
			return next.Handle(c)
		}
	})

	e.Use(func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("1")
			return next.Handle(c)
		}
	})

	g := e.Group("/", func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("2")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("3")
			return next.Handle(c)
		}
	})

	// Route
	g.Get("", func(c Context) error {
		return c.String("OK")
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("4")
			return next.Handle(c)
		}
	}, func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			buf.WriteString("5")
			return next.Handle(c)
		}
	})

	e.RebuildRouter()

	c, b := request(GET, "/", e)
	assert.Equal(t, "-3-2-1012345", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)

	buf = new(bytes.Buffer)
	e.RebuildRouter()
	c, b = request(GET, "/", e)
	assert.Equal(t, "-3-2-1012345", buf.String())
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
}

func TestEchoHandler(t *testing.T) {
	e := New()

	// HandlerFunc
	e.Get("/ok", func(c Context) error {
		return c.String("OK")
	})
	e.Get("/view/:id", func(c Context) error {
		return c.String(c.Param(`id`))
	}).SetName(`view`)
	e.Get("/file/*", func(c Context) error {
		return c.String(c.P(0))
	})
	g := e.Host(".admpub.com")
	g.Get("/host", func(c Context) error {
		return c.String(c.Host())
	})

	e.RebuildRouter()

	assert.Equal(t, `/view/8`, e.URI(`view`, 8))

	c, b := request(GET, "/ok", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "OK", b)
	c, b = request(GET, "/view/123", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "123", b)
	c, b = request(POST, "/view/0", e)
	assert.Equal(t, http.StatusMethodNotAllowed, c)
	assert.Equal(t, "Method Not Allowed", b)
	c, b = request(GET, "/file/path/to/file.js", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "path/to/file.js", b)
	c, b = request(GET, "/host", e, func(req *http.Request) {
		req.Host = "test.admpub.com"
	})
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "test.admpub.com", b)
	c, b = request(GET, "/host", e, func(req *http.Request) {
		req.Host = "test-b.admpub.com"
	})
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "test-b.admpub.com", b)
}

func TestEchoRouter(t *testing.T) {
	e := New()

	e.Get("/router/:n/list", func(c Context) error {
		//Dump(c.Route())
		return c.String(c.Param(`n`))
	})
	e.RebuildRouter()

	c, b := request(GET, "/router/123/list", e)
	assert.Equal(t, http.StatusOK, c)
	assert.Equal(t, "123", b)
}

func TestEchoMeta(t *testing.T) {
	e := New()

	g := e.Group("/root")

	g.Get("/", e.MetaHandler(
		H{"version": 1.0, "data": H{"by": "handler"}},
		func(c Context) error {
			return c.JSON(c.Route().Meta)
		},
	))

	e.RebuildRouter()

	var meta H

	for _, route := range e.Routes() {
		if route.Path == "/root/" {
			meta = route.Meta
		}
	}
	expected := H{
		"version": 1.0,
		"data": H{
			"by": "handler",
		},
	}
	assert.Equal(t, expected, meta)

	c, b := request(GET, "/root/", e)
	assert.Equal(t, http.StatusOK, c)
	expected2, _ := json.Marshal(expected)
	assert.Equal(t, string(expected2), b)
}

func TestEchoData(t *testing.T) {
	data := NewData(nil)
	data.SetCode(0)
	assert.Equal(t, 0, data.Code.Int())
	assert.Equal(t, `0`, fmt.Sprintf(`%d`, data.Code))
	assert.Equal(t, `Failure`, fmt.Sprintf(`%v`, data.Code))
	assert.Equal(t, `Failure`, fmt.Sprintf(`%s`, data.Code))
	assert.Equal(t, `Failure`, data.State)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"os"
)

func main() {
	viper.SetConfigName("config")
	viper.SetConfigType("yaml")

	for _, p := range []string{os.Getenv("ECTL_CONFIG_PATH"), ".", "/etc/ectl", "~/.ectl", "~/.config/ectl"} {
		viper.AddConfigPath(p)
	}

	err := viper.ReadInConfig()
	if err != nil {
		glog.Errorf("Fatal error config file: %s \n", err)
		os.Exit(1)
	}

	var run Runtime
	err = viper.Sub("enjoliver").Unmarshal(&run.Config)
	if err != nil {
		glog.Errorf("unable to decode into struct, %v", err)
		os.Exit(1)
	}

	var getCmd = &cobra.Command{
		Use:   "get",
		Short: "Visualize verb",
	}

	var endpointCmd = &cobra.Command{
		Use:     "endpoint",
		Aliases: []string{"ep"},
		Short:   "URI for applications in the cluster",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayEndpoints()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var componentStatusCmd = &cobra.Command{
		Use:     "componentstatus",
		Aliases: []string{"cs", "componentstatuses"},
		Short:   "Status of components",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			if cmd.Flag("control-plane").Value.String() == "false" && cmd.Flag("node").Value.String() == "false" {
				return fmt.Errorf("Need to select at least one machine role: [--%s -%s] or [--%s -%s]",
					cmd.Flag("control-plane").Name, cmd.Flag("control-plane").Shorthand,
					cmd.Flag("node").Name, cmd.Flag("node").Shorthand)
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayComponentStatus()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var binaryVersionCmd = &cobra.Command{
		Use:     "binaryversion",
		Aliases: []string{"bv"},
		Short:   "Version of binaries",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			if cmd.Flag("control-plane").Value.String() == "false" && cmd.Flag("node").Value.String() == "false" {
				return fmt.Errorf("Need to select at least one machine role: [--%s -%s] or [--%s -%s]",
					cmd.Flag("control-plane").Name, cmd.Flag("control-plane").Shorthand,
					cmd.Flag("node").Name, cmd.Flag("node").Shorthand)
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayBinaryVersion()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var rootCmd = &cobra.Command{Use: "Enjoliver control"}

	rootCmd.AddCommand(getCmd)
	getCmd.PersistentFlags().StringVarP(&run.Cluster, "cluster", "c", "", fmt.Sprintf("Cluster in [%s]", joinMap(run.Config.Clusters, " ")))
	getCmd.PersistentFlags().StringVarP(&run.Output, "output", "o", AsciiDisplay, "formatting output for the console")
	getCmd.PersistentFlags().BoolVar(&run.HideAsciiHeader, "no-header", false, "with header in ascii display")
	getCmd.AddCommand(endpointCmd)

	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Fleet, "fleet", "F", false, "Fleet")
	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Kubernetes, "kubernetes", "K", false, "Kubernetes")
	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Vault, "vault", "V", false, "Vault")

	getCmd.AddCommand(componentStatusCmd)
	componentStatusCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesControlPlane, "control-plane", "C", false, "Kubernetes control plane")
	componentStatusCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesNode, "node", "N", false, "Kubernetes node")

	getCmd.AddCommand(binaryVersionCmd)
	binaryVersionCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesControlPlane, "control-plane", "C", false, "Kubernetes control plane")
	binaryVersionCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesNode, "node", "N", false, "Kubernetes node")

	rootCmd.Execute()
	return
}
<commit_msg>Load from environment variable the configuration path<commit_after>package main

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"os"
)

func main() {
	viper.SetConfigName("config")
	viper.SetConfigType("yaml")

	envConfig := os.Getenv("ECTL_CONFIG_PATH")
	if envConfig != "" {
		viper.AddConfigPath(envConfig)
	}
	for _, p := range []string{".", "/etc/ectl", "~/.ectl", "~/.config/ectl"} {
		viper.AddConfigPath(p)
	}

	err := viper.ReadInConfig()
	if err != nil {
		glog.Errorf("Fatal error config file: %s \n", err)
		os.Exit(1)
	}

	var run Runtime
	err = viper.Sub("enjoliver").Unmarshal(&run.Config)
	if err != nil {
		glog.Errorf("unable to decode into struct, %v", err)
		os.Exit(1)
	}

	var getCmd = &cobra.Command{
		Use:   "get",
		Short: "Visualize verb",
	}

	var endpointCmd = &cobra.Command{
		Use:     "endpoint",
		Aliases: []string{"ep"},
		Short:   "URI for applications in the cluster",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayEndpoints()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var componentStatusCmd = &cobra.Command{
		Use:     "componentstatus",
		Aliases: []string{"cs", "componentstatuses"},
		Short:   "Status of components",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			if cmd.Flag("control-plane").Value.String() == "false" && cmd.Flag("node").Value.String() == "false" {
				return fmt.Errorf("Need to select at least one machine role: [--%s -%s] or [--%s -%s]",
					cmd.Flag("control-plane").Name, cmd.Flag("control-plane").Shorthand,
					cmd.Flag("node").Name, cmd.Flag("node").Shorthand)
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayComponentStatus()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var binaryVersionCmd = &cobra.Command{
		Use:     "binaryversion",
		Aliases: []string{"bv"},
		Short:   "Version of binaries",
		Long:    "long",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			clusterName := cmd.Flag("cluster").Value.String()
			_, ok := run.Config.Clusters[clusterName]
			if clusterName == "" || !ok {
				return fmt.Errorf("--cluster %q is invalid, valid are: [%s]\n", clusterName, joinMap(run.Config.Clusters, " "))
			}
			if cmd.Flag("control-plane").Value.String() == "false" && cmd.Flag("node").Value.String() == "false" {
				return fmt.Errorf("Need to select at least one machine role: [--%s -%s] or [--%s -%s]",
					cmd.Flag("control-plane").Name, cmd.Flag("control-plane").Shorthand,
					cmd.Flag("node").Name, cmd.Flag("node").Shorthand)
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			err := run.DisplayBinaryVersion()
			if err != nil {
				glog.Errorf("err: %s", err.Error())
				os.Exit(2)
			}
		},
	}

	var rootCmd = &cobra.Command{Use: "Enjoliver control"}

	rootCmd.AddCommand(getCmd)
	getCmd.PersistentFlags().StringVarP(&run.Cluster, "cluster", "c", "", fmt.Sprintf("Cluster in [%s]", joinMap(run.Config.Clusters, " ")))
	getCmd.PersistentFlags().StringVarP(&run.Output, "output", "o", AsciiDisplay, "formatting output for the console")
	getCmd.PersistentFlags().BoolVar(&run.HideAsciiHeader, "no-header", false, "with header in ascii display")
	getCmd.AddCommand(endpointCmd)

	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Fleet, "fleet", "F", false, "Fleet")
	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Kubernetes, "kubernetes", "K", false, "Kubernetes")
	endpointCmd.Flags().BoolVarP(&run.EndpointDisplay.Vault, "vault", "V", false, "Vault")

	getCmd.AddCommand(componentStatusCmd)
	componentStatusCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesControlPlane, "control-plane", "C", false, "Kubernetes control plane")
	componentStatusCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesNode, "node", "N", false, "Kubernetes node")

	getCmd.AddCommand(binaryVersionCmd)
	binaryVersionCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesControlPlane, "control-plane", "C", false, "Kubernetes control plane")
	binaryVersionCmd.Flags().BoolVarP(&run.ComponentStatusDisplay.KubernetesNode, "node", "N", false, "Kubernetes node")

	rootCmd.Execute()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
)

var (
	ReleaseId    = flag.Int("id", 0, "github/git-lfs Release ID")
	uploadUrlFmt = "https://uploads.github.com/repos/github/git-lfs/releases/%d/assets?%s"
)

func mainRelease() {
	if *ReleaseId < 1 {
		fmt.Println("Need a valid github/git-lfs release id.")
		fmt.Println("usage: script/release -id")
		os.Exit(1)
	}

	file, err := os.Open("bin/releases/build_matrix.json")
	if err != nil {
		fmt.Println("Error opening build_matrix.json:", err)
		fmt.Println("Ensure `script/bootstrap -all` has completed successfully")
		os.Exit(1)
	}

	defer file.Close()

	buildMatrix := make(map[string]Release)
	if err := json.NewDecoder(file).Decode(&buildMatrix); err != nil {
		fmt.Println("Error reading build_matrix.json:", err)
		os.Exit(1)
	}

	for _, rel := range buildMatrix {
		release(rel)
		fmt.Println()
	}
}

func release(rel Release) {
	query := url.Values{}
	query.Add("name", rel.Filename)
	query.Add("label", rel.Label)

	args := []string{
		"-in",
		"-H", "Content-Type: application/octet-stream",
		"-X", "POST",
		"--data-binary", "@bin/releases/" + rel.Filename,
		fmt.Sprintf(uploadUrlFmt, *ReleaseId, query.Encode()),
	}

	fmt.Println("curl", strings.Join(args, " "))

	cmd := exec.Command("curl", args...)

	by, err := cmd.Output()
	if err != nil {
		fmt.Println("Error running curl:", err)
		os.Exit(1)
	}

	fmt.Println(string(by))
}
<commit_msg>tweak release script temporarily<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"strings"
)

var (
	ReleaseId    = flag.Int("id", 0, "github/git-lfs Release ID")
	uploadUrlFmt = "https://uploads.github.com/repos/github/git-lfs-interim/releases/%d/assets?%s"
)

func mainRelease() {
	if *ReleaseId < 1 {
		fmt.Println("Need a valid github/git-lfs release id.")
		fmt.Println("usage: script/release -id")
		os.Exit(1)
	}

	file, err := os.Open("bin/releases/build_matrix.json")
	if err != nil {
		fmt.Println("Error opening build_matrix.json:", err)
		fmt.Println("Ensure `script/bootstrap -all` has completed successfully")
		os.Exit(1)
	}

	defer file.Close()

	buildMatrix := make(map[string]Release)
	if err := json.NewDecoder(file).Decode(&buildMatrix); err != nil {
		fmt.Println("Error reading build_matrix.json:", err)
		os.Exit(1)
	}

	for _, rel := range buildMatrix {
		release(rel)
		fmt.Println()
	}
}

func release(rel Release) {
	query := url.Values{}
	query.Add("name", rel.Filename)
	query.Add("label", rel.Label)

	args := []string{
		"-in",
		"-H", "Content-Type: application/octet-stream",
		"-X", "POST",
		"--data-binary", "@bin/releases/" + rel.Filename,
		fmt.Sprintf(uploadUrlFmt, *ReleaseId, query.Encode()),
	}

	fmt.Println("curl", strings.Join(args, " "))

	cmd := exec.Command("curl", args...)

	by, err := cmd.Output()
	if err != nil {
		fmt.Println("Error running curl:", err)
		os.Exit(1)
	}

	fmt.Println(string(by))
}
<|endoftext|>