<commit_before>package main

import (
	"database/sql"
	"strings"

	"sync"

	"errors"

	"github.com/piotrkowalczuk/charon"
)

// Permission returns Permission value that is concatenated
// using entity properties like subsystem, module and action.
func (pe *permissionEntity) Permission() charon.Permission {
	return charon.Permission(pe.Subsystem + ":" + pe.Module + ":" + pe.Action)
}

// PermissionRepository ...
type PermissionRepository interface {
	Find(criteria *permissionCriteria) ([]*permissionEntity, error)
	FindOneByID(id int64) (entity *permissionEntity, err error)
	// FindByUserID retrieves all permissions for user represented by given id.
	FindByUserID(userID int64) (entities []*permissionEntity, err error)
	// FindByGroupID retrieves all permissions for group represented by given id.
	FindByGroupID(groupID int64) (entities []*permissionEntity, err error)
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
	Insert(entity *permissionEntity) (*permissionEntity, error)
}

func newPermissionRepository(dbPool *sql.DB) *permissionRepository {
	return &permissionRepository{
		db:      dbPool,
		table:   tablePermission,
		columns: tablePermissionColumns,
	}
}

// FindByUserID implements PermissionRepository interface.
func (pr *permissionRepository) FindByUserID(userID int64) ([]*permissionEntity, error) {
	// TODO: does it work?
	return pr.findBy(`
		SELECT DISTINCT ON (p.id)
			`+columns(tablePermissionColumns, "p")+`
		FROM `+pr.table+` AS p
		LEFT JOIN `+tableUserPermissions+` AS up
			ON up.`+tableUserPermissionsColumnPermissionSubsystem+` = p.`+tablePermissionColumnSubsystem+`
			ON up.`+tableUserPermissionsColumnPermissionModule+` = p.`+tablePermissionColumnModule+`
			ON up.`+tableUserPermissionsColumnPermissionAction+` = p.`+tablePermissionColumnAction+`
			AND up.`+tableUserPermissionsColumnUserID+` = $1
		LEFT JOIN `+tableUserGroups+` AS ug ON ug.`+tableUserGroupsColumnUserID+` = $1
		LEFT JOIN `+tableGroupPermissions+` AS gp
			ON gp.`+tableGroupPermissionsColumnPermissionSubsystem+` = p.`+tablePermissionColumnSubsystem+`
			ON gp.`+tableGroupPermissionsColumnPermissionModule+` = p.`+tablePermissionColumnModule+`
			ON gp.`+tableGroupPermissionsColumnPermissionAction+` = p.`+tablePermissionColumnAction+`
			AND gp.`+tableGroupPermissionsColumnGroupID+` = ug.`+tableUserGroupsColumnGroupID+`
		WHERE up.`+tableUserPermissionsColumnUserID+` = $1 OR ug.`+tableUserGroupsColumnUserID+` = $1
	`, userID)
}

// FindByGroupID implements PermissionRepository interface.
func (pr *permissionRepository) FindByGroupID(userID int64) ([]*permissionEntity, error) {
	// TODO: does it work?
	return pr.findBy(`
		SELECT DISTINCT ON (p.id)
			`+columns(tablePermissionColumns, "p")+`
		FROM `+pr.table+` AS p
		LEFT JOIN `+tableGroupPermissions+` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`, userID)
}

func (pr *permissionRepository) findBy(query string, args ...interface{}) ([]*permissionEntity, error) {
	rows, err := pr.db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}

func (pr *permissionRepository) findOneStmt() (*sql.Stmt, error) {
	return pr.db.Prepare(
		"SELECT " + strings.Join(tablePermissionColumns, ",") + " " +
			"FROM " + pr.table + " AS p " +
			"WHERE p.subsystem = $1 AND p.module = $2 AND p.action = $3",
	)
}

// Register implements PermissionRepository interface.
func (pr *permissionRepository) Register(permissions charon.Permissions) (created, unt, removed int64, err error) {
	var (
		tx             *sql.Tx
		insert, delete *sql.Stmt
		rows           *sql.Rows
		res            sql.Result
		subsystem      string
		entities       []*permissionEntity
		affected       int64
	)
	if len(permissions) == 0 {
		return 0, 0, 0, errors.New("charond: empty slice, permissions cannot be registered")
	}

	subsystem = permissions[0].Subsystem()
	if subsystem == "" {
		return 0, 0, 0, errors.New("charond: subsystem name is empty string, permissions cannot be registered")
	}

	for _, p := range permissions {
		if p.Subsystem() != subsystem {
			return 0, 0, 0, errors.New("charond: provided permissions do not belong to one subsystem, permissions cannot be registered")
		}
	}

	tx, err = pr.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
			unt = untouched(int64(len(permissions)), created, removed)
		}
	}()

	rows, err = tx.Query("SELECT "+strings.Join(tablePermissionColumns, ",")+" FROM "+pr.table+" AS p WHERE p.subsystem = $1", subsystem)
	if err != nil {
		return
	}
	defer rows.Close()

	entities = []*permissionEntity{}
	for rows.Next() {
		var entity permissionEntity
		err = rows.Scan(
			&entity.Action,
			&entity.CreatedAt,
			&entity.ID,
			&entity.Module,
			&entity.Subsystem,
			&entity.UpdatedAt,
		)
		if err != nil {
			return
		}
		entities = append(entities, &entity)
	}
	if rows.Err() != nil {
		return 0, 0, 0, rows.Err()
	}

	insert, err = tx.Prepare("INSERT INTO " + pr.table + " (subsystem, module, action) VALUES ($1, $2, $3)")
	if err != nil {
		return
	}

MissingPermissionsLoop:
	for _, p := range permissions {
		for _, e := range entities {
			if p == e.Permission() {
				continue MissingPermissionsLoop
			}
		}

		if res, err = insert.Exec(p.Split()); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}
		created += affected
	}

	delete, err = tx.Prepare("DELETE FROM " + pr.table + " AS p WHERE p.id = $1")
	if err != nil {
		return
	}

RedundantPermissionsLoop:
	for _, e := range entities {
		for _, p := range permissions {
			if e.Permission() == p {
				continue RedundantPermissionsLoop
			}
		}

		if res, err = delete.Exec(e.ID); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}

		removed += affected
	}

	return
}

// PermissionRegistry is an interface that describes in memory storage that holds information
// about permissions that was registered by 3rd party services.
// Should be only used as a proxy for registration process to avoid multiple sql hits.
type PermissionRegistry interface {
	// Exists returns true if given Permission was already registered.
	Exists(permission charon.Permission) (exists bool)
	// Register checks if given collection is valid and
	// calls PermissionRepository to store provided permissions
	// in persistent way.
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
}

type permissionRegistry struct {
	sync.RWMutex
	repository  PermissionRepository
	permissions map[charon.Permission]struct{}
}

func newPermissionRegistry(r PermissionRepository) PermissionRegistry {
	return &permissionRegistry{
		repository:  r,
		permissions: make(map[charon.Permission]struct{}),
	}
}

// Exists implements PermissionRegistry interface.
func (pr *permissionRegistry) Exists(permission charon.Permission) (ok bool) {
	pr.RLock()
	pr.RUnlock()

	_, ok = pr.permissions[permission]
	return
}

// Register implements PermissionRegistry interface.
func (pr *permissionRegistry) Register(permissions charon.Permissions) (created, untouched, removed int64, err error) {
	pr.Lock()
	defer pr.Unlock()

	nb := 0
	for _, p := range permissions {
		if _, ok := pr.permissions[p]; !ok {
			pr.permissions[p] = struct{}{}
			nb++
		}
	}

	if nb > 0 {
		return pr.repository.Register(permissions)
	}

	return 0, 0, 0, nil
}

// FindByTag implements PermissionRepository interface.
func (pr *permissionRepository) FindByTag(userID int64) ([]*permissionEntity, error) {
	query := `
		SELECT DISTINCT ON (p.id)
			` + columns(tablePermissionColumns, "p") + `
		FROM ` + pr.table + ` AS p
		LEFT JOIN ` + tableUserPermissions + ` AS up ON up.permission_id = p.id AND up.user_id = $1
		LEFT JOIN ` + tableUserGroups + ` AS ug ON ug.user_id = $1
		LEFT JOIN ` + tableGroupPermissions + ` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`

	rows, err := pr.db.Query(query, userID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}
<commit_msg>PermissionRepository.FindByUserID fix<commit_after>package main

import (
	"database/sql"
	"strings"

	"sync"

	"errors"

	"github.com/piotrkowalczuk/charon"
)

// Permission returns Permission value that is concatenated
// using entity properties like subsystem, module and action.
func (pe *permissionEntity) Permission() charon.Permission {
	return charon.Permission(pe.Subsystem + ":" + pe.Module + ":" + pe.Action)
}

// PermissionRepository ...
type PermissionRepository interface {
	Find(criteria *permissionCriteria) ([]*permissionEntity, error)
	FindOneByID(id int64) (entity *permissionEntity, err error)
	// FindByUserID retrieves all permissions for user represented by given id.
	FindByUserID(userID int64) (entities []*permissionEntity, err error)
	// FindByGroupID retrieves all permissions for group represented by given id.
	FindByGroupID(groupID int64) (entities []*permissionEntity, err error)
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
	Insert(entity *permissionEntity) (*permissionEntity, error)
}

func newPermissionRepository(dbPool *sql.DB) *permissionRepository {
	return &permissionRepository{
		db:      dbPool,
		table:   tablePermission,
		columns: tablePermissionColumns,
	}
}

// FindByUserID implements PermissionRepository interface.
func (pr *permissionRepository) FindByUserID(userID int64) ([]*permissionEntity, error) {
	// TODO: does it work?
	return pr.findBy(`
		SELECT DISTINCT ON (p.id)
			`+columns(tablePermissionColumns, "p")+`
		FROM `+pr.table+` AS p
		LEFT JOIN `+tableUserPermissions+` AS up
			ON up.`+tableUserPermissionsColumnPermissionSubsystem+` = p.`+tablePermissionColumnSubsystem+`
			AND up.`+tableUserPermissionsColumnPermissionModule+` = p.`+tablePermissionColumnModule+`
			AND up.`+tableUserPermissionsColumnPermissionAction+` = p.`+tablePermissionColumnAction+`
		LEFT JOIN `+tableUserGroups+` AS ug ON ug.`+tableUserGroupsColumnUserID+` = $1
		LEFT JOIN `+tableGroupPermissions+` AS gp
			ON gp.`+tableGroupPermissionsColumnPermissionSubsystem+` = p.`+tablePermissionColumnSubsystem+`
			AND gp.`+tableGroupPermissionsColumnPermissionModule+` = p.`+tablePermissionColumnModule+`
			AND gp.`+tableGroupPermissionsColumnPermissionAction+` = p.`+tablePermissionColumnAction+`
			AND gp.`+tableGroupPermissionsColumnGroupID+` = ug.`+tableUserGroupsColumnGroupID+`
		WHERE up.`+tableUserPermissionsColumnUserID+` = $1 OR ug.`+tableUserGroupsColumnUserID+` = $1
	`, userID)
}

// FindByGroupID implements PermissionRepository interface.
func (pr *permissionRepository) FindByGroupID(userID int64) ([]*permissionEntity, error) {
	// TODO: does it work?
	return pr.findBy(`
		SELECT DISTINCT ON (p.id)
			`+columns(tablePermissionColumns, "p")+`
		FROM `+pr.table+` AS p
		LEFT JOIN `+tableGroupPermissions+` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`, userID)
}

func (pr *permissionRepository) findBy(query string, args ...interface{}) ([]*permissionEntity, error) {
	rows, err := pr.db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}

func (pr *permissionRepository) findOneStmt() (*sql.Stmt, error) {
	return pr.db.Prepare(
		"SELECT " + strings.Join(tablePermissionColumns, ",") + " " +
			"FROM " + pr.table + " AS p " +
			"WHERE p.subsystem = $1 AND p.module = $2 AND p.action = $3",
	)
}

// Register implements PermissionRepository interface.
func (pr *permissionRepository) Register(permissions charon.Permissions) (created, unt, removed int64, err error) {
	var (
		tx             *sql.Tx
		insert, delete *sql.Stmt
		rows           *sql.Rows
		res            sql.Result
		subsystem      string
		entities       []*permissionEntity
		affected       int64
	)
	if len(permissions) == 0 {
		return 0, 0, 0, errors.New("charond: empty slice, permissions cannot be registered")
	}

	subsystem = permissions[0].Subsystem()
	if subsystem == "" {
		return 0, 0, 0, errors.New("charond: subsystem name is empty string, permissions cannot be registered")
	}

	for _, p := range permissions {
		if p.Subsystem() != subsystem {
			return 0, 0, 0, errors.New("charond: provided permissions do not belong to one subsystem, permissions cannot be registered")
		}
	}

	tx, err = pr.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
			unt = untouched(int64(len(permissions)), created, removed)
		}
	}()

	rows, err = tx.Query("SELECT "+strings.Join(tablePermissionColumns, ",")+" FROM "+pr.table+" AS p WHERE p.subsystem = $1", subsystem)
	if err != nil {
		return
	}
	defer rows.Close()

	entities = []*permissionEntity{}
	for rows.Next() {
		var entity permissionEntity
		err = rows.Scan(
			&entity.Action,
			&entity.CreatedAt,
			&entity.ID,
			&entity.Module,
			&entity.Subsystem,
			&entity.UpdatedAt,
		)
		if err != nil {
			return
		}
		entities = append(entities, &entity)
	}
	if rows.Err() != nil {
		return 0, 0, 0, rows.Err()
	}

	insert, err = tx.Prepare("INSERT INTO " + pr.table + " (subsystem, module, action) VALUES ($1, $2, $3)")
	if err != nil {
		return
	}

MissingPermissionsLoop:
	for _, p := range permissions {
		for _, e := range entities {
			if p == e.Permission() {
				continue MissingPermissionsLoop
			}
		}

		if res, err = insert.Exec(p.Split()); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}
		created += affected
	}

	delete, err = tx.Prepare("DELETE FROM " + pr.table + " AS p WHERE p.id = $1")
	if err != nil {
		return
	}

RedundantPermissionsLoop:
	for _, e := range entities {
		for _, p := range permissions {
			if e.Permission() == p {
				continue RedundantPermissionsLoop
			}
		}

		if res, err = delete.Exec(e.ID); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}

		removed += affected
	}

	return
}

// PermissionRegistry is an interface that describes in memory storage that holds information
// about permissions that was registered by 3rd party services.
// Should be only used as a proxy for registration process to avoid multiple sql hits.
type PermissionRegistry interface {
	// Exists returns true if given Permission was already registered.
	Exists(permission charon.Permission) (exists bool)
	// Register checks if given collection is valid and
	// calls PermissionRepository to store provided permissions
	// in persistent way.
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
}

type permissionRegistry struct {
	sync.RWMutex
	repository  PermissionRepository
	permissions map[charon.Permission]struct{}
}

func newPermissionRegistry(r PermissionRepository) PermissionRegistry {
	return &permissionRegistry{
		repository:  r,
		permissions: make(map[charon.Permission]struct{}),
	}
}

// Exists implements PermissionRegistry interface.
func (pr *permissionRegistry) Exists(permission charon.Permission) (ok bool) {
	pr.RLock()
	pr.RUnlock()

	_, ok = pr.permissions[permission]
	return
}

// Register implements PermissionRegistry interface.
func (pr *permissionRegistry) Register(permissions charon.Permissions) (created, untouched, removed int64, err error) {
	pr.Lock()
	defer pr.Unlock()

	nb := 0
	for _, p := range permissions {
		if _, ok := pr.permissions[p]; !ok {
			pr.permissions[p] = struct{}{}
			nb++
		}
	}

	if nb > 0 {
		return pr.repository.Register(permissions)
	}

	return 0, 0, 0, nil
}

// FindByTag implements PermissionRepository interface.
func (pr *permissionRepository) FindByTag(userID int64) ([]*permissionEntity, error) {
	query := `
		SELECT DISTINCT ON (p.id)
			` + columns(tablePermissionColumns, "p") + `
		FROM ` + pr.table + ` AS p
		LEFT JOIN ` + tableUserPermissions + ` AS up ON up.permission_id = p.id AND up.user_id = $1
		LEFT JOIN ` + tableUserGroups + ` AS ug ON ug.user_id = $1
		LEFT JOIN ` + tableGroupPermissions + ` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`

	rows, err := pr.db.Query(query, userID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}
<|endoftext|>


<commit_before>package torrent

import (
	"encoding/hex"
	"github.com/nictuku/dht"
	"golang.org/x/net/proxy"
	"log"
	"os"
	"os/signal"
)

type TorrentFlags struct {
	Port                int
	FileDir             string
	SeedRatio           float64
	UseDeadlockDetector bool
	UseLPD              bool
	UseDHT              bool
	UseUPnP             bool
	UseNATPMP           bool
	TrackerlessMode     bool
	ExecOnSeeding       string

	// The dial function to use. Nil means use net.Dial
	Dial proxy.Dialer

	// IP address of gateway used for NAT-PMP
	Gateway string

	//Provides the filesystems added torrents are saved to
	FileSystemProvider FsProvider

	//Whether to check file hashes when adding torrents
	InitialCheck bool

	//Provides cache to each torrent
	Cacher CacheProvider

	//Whether to write and use *.haveBitset resume data
	QuickResume bool

	//How many torrents should be active at a time
	MaxActive int
}

func RunTorrents(flags *TorrentFlags, torrentFiles []string) (err error) {
	conChan, listenPort, err := ListenForPeerConnections(flags)
	if err != nil {
		log.Println("Couldn't listen for peers connection: ", err)
		return
	}
	quitChan := listenSigInt()

	createChan := make(chan string, flags.MaxActive)
	startChan := make(chan *TorrentSession, 1)
	doneChan := make(chan *TorrentSession, 1)

	var dhtNode dht.DHT
	if flags.UseDHT {
		dhtNode = *startDHT(flags.Port)
	}

	torrentSessions := make(map[string]*TorrentSession)

	go func() {
		for torrentFile := range createChan {
			ts, err := NewTorrentSession(flags, torrentFile, uint16(listenPort))
			if err != nil {
				log.Println("Could not create torrent session.", err)
				doneChan <- &TorrentSession{}
			} else {
				log.Printf("Created torrent session for %s", ts.M.Info.Name)
				startChan <- ts
			}
		}
	}()

	torrentQueue := []string{}
	if len(torrentFiles) > flags.MaxActive {
		torrentQueue = torrentFiles[flags.MaxActive:]
	}

	for i, torrentFile := range torrentFiles {
		if i < flags.MaxActive {
			createChan <- torrentFile
		} else {
			break
		}
	}

	lpd := &Announcer{}
	if flags.UseLPD {
		lpd, err = NewAnnouncer(uint16(listenPort))
		if err != nil {
			log.Println("Couldn't listen for Local Peer Discoveries: ", err)
			flags.UseLPD = false
		}
	}

	theWorldisEnding := false
mainLoop:
	for {
		select {
		case ts := <-startChan:
			if !theWorldisEnding {
				ts.dht = &dhtNode
				if flags.UseLPD {
					lpd.Announce(ts.M.InfoHash)
				}
				torrentSessions[ts.M.InfoHash] = ts
				log.Printf("Starting torrent session for %s", ts.M.Info.Name)
				go func(t *TorrentSession) {
					t.DoTorrent()
					doneChan <- t
				}(ts)
			}
		case ts := <-doneChan:
			delete(torrentSessions, ts.M.InfoHash)
			if flags.UseLPD {
				lpd.StopAnnouncing(ts.M.InfoHash)
			}
			if !theWorldisEnding && len(torrentQueue) > 0 {
				createChan <- torrentQueue[0]
				torrentQueue = torrentQueue[1:]
				continue mainLoop
			}

			if len(torrentSessions) == 0 {
				break mainLoop
			}
		case <-quitChan:
			theWorldisEnding = true
			for _, ts := range torrentSessions {
				go ts.Quit()
			}
		case c := <-conChan:
			log.Printf("New bt connection for ih %x", c.Infohash)
			if ts, ok := torrentSessions[c.Infohash]; ok {
				ts.AcceptNewPeer(c)
			}
		case dhtPeers := <-dhtNode.PeersRequestResults:
			for key, peers := range dhtPeers {
				if ts, ok := torrentSessions[string(key)]; ok {
					log.Printf("Received %d DHT peers for torrent session %x\n", len(peers), []byte(key))
					for _, peer := range peers {
						peer = dht.DecodePeerAddress(peer)
						ts.HintNewPeer(peer)
					}
				} else {
					log.Printf("Received DHT peer for an unknown torrent session %x\n", []byte(key))
				}
			}
		case announce := <-lpd.Announces:
			hexhash, err := hex.DecodeString(announce.Infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
			}
			if ts, ok := torrentSessions[string(hexhash)]; ok {
				log.Printf("Received LPD announce for ih %s", announce.Infohash)
				ts.HintNewPeer(announce.Peer)
			}
		}
	}
	if flags.UseDHT {
		dhtNode.Stop()
	}
	return
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func startDHT(listenPort int) *dht.DHT {
	// TODO: UPnP UDP port mapping.
	cfg := dht.NewConfig()
	cfg.Port = listenPort
	cfg.NumTargetPeers = TARGET_NUM_PEERS
	dhtnode, err := dht.New(cfg)
	if err != nil {
		log.Println("DHT node creation error:", err)
		return nil
	}

	go dhtnode.Run()

	return dhtnode
}
<commit_msg>torrentloop: AcceptPeer / HintNewPeer in routines avoids deadlocks #55<commit_after>package torrent

import (
	"encoding/hex"
	"github.com/nictuku/dht"
	"golang.org/x/net/proxy"
	"log"
	"os"
	"os/signal"
)

type TorrentFlags struct {
	Port                int
	FileDir             string
	SeedRatio           float64
	UseDeadlockDetector bool
	UseLPD              bool
	UseDHT              bool
	UseUPnP             bool
	UseNATPMP           bool
	TrackerlessMode     bool
	ExecOnSeeding       string

	// The dial function to use. Nil means use net.Dial
	Dial proxy.Dialer

	// IP address of gateway used for NAT-PMP
	Gateway string

	//Provides the filesystems added torrents are saved to
	FileSystemProvider FsProvider

	//Whether to check file hashes when adding torrents
	InitialCheck bool

	//Provides cache to each torrent
	Cacher CacheProvider

	//Whether to write and use *.haveBitset resume data
	QuickResume bool

	//How many torrents should be active at a time
	MaxActive int
}

func RunTorrents(flags *TorrentFlags, torrentFiles []string) (err error) {
	conChan, listenPort, err := ListenForPeerConnections(flags)
	if err != nil {
		log.Println("Couldn't listen for peers connection: ", err)
		return
	}
	quitChan := listenSigInt()

	createChan := make(chan string, flags.MaxActive)
	startChan := make(chan *TorrentSession, 1)
	doneChan := make(chan *TorrentSession, 1)

	var dhtNode dht.DHT
	if flags.UseDHT {
		dhtNode = *startDHT(flags.Port)
	}

	torrentSessions := make(map[string]*TorrentSession)

	go func() {
		for torrentFile := range createChan {
			ts, err := NewTorrentSession(flags, torrentFile, uint16(listenPort))
			if err != nil {
				log.Println("Could not create torrent session.", err)
				doneChan <- &TorrentSession{}
			} else {
				log.Printf("Created torrent session for %s", ts.M.Info.Name)
				startChan <- ts
			}
		}
	}()

	torrentQueue := []string{}
	if len(torrentFiles) > flags.MaxActive {
		torrentQueue = torrentFiles[flags.MaxActive:]
	}

	for i, torrentFile := range torrentFiles {
		if i < flags.MaxActive {
			createChan <- torrentFile
		} else {
			break
		}
	}

	lpd := &Announcer{}
	if flags.UseLPD {
		lpd, err = NewAnnouncer(uint16(listenPort))
		if err != nil {
			log.Println("Couldn't listen for Local Peer Discoveries: ", err)
			flags.UseLPD = false
		}
	}

	theWorldisEnding := false
mainLoop:
	for {
		select {
		case ts := <-startChan:
			if !theWorldisEnding {
				ts.dht = &dhtNode
				if flags.UseLPD {
					lpd.Announce(ts.M.InfoHash)
				}
				torrentSessions[ts.M.InfoHash] = ts
				log.Printf("Starting torrent session for %s", ts.M.Info.Name)
				go func(t *TorrentSession) {
					t.DoTorrent()
					doneChan <- t
				}(ts)
			}
		case ts := <-doneChan:
			delete(torrentSessions, ts.M.InfoHash)
			if flags.UseLPD {
				lpd.StopAnnouncing(ts.M.InfoHash)
			}
			if !theWorldisEnding && len(torrentQueue) > 0 {
				createChan <- torrentQueue[0]
				torrentQueue = torrentQueue[1:]
				continue mainLoop
			}

			if len(torrentSessions) == 0 {
				break mainLoop
			}
		case <-quitChan:
			theWorldisEnding = true
			for _, ts := range torrentSessions {
				go ts.Quit()
			}
		case c := <-conChan:
			log.Printf("New bt connection for ih %x", c.Infohash)
			if ts, ok := torrentSessions[c.Infohash]; ok {
				go ts.AcceptNewPeer(c)
			}
		case dhtPeers := <-dhtNode.PeersRequestResults:
			for key, peers := range dhtPeers {
				if ts, ok := torrentSessions[string(key)]; ok {
					log.Printf("Received %d DHT peers for torrent session %x\n", len(peers), []byte(key))
					for _, peer := range peers {
						peer = dht.DecodePeerAddress(peer)
						go ts.HintNewPeer(peer)
					}
				} else {
					log.Printf("Received DHT peer for an unknown torrent session %x\n", []byte(key))
				}
			}
		case announce := <-lpd.Announces:
			hexhash, err := hex.DecodeString(announce.Infohash)
			if err != nil {
				log.Println("Err with hex-decoding:", err)
			}
			if ts, ok := torrentSessions[string(hexhash)]; ok {
				log.Printf("Received LPD announce for ih %s", announce.Infohash)
				go ts.HintNewPeer(announce.Peer)
			}
		}
	}
	if flags.UseDHT {
		dhtNode.Stop()
	}
	return
}

func listenSigInt() chan os.Signal {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	return c
}

func startDHT(listenPort int) *dht.DHT {
	// TODO: UPnP UDP port mapping.
	cfg := dht.NewConfig()
	cfg.Port = listenPort
	cfg.NumTargetPeers = TARGET_NUM_PEERS
	dhtnode, err := dht.New(cfg)
	if err != nil {
		log.Println("DHT node creation error:", err)
		return nil
	}

	go dhtnode.Run()

	return dhtnode
}
<|endoftext|>


<commit_before>package extra

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
}

func New() Env {
	return Env{cmds: map[string]typed.Command{
		"exec": execCommand,
		"cd":   cdCommand,
		"exit": exitCommand,
	}}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) error {
	tc, found := e.cmds[command.Name]
	if !found {
		return fmt.Errorf("no such typed command: %q", command.Name)
	}
	if len(command.Args) != len(tc.Params) {
		return fmt.Errorf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params))
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return fmt.Errorf("type mismatch: %v != %v", arg.Type(), tc.Params[i])
		}
	}
	return tc.Fn(command.Args)
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr) error {
		var cmdArgs []string
	loop:
		for cons := args[1].(*ast.Cons); ; {
			cmdArgs = append(cmdArgs, cons.Head)
			switch x := cons.Tail.(type) {
			case *ast.Empty:
				break loop
			case *ast.Cons:
				cons = x
			default:
				panic(fmt.Sprintf("exec: unexpected list type: %T", x))
			}
		}
		return exec.Command(args[0].(*ast.String).Lit, cmdArgs...).Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}
<commit_msg>Extract toSilce<commit_after>package extra

import (
	"fmt"
	"os"
	"os/exec"
	"strconv"

	"github.com/pkg/errors"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
}

func New() Env {
	return Env{cmds: map[string]typed.Command{
		"exec": execCommand,
		"cd":   cdCommand,
		"exit": exitCommand,
	}}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) error {
	tc, found := e.cmds[command.Name]
	if !found {
		return fmt.Errorf("no such typed command: %q", command.Name)
	}
	if len(command.Args) != len(tc.Params) {
		return fmt.Errorf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params))
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return fmt.Errorf("type mismatch: %v != %v", arg.Type(), tc.Params[i])
		}
	}
	return tc.Fn(command.Args)
}

func toSilce(cons *ast.Cons) ([]string, error) {
	var ret []string
	for {
		ret = append(ret, cons.Head)
		switch x := cons.Tail.(type) {
		case *ast.Cons:
			cons = x
		case *ast.Empty:
			return ret, nil
		default:
			return nil, fmt.Errorf("unexpected list type: %T", x)
		}
	}
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr) error {
		cmdArgs, err := toSilce(args[1].(*ast.Cons))
		if err != nil {
			return errors.Wrap(err, "exec")
		}
		return exec.Command(args[0].(*ast.String).Lit, cmdArgs...).Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}
<|endoftext|>


<commit_before>// This is a simple package for Golang that provides a very
// rudimentary implementation of a Binary Tree, provides insertion,
// deletion, and a search. To try to support some custom types instead
// of just limiting the usage to ints or strings, I've addded an
// interface that asks you to implement three comparison functions to
// enable the tree comparison steps. This could probably be done
// better but I'm just trying to get back into the swing of my
// favourite data structures.ma
package BinaryTree

type BTreeCompare interface {
	LessThan(other BTreeCompare) bool
	GreaterThan(other BTreeCompare) bool
	EqualTo(other BTreeCompare) bool
}

type BinaryTree struct {
	value BTreeCompare // value of this node
	left  *BinaryTree  // next smaller value
	right *BinaryTree  // next larger value
}

func New(value BTreeCompare) *BinaryTree {
	t := new(BinaryTree)
	t.value = value
	return t
}

func (root *BinaryTree) Contains(value BTreeCompare) bool {
	switch {
	case root.value == value:
		return true
	case value.LessThan(root.value) && root.left != nil:
		return root.left.Contains(value)
	case value.GreaterThan(root.value) && root.right != nil:
		return root.right.Contains(value)
	}
	return false
}

func (root *BinaryTree) Delete(value BTreeCompare) {
	if root.Contains(value) {
		delete(value, root, nil)
	}
}

func delete(value BTreeCompare, current, previous *BinaryTree) {
	if value.EqualTo(current.value) {
		switch {
		case previous == nil && current.left == nil && current.right == nil:
			// Errmmmmmmm....
			current = new(BinaryTree)
		case current.left == nil && current.right == nil:
			deleteHelper(value, previous, nil)

		case current.left != nil:
			tmp := current.left
			tmp.right = current.right
			deleteHelper(tmp.value, previous, tmp)

		case current.right != nil:
			tmp := current.right
			deleteHelper(tmp.value, previous, tmp)
		}
	} else {
		switch {
		case value.LessThan(current.value) && current.left != nil:
			delete(value, current.left, current)
		case value.GreaterThan(current.value) && current.right != nil:
			delete(value, current.right, current)
		}
	}
}

func deleteHelper(val BTreeCompare, target, forTarget *BinaryTree) {
	if val.GreaterThan(target.value) {
		target.right = forTarget
	} else {
		target.left = forTarget
	}
}

func (root *BinaryTree) Insert(value BTreeCompare) {
	switch {
	case value.GreaterThan(root.value) && root.right == nil:
		n := new(BinaryTree)
		n.value = value
		root.right = n
	case value.LessThan(root.value) && root.left == nil:
		n := new(BinaryTree)
		n.value = value
		root.left = n
	case value.GreaterThan(root.value):
		root.right.Insert(value)
	case value.LessThan(root.value):
		root.left.Insert(value)
	}
}
<commit_msg>Switcheroo<commit_after>// This is a simple package for Golang that provides a very
// rudimentary implementation of a Binary Tree, provides insertion,
// deletion, and a search. To try to support some custom types instead
// of just limiting the usage to ints or strings, I've addded an
// interface that asks you to implement three comparison functions to
// enable the tree comparison steps. This could probably be done
// better but I'm just trying to get back into the swing of my
// favourite data structures.ma
package BinaryTree

type BTreeCompare interface {
	LessThan(other BTreeCompare) bool
	GreaterThan(other BTreeCompare) bool
	EqualTo(other BTreeCompare) bool
}

type BinaryTree struct {
	value BTreeCompare // value of this node
	left  *BinaryTree  // next smaller value
	right *BinaryTree  // next larger value
}

func New(value BTreeCompare) *BinaryTree {
	t := new(BinaryTree)
	t.value = value
	return t
}

func (root *BinaryTree) Contains(value BTreeCompare) bool {
	switch {
	case root.value == value:
		return true
	case value.LessThan(root.value) && root.left != nil:
		return root.left.Contains(value)
	case value.GreaterThan(root.value) && root.right != nil:
		return root.right.Contains(value)
	}
	return false
}

func (root *BinaryTree) Delete(value BTreeCompare) {
	if root.Contains(value) {
		delete(value, root, nil)
	}
}

func (root *BinaryTree) Insert(value BTreeCompare) {
	switch {
	case value.GreaterThan(root.value) && root.right == nil:
		n := new(BinaryTree)
		n.value = value
		root.right = n
	case value.LessThan(root.value) && root.left == nil:
		n := new(BinaryTree)
		n.value = value
		root.left = n
	case value.GreaterThan(root.value):
		root.right.Insert(value)
	case value.LessThan(root.value):
		root.left.Insert(value)
	}
}

func delete(value BTreeCompare, current, previous *BinaryTree) {
	if value.EqualTo(current.value) {
		switch {
		case previous == nil && current.left == nil && current.right == nil:
			// Errmmmmmmm....
			current = new(BinaryTree)
		case current.left == nil && current.right == nil:
			deleteHelper(value, previous, nil)

		case current.left != nil:
			tmp := current.left
			tmp.right = current.right
			deleteHelper(tmp.value, previous, tmp)

		case current.right != nil:
			tmp := current.right
			deleteHelper(tmp.value, previous, tmp)
		}
	} else {
		switch {
		case value.LessThan(current.value) && current.left != nil:
			delete(value, current.left, current)
		case value.GreaterThan(current.value) && current.right != nil:
			delete(value, current.right, current)
		}
	}
}

func deleteHelper(val BTreeCompare, target, forTarget *BinaryTree) {
	if val.GreaterThan(target.value) {
		target.right = forTarget
	} else {
		target.left = forTarget
	}
}<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcws

import (
	"encoding/json"
	"errors"
	"github.com/conformal/btcjson"
)

var (
	// ErrNtfnUnexpected describes an error where an unexpected
	// notification is received when unmarshaling into a concrete
	// Notification variable.
	ErrNtfnUnexpected = errors.New("notification unexpected")

	// ErrNtfnNotFound describes an error where a parser does not
	// handle unmarshaling a notification.
	ErrNtfnNotFound = errors.New("notification not found")
)

const (
	// BlockConnectedNtfnId is the id of the btcd blockconnected
	// notification.
	BlockConnectedNtfnId = "btcd:blockconnected"

	// BlockDisconnectedNtfnId is the id of the btcd blockdisconnected
	// notification.
	BlockDisconnectedNtfnId = "btcd:blockdisconnected"

	// TxMinedNtfnId is the id of the btcd txmined notification.
	TxMinedNtfnId = "btcd:txmined"

	// TxNtfnId is the id of the btcwallet newtx notification.
	TxNtfnId = "btcwallet:newtx"
)

type newNtfnFn func() Notification

func newBlockConnectedNtfn() Notification {
	return &BlockConnectedNtfn{}
}

func newBlockDisconnectedNtfn() Notification {
	return &BlockDisconnectedNtfn{}
}

func newTxMinedNtfn() Notification {
	return &TxMinedNtfn{}
}

func newTxNtfn() Notification {
	return &TxNtfn{}
}

var newNtfnFns = map[string]newNtfnFn{
	BlockConnectedNtfnId:    newBlockConnectedNtfn,
	BlockDisconnectedNtfnId: newBlockDisconnectedNtfn,
	TxMinedNtfnId:           newTxMinedNtfn,
	TxNtfnId:                newTxNtfn,
}

// ParseMarshaledNtfn attempts to unmarshal a marshaled notification
// into the notification described by id.
func ParseMarshaledNtfn(id string, b []byte) (Notification, error) {
	if newFn, ok := newNtfnFns[id]; ok {
		n := newFn()
		if err := n.UnmarshalJSON(b); err != nil {
			return nil, err
		}
		return n, nil
	}
	return nil, ErrNtfnNotFound
}

// Notification is an interface implemented by all notification types.
type Notification interface {
	json.Marshaler
	json.Unmarshaler
	Id() interface{}
}

// BlockConnectedNtfn is a type handling custom marshaling and
// unmarshaling of blockconnected JSON websocket notifications.
type BlockConnectedNtfn struct {
	Hash   string `json:"hash"`
	Height int32  `json:"height"`
}

type blockConnectedResult BlockConnectedNtfn

// Enforce that BlockConnectedNtfn satisfies the Notification interface.
var _ Notification = &BlockConnectedNtfn{}

// NewBlockConnectedNtfn creates a new BlockConnectedNtfn.
func NewBlockConnectedNtfn(hash string, height int32) *BlockConnectedNtfn {
	return &BlockConnectedNtfn{
		Hash:   hash,
		Height: height,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *BlockConnectedNtfn) Id() interface{} {
	return BlockConnectedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *BlockConnectedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *BlockConnectedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result blockConnectedResult `json:"result"`
		Error  *btcjson.Error       `json:"error"`
		Id     interface{}          `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = BlockConnectedNtfn(ntfn.Result)
	return nil
}

// BlockDisconnectedNtfn is a type handling custom marshaling and
// unmarshaling of blockdisconnected JSON websocket notifications.
type BlockDisconnectedNtfn struct {
	Hash   string `json:"hash"`
	Height int32  `json:"height"`
}

type blockDisconnectedResult BlockDisconnectedNtfn

// Enforce that BlockDisconnectedNtfn satisfies the Notification interface.
var _ Notification = &BlockDisconnectedNtfn{}

// NewBlockDisconnectedNtfn creates a new BlockConnectedNtfn.
func NewBlockDisconnectedNtfn(hash string, height int32) *BlockDisconnectedNtfn {
	return &BlockDisconnectedNtfn{
		Hash:   hash,
		Height: height,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *BlockDisconnectedNtfn) Id() interface{} {
	return BlockDisconnectedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *BlockDisconnectedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *BlockDisconnectedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result blockDisconnectedResult `json:"result"`
		Error  *btcjson.Error          `json:"error"`
		Id     interface{}             `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = BlockDisconnectedNtfn(ntfn.Result)
	return nil
}

// TxMinedNtfn is a type handling custom marshaling and
// unmarshaling of txmined JSON websocket notifications.
type TxMinedNtfn struct {
	TxID        string `json:"txid"`
	BlockHash   string `json:"blockhash"`
	BlockHeight int32  `json:"blockheight"`
	BlockTime   int64  `json:"blocktime"`
	Index       int    `json:"index"`
}

type txMinedResult TxMinedNtfn

// Enforce that TxMinedNtfn satisfies the Notification interface.
var _ Notification = &TxMinedNtfn{}

// NewTxMinedNtfn creates a new TxMinedNtfn.
func NewTxMinedNtfn(txid, blockhash string, blockheight int32,
	blocktime int64, index int) *TxMinedNtfn {

	return &TxMinedNtfn{
		TxID:        txid,
		BlockHash:   blockhash,
		BlockHeight: blockheight,
		BlockTime:   blocktime,
		Index:       index,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *TxMinedNtfn) Id() interface{} {
	return TxMinedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *TxMinedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *TxMinedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result txMinedResult  `json:"result"`
		Error  *btcjson.Error `json:"error"`
		Id     interface{}    `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = TxMinedNtfn(ntfn.Result)
	return nil
}

// TxNtfn is a type handling custom marshaling and
// unmarshaling of newtx JSON websocket notifications.
type TxNtfn struct {
	Account string `json:"account"`
	Details map[string]interface{} `json:"details"`
}

type txNtfnResult TxNtfn

// Enforce that TxNtfn satisfies the Notification interface.
var _ Notification = &TxNtfn{}

// NewTxNtfn creates a new TxNtfn.
func NewTxNtfn(account string, details map[string]interface{}) *TxNtfn {
	return &TxNtfn{
		Account: account,
		Details: details,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *TxNtfn) Id() interface{} {
	return TxNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *TxNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *TxNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result txNtfnResult   `json:"result"`
		Error  *btcjson.Error `json:"error"`
		Id     interface{}    `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = TxNtfn(ntfn.Result)
	return nil
}
<commit_msg>go fmt<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcws

import (
	"encoding/json"
	"errors"
	"github.com/conformal/btcjson"
)

var (
	// ErrNtfnUnexpected describes an error where an unexpected
	// notification is received when unmarshaling into a concrete
	// Notification variable.
	ErrNtfnUnexpected = errors.New("notification unexpected")

	// ErrNtfnNotFound describes an error where a parser does not
	// handle unmarshaling a notification.
	ErrNtfnNotFound = errors.New("notification not found")
)

const (
	// BlockConnectedNtfnId is the id of the btcd blockconnected
	// notification.
	BlockConnectedNtfnId = "btcd:blockconnected"

	// BlockDisconnectedNtfnId is the id of the btcd blockdisconnected
	// notification.
	BlockDisconnectedNtfnId = "btcd:blockdisconnected"

	// TxMinedNtfnId is the id of the btcd txmined notification.
	TxMinedNtfnId = "btcd:txmined"

	// TxNtfnId is the id of the btcwallet newtx notification.
	TxNtfnId = "btcwallet:newtx"
)

type newNtfnFn func() Notification

func newBlockConnectedNtfn() Notification {
	return &BlockConnectedNtfn{}
}

func newBlockDisconnectedNtfn() Notification {
	return &BlockDisconnectedNtfn{}
}

func newTxMinedNtfn() Notification {
	return &TxMinedNtfn{}
}

func newTxNtfn() Notification {
	return &TxNtfn{}
}

var newNtfnFns = map[string]newNtfnFn{
	BlockConnectedNtfnId:    newBlockConnectedNtfn,
	BlockDisconnectedNtfnId: newBlockDisconnectedNtfn,
	TxMinedNtfnId:           newTxMinedNtfn,
	TxNtfnId:                newTxNtfn,
}

// ParseMarshaledNtfn attempts to unmarshal a marshaled notification
// into the notification described by id.
func ParseMarshaledNtfn(id string, b []byte) (Notification, error) {
	if newFn, ok := newNtfnFns[id]; ok {
		n := newFn()
		if err := n.UnmarshalJSON(b); err != nil {
			return nil, err
		}
		return n, nil
	}
	return nil, ErrNtfnNotFound
}

// Notification is an interface implemented by all notification types.
type Notification interface {
	json.Marshaler
	json.Unmarshaler
	Id() interface{}
}

// BlockConnectedNtfn is a type handling custom marshaling and
// unmarshaling of blockconnected JSON websocket notifications.
type BlockConnectedNtfn struct {
	Hash   string `json:"hash"`
	Height int32  `json:"height"`
}

type blockConnectedResult BlockConnectedNtfn

// Enforce that BlockConnectedNtfn satisfies the Notification interface.
var _ Notification = &BlockConnectedNtfn{}

// NewBlockConnectedNtfn creates a new BlockConnectedNtfn.
func NewBlockConnectedNtfn(hash string, height int32) *BlockConnectedNtfn {
	return &BlockConnectedNtfn{
		Hash:   hash,
		Height: height,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *BlockConnectedNtfn) Id() interface{} {
	return BlockConnectedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *BlockConnectedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *BlockConnectedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result blockConnectedResult `json:"result"`
		Error  *btcjson.Error       `json:"error"`
		Id     interface{}          `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = BlockConnectedNtfn(ntfn.Result)
	return nil
}

// BlockDisconnectedNtfn is a type handling custom marshaling and
// unmarshaling of blockdisconnected JSON websocket notifications.
type BlockDisconnectedNtfn struct {
	Hash   string `json:"hash"`
	Height int32  `json:"height"`
}

type blockDisconnectedResult BlockDisconnectedNtfn

// Enforce that BlockDisconnectedNtfn satisfies the Notification interface.
var _ Notification = &BlockDisconnectedNtfn{}

// NewBlockDisconnectedNtfn creates a new BlockConnectedNtfn.
func NewBlockDisconnectedNtfn(hash string, height int32) *BlockDisconnectedNtfn {
	return &BlockDisconnectedNtfn{
		Hash:   hash,
		Height: height,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *BlockDisconnectedNtfn) Id() interface{} {
	return BlockDisconnectedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *BlockDisconnectedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *BlockDisconnectedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result blockDisconnectedResult `json:"result"`
		Error  *btcjson.Error          `json:"error"`
		Id     interface{}             `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = BlockDisconnectedNtfn(ntfn.Result)
	return nil
}

// TxMinedNtfn is a type handling custom marshaling and
// unmarshaling of txmined JSON websocket notifications.
type TxMinedNtfn struct {
	TxID        string `json:"txid"`
	BlockHash   string `json:"blockhash"`
	BlockHeight int32  `json:"blockheight"`
	BlockTime   int64  `json:"blocktime"`
	Index       int    `json:"index"`
}

type txMinedResult TxMinedNtfn

// Enforce that TxMinedNtfn satisfies the Notification interface.
var _ Notification = &TxMinedNtfn{}

// NewTxMinedNtfn creates a new TxMinedNtfn.
func NewTxMinedNtfn(txid, blockhash string, blockheight int32,
	blocktime int64, index int) *TxMinedNtfn {

	return &TxMinedNtfn{
		TxID:        txid,
		BlockHash:   blockhash,
		BlockHeight: blockheight,
		BlockTime:   blocktime,
		Index:       index,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *TxMinedNtfn) Id() interface{} {
	return TxMinedNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *TxMinedNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *TxMinedNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result txMinedResult  `json:"result"`
		Error  *btcjson.Error `json:"error"`
		Id     interface{}    `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = TxMinedNtfn(ntfn.Result)
	return nil
}

// TxNtfn is a type handling custom marshaling and
// unmarshaling of newtx JSON websocket notifications.
type TxNtfn struct {
	Account string                 `json:"account"`
	Details map[string]interface{} `json:"details"`
}

type txNtfnResult TxNtfn

// Enforce that TxNtfn satisfies the Notification interface.
var _ Notification = &TxNtfn{}

// NewTxNtfn creates a new TxNtfn.
func NewTxNtfn(account string, details map[string]interface{}) *TxNtfn {
	return &TxNtfn{
		Account: account,
		Details: details,
	}
}

// Id satisifies the Notification interface by returning the id of the
// notification.
func (n *TxNtfn) Id() interface{} {
	return TxNtfnId
}

// MarshalJSON returns the JSON encoding of n.  Part of the Notification
// interface.
func (n *TxNtfn) MarshalJSON() ([]byte, error) {
	id := n.Id()
	reply := btcjson.Reply{
		Result: *n,
		Id:     &id,
	}
	return json.Marshal(reply)
}

// UnmarshalJSON unmarshals the JSON encoding of n into n.  Part of
// the Notification interface.
func (n *TxNtfn) UnmarshalJSON(b []byte) error {
	var ntfn struct {
		Result txNtfnResult   `json:"result"`
		Error  *btcjson.Error `json:"error"`
		Id     interface{}    `json:"id"`
	}
	if err := json.Unmarshal(b, &ntfn); err != nil {
		return err
	}

	// Notification IDs must match expected.
	if n.Id() != ntfn.Id {
		return ErrNtfnUnexpected
	}

	*n = TxNtfn(ntfn.Result)
	return nil
}
<|endoftext|>


<commit_before>package faker_test

import (
	"bytes"
	"github.com/olekukonko/tablewriter"
	"github.com/raitucarp/faker"
	"testing"
)

type Table struct {
	data   [][]string
	header []string
	buf    *bytes.Buffer
	table  *tablewriter.Table
}

func (t *Table) SetHeader(header ...string) {
	t.table.SetHeader(header)
}

func (t *Table) AddData(data ...string) {
	t.table.Append(data)
}

func (t *Table) Render() {
	t.table.Render()
}

func (t *Table) String() string {
	return t.buf.String()
}

func NewTable() Table {
	t := Table{}
	t.buf = new(bytes.Buffer)
	t.table = tablewriter.NewWriter(t.buf)

	return t
}

// Testing begin here
func TestAddress(t *testing.T) {
	f := faker.New()
	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("ZipCode", f.Address.ZipCode_())
	tbl.AddData("CityPrefix", f.Address.CityPrefix_())
	tbl.AddData("City", f.Address.City_())
	tbl.AddData("CitySuffix", f.Address.CitySuffix_())
	tbl.AddData("StreetName", f.Address.StreetName_())
	tbl.AddData("StreetAddress", f.Address.StreetAddress_())
	tbl.AddData("StreetSuffix", f.Address.StreetSuffix_())
	tbl.AddData("StreetPrefix", f.Address.StreetPrefix_())
	tbl.AddData("SecondaryAddress", f.Address.SecondaryAddress_())
	tbl.AddData("County", f.Address.County_())
	tbl.AddData("Country", f.Address.Country_())
	tbl.AddData("CountryCode", f.Address.CountryCode_())
	tbl.AddData("State", f.Address.State_())
	tbl.AddData("StateAbbr", f.Address.StateAbbr_())
	tbl.AddData("Latitude", f.Address.Latitude_())
	tbl.AddData("Longitude", f.Address.Longitude_())

	JSON, err := f.Address.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Address.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Address\n", tbl.String())
}

func TestCommerce(t *testing.T) {
	f := faker.New()
	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("Color", f.Commerce.Color_())
	tbl.AddData("Department", f.Commerce.Department_())
	tbl.AddData("ProductName", f.Commerce.ProductName_())
	tbl.AddData("Price", f.Commerce.Price_(20000, 3, "$"))
	tbl.AddData("ProductAdjective", f.Commerce.ProductAdjective_())
	tbl.AddData("ProductMaterial", f.Commerce.ProductMaterial_())
	tbl.AddData("Product", f.Commerce.Product_())

	JSON, err := f.Commerce.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Commerce.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Commerce\n", tbl.String())
}

func TestName(t *testing.T) {
	f := faker.New()

	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("FirstName", f.Name.FirstName_())
	tbl.AddData("LastName", f.Name.LastName_())
	tbl.AddData("FindName", f.Name.FindName_())
	tbl.AddData("JobTitle", f.Name.JobTitle_())
	tbl.AddData("Prefix", f.Name.Prefix_())
	tbl.AddData("Suffix", f.Name.Suffix_())
	tbl.AddData("Title", f.Name.Title_())
	tbl.AddData("JobDescriptor", f.Name.JobDescriptor_())
	tbl.AddData("JobArea", f.Name.JobArea_())
	tbl.AddData("JobType", f.Name.JobType_())

	JSON, err := f.Name.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Name.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Name\n", tbl.String())
}

func checkError(t *testing.T, err error) {
	t.Error(err)
}
<commit_msg>add test for company<commit_after>package faker_test

import (
	"bytes"
	"github.com/olekukonko/tablewriter"
	"github.com/raitucarp/faker"
	"testing"
	"strings"
)

type Table struct {
	data   [][]string
	header []string
	buf    *bytes.Buffer
	table  *tablewriter.Table
}

func (t *Table) SetHeader(header ...string) {
	t.table.SetHeader(header)
}

func (t *Table) AddData(data ...string) {
	t.table.Append(data)
}

func (t *Table) Render() {
	t.table.Render()
}

func (t *Table) String() string {
	return t.buf.String()
}

func NewTable() Table {
	t := Table{}
	t.buf = new(bytes.Buffer)
	t.table = tablewriter.NewWriter(t.buf)

	return t
}

// Testing begin here
func TestAddress(t *testing.T) {
	f := faker.New()
	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("ZipCode", f.Address.ZipCode_())
	tbl.AddData("CityPrefix", f.Address.CityPrefix_())
	tbl.AddData("City", f.Address.City_())
	tbl.AddData("CitySuffix", f.Address.CitySuffix_())
	tbl.AddData("StreetName", f.Address.StreetName_())
	tbl.AddData("StreetAddress", f.Address.StreetAddress_())
	tbl.AddData("StreetSuffix", f.Address.StreetSuffix_())
	tbl.AddData("StreetPrefix", f.Address.StreetPrefix_())
	tbl.AddData("SecondaryAddress", f.Address.SecondaryAddress_())
	tbl.AddData("County", f.Address.County_())
	tbl.AddData("Country", f.Address.Country_())
	tbl.AddData("CountryCode", f.Address.CountryCode_())
	tbl.AddData("State", f.Address.State_())
	tbl.AddData("StateAbbr", f.Address.StateAbbr_())
	tbl.AddData("Latitude", f.Address.Latitude_())
	tbl.AddData("Longitude", f.Address.Longitude_())

	JSON, err := f.Address.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Address.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Address\n", tbl.String())
}

func TestCommerce(t *testing.T) {
	f := faker.New()
	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("Color", f.Commerce.Color_())
	tbl.AddData("Department", f.Commerce.Department_())
	tbl.AddData("ProductName", f.Commerce.ProductName_())
	tbl.AddData("Price", f.Commerce.Price_(20000, 3, "$"))
	tbl.AddData("ProductAdjective", f.Commerce.ProductAdjective_())
	tbl.AddData("ProductMaterial", f.Commerce.ProductMaterial_())
	tbl.AddData("Product", f.Commerce.Product_())

	JSON, err := f.Commerce.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Commerce.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Commerce\n", tbl.String())
}

func TestCompany(t *testing.T) {
	f := faker.New()
	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("Suffixes", strings.Join(f.Company.Suffixes_(), ","))
	tbl.AddData("CompanyName", f.Company.CompanyName_())
	tbl.AddData("CompanySuffix", f.Company.CompanySuffix_())
	tbl.AddData("CatchPhrase", f.Company.CatchPhrase_())
	tbl.AddData("CatchPhraseAdjective", f.Company.CatchPhraseAdjective_())
	tbl.AddData("CatchPhraseDescriptor", f.Company.CatchPhraseAdjective_())
	tbl.AddData("CatchPhraseNoun", f.Company.CatchPhraseNoun_())
	tbl.AddData("Bs", f.Company.BS_())
	tbl.AddData("BsAdjective", f.Company.BsAdjective_())
	tbl.AddData("BsBuzz", f.Company.BsBuzz_())
	tbl.AddData("BsNoun", f.Company.BsNoun_())
	tbl.Render()

	// logging
	t.Log("Fake Company\n", tbl.String())

}

func TestName(t *testing.T) {
	f := faker.New()

	tbl := NewTable()
	tbl.SetHeader("Field Name", "Data")
	tbl.AddData("FirstName", f.Name.FirstName_())
	tbl.AddData("LastName", f.Name.LastName_())
	tbl.AddData("FindName", f.Name.FindName_())
	tbl.AddData("JobTitle", f.Name.JobTitle_())
	tbl.AddData("Prefix", f.Name.Prefix_())
	tbl.AddData("Suffix", f.Name.Suffix_())
	tbl.AddData("Title", f.Name.Title_())
	tbl.AddData("JobDescriptor", f.Name.JobDescriptor_())
	tbl.AddData("JobArea", f.Name.JobArea_())
	tbl.AddData("JobType", f.Name.JobType_())

	JSON, err := f.Name.ToJSON()
	checkError(t, err)
	tbl.AddData("JSON", JSON)

	XML, err := f.Name.ToXML()
	checkError(t, err)
	tbl.AddData("XML", XML)

	tbl.Render()

	// logging
	t.Log("Fake Name\n", tbl.String())
}

func checkError(t *testing.T, err error) {
	t.Error(err)
}
<|endoftext|>


<commit_before>package useragent

import (
	"fmt"
	"os"
)

func UserAgent(version, containerType, productSlug string) string {
	atcExternalURL := os.Getenv("ATC_EXTERNAL_URL")
	buildPipelineName := os.Getenv("BUILD_PIPELINE_NAME")
	buildJobName := os.Getenv("BUILD_JOB_NAME")
	buildName := os.Getenv("BUILD_NAME")

	userAgent := fmt.Sprintf(
		"pivnet-resource/%s (%s/pipelines/%s/jobs/%s/builds/%s %s/%s)",
		version,
		atcExternalURL,
		buildPipelineName,
		buildJobName,
		buildName,
		containerType,
		productSlug,
	)
	return userAgent
}
<commit_msg>Improve readability of user-agent<commit_after>package useragent

import (
	"fmt"
	"os"
)

func UserAgent(version, containerType, productSlug string) string {
	atcExternalURL := os.Getenv("ATC_EXTERNAL_URL")
	buildPipelineName := os.Getenv("BUILD_PIPELINE_NAME")
	buildJobName := os.Getenv("BUILD_JOB_NAME")
	buildName := os.Getenv("BUILD_NAME")

	userAgent := fmt.Sprintf(
		"pivnet-resource/%s (%s/pipelines/%s/jobs/%s/builds/%s -- %s/%s)",
		version,
		atcExternalURL,
		buildPipelineName,
		buildJobName,
		buildName,
		productSlug,
		containerType,
	)
	return userAgent
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"

	"github.com/spf13/pflag"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/cloud-provider/app"
	"k8s.io/cloud-provider/app/config"
	"k8s.io/cloud-provider/options"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
)

// TearDownFunc is to be called to tear down a test server.
type TearDownFunc func()

// TestServer return values supplied by kube-test-ApiServer
type TestServer struct {
	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
	Options              *options.CloudControllerManagerOptions
	Config               *config.Config
	TearDownFn           TearDownFunc // TearDown function
	TmpDir               string       // Temp Dir used, by the apiserver
}

// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
type Logger interface {
	Errorf(format string, args ...interface{})
	Fatalf(format string, args ...interface{})
	Logf(format string, args ...interface{})
}

// StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
// 		 enough time to remove temporary files.
func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
	stopCh := make(chan struct{})
	tearDown := func() {
		close(stopCh)
		if len(result.TmpDir) != 0 {
			os.RemoveAll(result.TmpDir)
		}
	}
	defer func() {
		if result.TearDownFn == nil {
			tearDown()
		}
	}()

	result.TmpDir, err = ioutil.TempDir("", "cloud-controller-manager")
	if err != nil {
		return result, fmt.Errorf("failed to create temp dir: %v", err)
	}

	fs := pflag.NewFlagSet("test", pflag.PanicOnError)

	s, err := options.NewCloudControllerManagerOptions()
	if err != nil {
		return TestServer{}, err
	}
	namedFlagSets := s.Flags([]string{}, []string{})
	for _, f := range namedFlagSets.FlagSets {
		fs.AddFlagSet(f)
	}
	fs.Parse(customFlags)
	if s.SecureServing.BindPort != 0 {
		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
		if err != nil {
			return result, fmt.Errorf("failed to create listener: %v", err)
		}
		s.SecureServing.ServerCert.CertDirectory = result.TmpDir

		t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
	}

	if s.InsecureServing.BindPort != 0 {
		s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
		if err != nil {
			return result, fmt.Errorf("failed to create listener: %v", err)
		}

		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
	}

	config, err := s.Config([]string{}, []string{})
	if err != nil {
		return result, fmt.Errorf("failed to create config from options: %v", err)
	}
	cloudconfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider
	cloud, err := cloudprovider.InitCloudProvider(cloudconfig.Name, cloudconfig.CloudConfigFile)
	if err != nil {
		return result, fmt.Errorf("Cloud provider could not be initialized: %v", err)
	}
	if cloud == nil {
		return result, fmt.Errorf("cloud provider is nil")
	}

	errCh := make(chan error)
	go func(stopCh <-chan struct{}) {
		if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
			errCh <- err
		}
	}(stopCh)

	t.Logf("Waiting for /healthz to be ok...")
	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
	if err != nil {
		return result, fmt.Errorf("failed to create a client: %v", err)
	}
	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
		select {
		case err := <-errCh:
			return false, err
		default:
		}

		result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
		status := 0
		result.StatusCode(&status)
		if status == 200 {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
	}

	// from here the caller must call tearDown
	result.LoopbackClientConfig = config.LoopbackClientConfig
	result.Options = s
	result.Config = config
	result.TearDownFn = tearDown

	return result, nil
}

// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t Logger, flags []string) *TestServer {
	result, err := StartTestServer(t, flags)
	if err == nil {
		return &result
	}

	t.Fatalf("failed to launch server: %v", err)
	return nil
}

func createListenerOnFreePort() (net.Listener, int, error) {
	ln, err := net.Listen("tcp", ":0")
	if err != nil {
		return nil, 0, err
	}

	// get port
	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		ln.Close()
		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
	}

	return ln, tcpAddr.Port, nil
}
<commit_msg>cleanup: fix some error log capitalization<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testing

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"

	"github.com/spf13/pflag"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/cloud-provider/app"
	"k8s.io/cloud-provider/app/config"
	"k8s.io/cloud-provider/options"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
)

// TearDownFunc is to be called to tear down a test server.
type TearDownFunc func()

// TestServer return values supplied by kube-test-ApiServer
type TestServer struct {
	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
	Options              *options.CloudControllerManagerOptions
	Config               *config.Config
	TearDownFn           TearDownFunc // TearDown function
	TmpDir               string       // Temp Dir used, by the apiserver
}

// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
type Logger interface {
	Errorf(format string, args ...interface{})
	Fatalf(format string, args ...interface{})
	Logf(format string, args ...interface{})
}

// StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
// 		 enough time to remove temporary files.
func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
	stopCh := make(chan struct{})
	tearDown := func() {
		close(stopCh)
		if len(result.TmpDir) != 0 {
			os.RemoveAll(result.TmpDir)
		}
	}
	defer func() {
		if result.TearDownFn == nil {
			tearDown()
		}
	}()

	result.TmpDir, err = ioutil.TempDir("", "cloud-controller-manager")
	if err != nil {
		return result, fmt.Errorf("failed to create temp dir: %v", err)
	}

	fs := pflag.NewFlagSet("test", pflag.PanicOnError)

	s, err := options.NewCloudControllerManagerOptions()
	if err != nil {
		return TestServer{}, err
	}
	namedFlagSets := s.Flags([]string{}, []string{})
	for _, f := range namedFlagSets.FlagSets {
		fs.AddFlagSet(f)
	}
	fs.Parse(customFlags)
	if s.SecureServing.BindPort != 0 {
		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
		if err != nil {
			return result, fmt.Errorf("failed to create listener: %v", err)
		}
		s.SecureServing.ServerCert.CertDirectory = result.TmpDir

		t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
	}

	if s.InsecureServing.BindPort != 0 {
		s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
		if err != nil {
			return result, fmt.Errorf("failed to create listener: %v", err)
		}

		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
	}

	config, err := s.Config([]string{}, []string{})
	if err != nil {
		return result, fmt.Errorf("failed to create config from options: %v", err)
	}
	cloudConfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider
	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
	if err != nil {
		return result, fmt.Errorf("Cloud provider could not be initialized: %v", err)
	}
	if cloud == nil {
		return result, fmt.Errorf("cloud provider is nil")
	}

	errCh := make(chan error)
	go func(stopCh <-chan struct{}) {
		if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
			errCh <- err
		}
	}(stopCh)

	t.Logf("Waiting for /healthz to be ok...")
	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
	if err != nil {
		return result, fmt.Errorf("failed to create a client: %v", err)
	}
	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
		select {
		case err := <-errCh:
			return false, err
		default:
		}

		result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
		status := 0
		result.StatusCode(&status)
		if status == 200 {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
	}

	// from here the caller must call tearDown
	result.LoopbackClientConfig = config.LoopbackClientConfig
	result.Options = s
	result.Config = config
	result.TearDownFn = tearDown

	return result, nil
}

// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t Logger, flags []string) *TestServer {
	result, err := StartTestServer(t, flags)
	if err == nil {
		return &result
	}

	t.Fatalf("failed to launch server: %v", err)
	return nil
}

func createListenerOnFreePort() (net.Listener, int, error) {
	ln, err := net.Listen("tcp", ":0")
	if err != nil {
		return nil, 0, err
	}

	// get port
	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		ln.Close()
		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
	}

	return ln, tcpAddr.Port, nil
}
<|endoftext|>


<commit_before>// Package utilities implements a host of helpful miscellaneous functions to the library.
package utilities

import (
	"fmt"
	rand "math/rand"
	"sort"
	"strconv"

	mat64 "github.com/gonum/matrix/mat64"
)

type sortedIntMap struct {
	m map[int]float64
	s []int
}

func (sm *sortedIntMap) Len() int {
	return len(sm.m)
}

func (sm *sortedIntMap) Less(i, j int) bool {
	return sm.m[sm.s[i]] < sm.m[sm.s[j]]
}

func (sm *sortedIntMap) Swap(i, j int) {
	sm.s[i], sm.s[j] = sm.s[j], sm.s[i]
}

func SortIntMap(m map[int]float64) []int {
	sm := new(sortedIntMap)
	sm.m = m
	sm.s = make([]int, len(m))
	i := 0
	for key := range m {
		sm.s[i] = key
		i++
	}
	sort.Sort(sm)
	return sm.s
}

type sortedStringMap struct {
	m map[string]int
	s []string
}

func (sm *sortedStringMap) Len() int {
	return len(sm.m)
}

func (sm *sortedStringMap) Less(i, j int) bool {
	return sm.m[sm.s[i]] < sm.m[sm.s[j]]
}

func (sm *sortedStringMap) Swap(i, j int) {
	sm.s[i], sm.s[j] = sm.s[j], sm.s[i]
}

func SortStringMap(m map[string]int) []string {
	sm := new(sortedStringMap)
	sm.m = m
	sm.s = make([]string, len(m))
	i := 0
	for key := range m {
		sm.s[i] = key
		i++
	}
	sort.Sort(sm)
	return sm.s
}

func RandomArray(n int, k int) []float64 {
	ReturnedArray := make([]float64, n)
	for i := 0; i < n; i++ {
		ReturnedArray[i] = rand.Float64() * float64(rand.Intn(k))
	}
	return ReturnedArray
}

func ConvertLabelsToFloat(labels []string) []float64 {
	floats := make([]float64, 0)
	for _, elem := range labels {
		converted, err := strconv.ParseFloat(elem, 64)

		if err != nil {
			fmt.Println(err)
		}

		floats = append(floats, converted)
	}
	return floats
}

func FloatsToMatrix(floats []float64) *mat64.Dense {
	return mat64.NewDense(1, len(floats), floats)
}
<commit_msg>Remove unused untested utility functions<commit_after>// Package utilities implements a host of helpful miscellaneous functions to the library.
package utilities

import (
	"sort"

	"github.com/gonum/matrix/mat64"
)

type sortedIntMap struct {
	m map[int]float64
	s []int
}

func (sm *sortedIntMap) Len() int {
	return len(sm.m)
}

func (sm *sortedIntMap) Less(i, j int) bool {
	return sm.m[sm.s[i]] < sm.m[sm.s[j]]
}

func (sm *sortedIntMap) Swap(i, j int) {
	sm.s[i], sm.s[j] = sm.s[j], sm.s[i]
}

func SortIntMap(m map[int]float64) []int {
	sm := new(sortedIntMap)
	sm.m = m
	sm.s = make([]int, len(m))
	i := 0
	for key := range m {
		sm.s[i] = key
		i++
	}
	sort.Sort(sm)
	return sm.s
}

func FloatsToMatrix(floats []float64) *mat64.Dense {
	return mat64.NewDense(1, len(floats), floats)
}
<|endoftext|>


<commit_before>package validator

import (
	"fmt"
	"math"
	"regexp"
	"strings"
	"unicode/utf8"
)

var (
	// ERR_MSG_X_MUST_BE_OF_TYPE_Y = `%s must be of type %s`

	// ERR_MSG_X_IS_MISSING_AND_REQUIRED  = `%s is missing and required`
	// ERR_MSG_MUST_BE_OF_TYPE_X          = `must be of type %s`
	// ERR_MSG_ARRAY_ITEMS_MUST_BE_UNIQUE = `array items must be unique`

	ERR_MSG_STRING_LENGTH_MUST_BE_GREATER_OR_EQUAL = `string length must be greater or equal to %d`
	ERR_MSG_STRING_LENGTH_MUST_BE_LOWER_OR_EQUAL   = `string length must be lower or equal to %d`
	ERR_MSG_DOES_NOT_MATCH_PATTERN                 = `does not match pattern '%s'`
	ERR_MSG_MUST_MATCH_ONE_ENUM_VALUES             = `must match one of the enum values [%s]`

	ERR_MSG_NUMBER_MUST_BE_GREATER = `must be greater than %f`
	ERR_MSG_NUMBER_MUST_BE_LOWER   = `must be lower than %f`
	ERR_MSG_MULTIPLE_OF            = `must be a multiple of %f`

	// ERR_MSG_NUMBER_MUST_BE_LOWER_OR_EQUAL   = `must be lower than or equal to %s`
	// ERR_MSG_NUMBER_MUST_BE_GREATER_OR_EQUAL = `must be greater than or equal to %f`

	// ERR_MSG_NUMBER_MUST_VALIDATE_ALLOF = `must validate all the schemas (allOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_ONEOF = `must validate one and only one schema (oneOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_ANYOF = `must validate at least one schema (anyOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_NOT   = `must not validate the schema (not)`

	// ERR_MSG_ARRAY_MIN_ITEMS = `array must have at least %d items`
	// ERR_MSG_ARRAY_MAX_ITEMS = `array must have at the most %d items`

	// ERR_MSG_ARRAY_MIN_PROPERTIES = `must have at least %d properties`
	// ERR_MSG_ARRAY_MAX_PROPERTIES = `must have at the most %d properties`

	// ERR_MSG_HAS_DEPENDENCY_ON = `has a dependency on %s`

	// ERR_MSG_ARRAY_NO_ADDITIONAL_ITEM = `no additional item allowed on array`

	// ERR_MSG_ADDITIONAL_PROPERTY_NOT_ALLOWED = `additional property "%s" is not allowed`
	// ERR_MSG_INVALID_PATTERN_PROPERTY        = `property "%s" does not match pattern %s`
)

func MinLength(data string, length int) error {
	if utf8.RuneCount([]byte(data)) < length {
		return fmt.Errorf(ERR_MSG_STRING_LENGTH_MUST_BE_GREATER_OR_EQUAL, length)
	}

	return nil
}

func MaxLength(data string, length int) error {
	if utf8.RuneCount([]byte(data)) > length {
		return fmt.Errorf(ERR_MSG_STRING_LENGTH_MUST_BE_LOWER_OR_EQUAL, length)
	}

	return nil
}

func Pattern(data string, pattern string) error {
	// TODO add caching for compile?
	regex, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}

	if !regex.MatchString(data) {
		return fmt.Errorf(ERR_MSG_DOES_NOT_MATCH_PATTERN, pattern)
	}

	return nil
}

func OneOf(data string, enums []string) error {
	for _, val := range enums {
		if val == data {
			return nil
		}
	}

	return fmt.Errorf(ERR_MSG_MUST_MATCH_ONE_ENUM_VALUES, strings.Join(enums, ","))
}

func Min(data float64, min float64) error {
	if data < min {
		return fmt.Errorf(ERR_MSG_NUMBER_MUST_BE_GREATER, min)
	}

	return nil
}

func Max(data float64, max float64) error {
	if data > max {
		return fmt.Errorf(ERR_MSG_NUMBER_MUST_BE_LOWER, max)
	}

	return nil
}

func MultipleOf(data float64, multipleOf float64) error {
	if math.Mod(data, multipleOf) == 0 {
		return fmt.Errorf(ERR_MSG_MULTIPLE_OF, multipleOf)
	}

	return nil
}
<commit_msg>Validator: they should satisfy Validate interface<commit_after>package validator

import (
	"fmt"
	"math"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	// ERR_MSG_X_MUST_BE_OF_TYPE_Y = `%s must be of type %s`

	// ERR_MSG_X_IS_MISSING_AND_REQUIRED  = `%s is missing and required`
	// ERR_MSG_MUST_BE_OF_TYPE_X          = `must be of type %s`
	// ERR_MSG_ARRAY_ITEMS_MUST_BE_UNIQUE = `array items must be unique`

	ERR_MSG_STRING_LENGTH_MUST_BE_GREATER_OR_EQUAL = `string length must be greater or equal to %d`
	ERR_MSG_STRING_LENGTH_MUST_BE_LOWER_OR_EQUAL   = `string length must be lower or equal to %d`
	ERR_MSG_DOES_NOT_MATCH_PATTERN                 = `does not match pattern '%s'`
	ERR_MSG_MUST_MATCH_ONE_ENUM_VALUES             = `must match one of the enum values [%s]`

	ERR_MSG_NUMBER_MUST_BE_GREATER = `must be greater than %f`
	ERR_MSG_NUMBER_MUST_BE_LOWER   = `must be lower than %f`
	ERR_MSG_MULTIPLE_OF            = `must be a multiple of %f`

	ERR_MSG_DATE = `date should not be zero`

	// ERR_MSG_NUMBER_MUST_BE_LOWER_OR_EQUAL   = `must be lower than or equal to %s`
	// ERR_MSG_NUMBER_MUST_BE_GREATER_OR_EQUAL = `must be greater than or equal to %f`

	// ERR_MSG_NUMBER_MUST_VALIDATE_ALLOF = `must validate all the schemas (allOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_ONEOF = `must validate one and only one schema (oneOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_ANYOF = `must validate at least one schema (anyOf)`
	// ERR_MSG_NUMBER_MUST_VALIDATE_NOT   = `must not validate the schema (not)`

	// ERR_MSG_ARRAY_MIN_ITEMS = `array must have at least %d items`
	// ERR_MSG_ARRAY_MAX_ITEMS = `array must have at the most %d items`

	// ERR_MSG_ARRAY_MIN_PROPERTIES = `must have at least %d properties`
	// ERR_MSG_ARRAY_MAX_PROPERTIES = `must have at the most %d properties`

	// ERR_MSG_HAS_DEPENDENCY_ON = `has a dependency on %s`

	// ERR_MSG_ARRAY_NO_ADDITIONAL_ITEM = `no additional item allowed on array`

	// ERR_MSG_ADDITIONAL_PROPERTY_NOT_ALLOWED = `additional property "%s" is not allowed`
	// ERR_MSG_INVALID_PATTERN_PROPERTY        = `property "%s" does not match pattern %s`
)

type Validator interface {
	Validate() error
}

type valide struct {
	f func() error
}

func (v *valide) Validate() error {
	return v.f()
}

func MinLength(data string, length int) Validator {
	return &valide{
		f: func() error {
			if utf8.RuneCount([]byte(data)) < length {
				return fmt.Errorf(ERR_MSG_STRING_LENGTH_MUST_BE_GREATER_OR_EQUAL, length)
			}

			return nil
		},
	}
}

func MaxLength(data string, length int) Validator {
	return &valide{
		f: func() error {
			if utf8.RuneCount([]byte(data)) > length {
				return fmt.Errorf(ERR_MSG_STRING_LENGTH_MUST_BE_LOWER_OR_EQUAL, length)
			}

			return nil
		},
	}
}

func Pattern(data string, pattern string) Validator {
	return &valide{
		f: func() error {
			// TODO add caching for compile?
			regex, err := regexp.Compile(pattern)
			if err != nil {
				return err
			}

			if !regex.MatchString(data) {
				return fmt.Errorf(ERR_MSG_DOES_NOT_MATCH_PATTERN, pattern)
			}

			return nil
		},
	}
}
func OneOf(data string, enums []string) Validator {
	return &valide{
		f: func() error {
			for _, val := range enums {
				if val == data {
					return nil
				}
			}

			return fmt.Errorf(ERR_MSG_MUST_MATCH_ONE_ENUM_VALUES, strings.Join(enums, ","))
		},
	}
}

func Min(data float64, min float64) Validator {
	return &valide{
		f: func() error {
			if data < min {
				return fmt.Errorf(ERR_MSG_NUMBER_MUST_BE_GREATER, min)
			}

			return nil
		},
	}
}
func Max(data float64, max float64) Validator {
	return &valide{
		f: func() error {
			if data > max {
				return fmt.Errorf(ERR_MSG_NUMBER_MUST_BE_LOWER, max)
			}

			return nil
		},
	}
}

func MultipleOf(data float64, multipleOf float64) Validator {
	return &valide{
		f: func() error {
			if math.Mod(data, multipleOf) == 0 {
				return fmt.Errorf(ERR_MSG_MULTIPLE_OF, multipleOf)
			}

			return nil
		},
	}
}

func Date(data time.Time) Validator {
	return &valide{
		f: func() error {
			if data.IsZero() {
				return fmt.Errorf(ERR_MSG_DATE)
			}

			return nil
		},
	}
}

func NewMulti(v ...Validator) Validator {
	return &valide{
		f: func() error {
			for _, vv := range v {
				if err := vv.Validate(); err != nil {
					return err
				}
			}

			return nil
		},
	}
}
<|endoftext|>


<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"os"
	"os/exec"
	"strings"
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Helper function for creating reference structure
/*
	'<hash>': {
		'references': {
			[{
				'parent_hash'
				'hash'
				'name'
			}, ]
		}
	}

	if (document_exists) {
		if (references_exists) {
			add_parent_hash to references
		} else {
			add references to document
		}
	} else {
		create document with references as only information
	}
*/
func construct_references(name string, parent_hash string, parent_name string) []map[string]interface{} {
	references := []map[string]interface{}{}

	if name != "" {
		references = []map[string]interface{}{
			{
				"name":        name,
				"parent_hash": parent_hash,
				"parent_name": parent_name,
			},
		}
	}

	return references
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic
		panic(err)

	}
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": construct_references(name, parent_hash, parent_name),
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
				ParentName: name,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	cmd := exec.Command("tika", "-j", path)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Standard error to system standard error
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	if err := json.NewDecoder(stdout).Decode(&metadata); err != nil {
		return err
	}

	return cmd.Wait()
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling file %s\n", hash)

	metadata := make(map[string]interface{})

	if size > 0 {
		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		if err := getMetadata(path, &metadata); err != nil {
			return err
		}
	}

	metadata["size"] = size
	metadata["references"] = construct_references(name, parent_hash, parent_name)

	c.id.IndexItem("file", hash, metadata)

	log.Printf("Finished file %s", hash)

	return nil
}
<commit_msg>Timeout on metadata command.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Helper function for creating reference structure
/*
	'<hash>': {
		'references': {
			[{
				'parent_hash'
				'hash'
				'name'
			}, ]
		}
	}

	if (document_exists) {
		if (references_exists) {
			add_parent_hash to references
		} else {
			add references to document
		}
	} else {
		create document with references as only information
	}
*/
func construct_references(name string, parent_hash string, parent_name string) []map[string]interface{} {
	references := []map[string]interface{}{}

	if name != "" {
		references = []map[string]interface{}{
			{
				"name":        name,
				"parent_hash": parent_hash,
				"parent_name": parent_name,
			},
		}
	}

	return references
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic
		panic(err)

	}
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": construct_references(name, parent_hash, parent_name),
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
				ParentName: name,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	cmd := exec.Command("tika", "-j", path)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Standard error to system standard error
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	if err := json.NewDecoder(stdout).Decode(&metadata); err != nil {
		return err
	}

	// Timeout process after set time
	timer := time.AfterFunc(2*time.Minute, func() {
		cmd.Process.Kill()
	})

	err = cmd.Wait()
	timer.Stop()

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling file %s\n", hash)

	metadata := make(map[string]interface{})

	if size > 0 {
		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		if err := getMetadata(path, &metadata); err != nil {
			return err
		}
	}

	metadata["size"] = size
	metadata["references"] = construct_references(name, parent_hash, parent_name)

	c.id.IndexItem("file", hash, metadata)

	log.Printf("Finished file %s", hash)

	return nil
}
<|endoftext|>


<commit_before>// Package implements asynchronous web sites crawler
// with ability to specify number of workers and tasks.
// Moreover, there is an option to check moztop 500 sites as example.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/fatih/color"
)

var result = map[bool]string{
	true:  color.GreenString("PASS"),
	false: color.RedString("FAIL"),
}

// HTTPTask represents HTTP task with required for processing fields.
// ok idecates that url is fetched without issues within timeout
// url is URL to fetch
// bsize is body size of responce
// start is start time of processing
// timeout is timeout for get request
// elapsed is elapsed time for processing
type HTTPTask struct {
	ok      bool
	url     string
	bsize   int64
	start   time.Time
	timeout time.Duration
	elapsed time.Duration
}

// Process processes and fills required fields of HTTPTask.
func (h *HTTPTask) Process() {
	// Remove creating new client for each task in case of global timeout.
	client := http.Client{Timeout: h.timeout}

	h.start = time.Now()
	resp, err := client.Get(h.url)
	h.elapsed = time.Since(h.start)

	if err != nil {
		h.ok = false
		return
	}
	defer resp.Body.Close()

	// TODO: Think about handling error for Copy.
	h.bsize, _ = io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode == http.StatusOK {
		h.ok = true
		return
	}
	h.ok = false
}

// Output prints out HTTPTask result to standart output.
func (h *HTTPTask) Output() {
	secs := color.BlueString("[%7.2fs]", h.elapsed.Seconds())
	bsize := color.YellowString("[%10db]", h.bsize)
	fmt.Printf("%s %s %s %s\n", result[h.ok], secs, bsize, h.url)
}

type Manufacture struct {
	source      io.Reader
	bufsize     int
	taskTimeout time.Duration
}

func (m *Manufacture) Bufsize() int {
	return m.bufsize
}

func (m *Manufacture) Create(url string) Task {
	h := new(HTTPTask)
	h.url = url
	h.timeout = m.taskTimeout
	return h
}

func (f *Manufacture) URLs() <-chan string {
	urls := make(chan string, f.bufsize)
	s := bufio.NewScanner(f.source)
	go func() {
		defer close(urls)
		for s.Scan() {

			line := strings.TrimSpace(s.Text())

			if line == "" || strings.HasPrefix(line, "#") {
				continue
			}
			urls <- line
			// fmt.Printf("Send URL: %s\n", line)
		}
	}()
	if s.Err() != nil {
		log.Fatalf("error reading: %s", s.Err())
	}
	return urls
}

func NewManufacture(r io.Reader, bufsize int, taskTimeout time.Duration) *Manufacture {
	manufacture := new(Manufacture)
	manufacture.source = r
	manufacture.bufsize = bufsize
	manufacture.taskTimeout = taskTimeout
	return manufacture
}

func main() {
	workers := flag.Int("workers", 2, "number of workers")
	bufsize := flag.Int("bufsize", 0, "size of tasks buffer")
	topsite := flag.Int("topsite", 0, "specify head of moz top sites, max 500")
	timeout := flag.Duration("timeout", 60*time.Second, "request timeout")

	flag.Parse()

	var reader io.Reader = os.Stdin
	if *topsite > 0 {
		if *topsite < 500 {
			reader = MozReader(*topsite)
		} else {
			fmt.Fprintf(os.Stderr,
				"error value: %d, specify top sites within range [1, 500]\n", *topsite)
			os.Exit(1)
		}
	}
	m := NewManufacture(reader, *bufsize, *timeout)
	Run(m, *workers)
}
<commit_msg>Added shared http client for all workers.<commit_after>// Package implements asynchronous web sites crawler
// with ability to specify number of workers and tasks.
// Moreover, there is an option to check moztop 500 sites as example.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/fatih/color"
)

var result = map[bool]string{
	true:  color.GreenString("PASS"),
	false: color.RedString("FAIL"),
}

// HTTPTask represents HTTP task with required for processing fields.
// ok idecates that url is fetched without issues within timeout
// url is URL to fetch
// bsize is body size of responce
// start is start time of processing
// client is http.Client with timeout
// elapsed is elapsed time for processing
type HTTPTask struct {
	ok      bool
	url     string
	bsize   int64
	start   time.Time
	client  *http.Client
	elapsed time.Duration
}

// Process processes and fills required fields of HTTPTask.
func (h *HTTPTask) Process() {
	h.start = time.Now()
	resp, err := h.client.Get(h.url)
	h.elapsed = time.Since(h.start)

	if err != nil {
		h.ok = false
		return
	}
	defer resp.Body.Close()

	// TODO: Think about handling error for Copy.
	h.bsize, _ = io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode == http.StatusOK {
		h.ok = true
		return
	}
	h.ok = false
}

// Output prints out HTTPTask result to standart output.
func (h *HTTPTask) Output() {
	secs := color.BlueString("[%7.2fs]", h.elapsed.Seconds())
	bsize := color.YellowString("[%10db]", h.bsize)
	fmt.Printf("%s %s %s %s\n", result[h.ok], secs, bsize, h.url)
}

type Manufacture struct {
	source     io.Reader
	bufsize    int
	httpClient *http.Client
}

func (m *Manufacture) Bufsize() int {
	return m.bufsize
}

func (m *Manufacture) Create(url string) Task {
	h := new(HTTPTask)
	h.url = url
	h.client = m.httpClient
	return h
}

func (f *Manufacture) URLs() <-chan string {
	urls := make(chan string, f.bufsize)
	s := bufio.NewScanner(f.source)
	go func() {
		defer close(urls)
		for s.Scan() {

			line := strings.TrimSpace(s.Text())

			if line == "" || strings.HasPrefix(line, "#") {
				continue
			}
			urls <- line
			// fmt.Printf("Send URL: %s\n", line)
		}
	}()
	if s.Err() != nil {
		log.Fatalf("error reading: %s", s.Err())
	}
	return urls
}

func NewManufacture(r io.Reader, bufsize int, client *http.Client) *Manufacture {
	m := new(Manufacture)
	m.source = r
	m.bufsize = bufsize
	m.httpClient = client
	return m
}

func main() {
	workers := flag.Int("workers", 2, "number of workers")
	bufsize := flag.Int("bufsize", 0, "size of tasks buffer")
	topsite := flag.Int("topsite", 0, "specify head of moz top sites, max 500")
	timeout := flag.Duration("timeout", 60*time.Second, "request timeout")

	flag.Parse()

	var reader io.Reader = os.Stdin
	if *topsite > 0 {
		if *topsite < 500 {
			reader = MozReader(*topsite)
		} else {
			fmt.Fprintf(os.Stderr,
				"error value: %d, specify top sites within range [1, 500]\n", *topsite)
			os.Exit(1)
		}
	}
	m := NewManufacture(reader, *bufsize, &http.Client{Timeout: *timeout})
	Run(m, *workers)
}
<|endoftext|>