<commit_before>package hostdb

// scan.go contains the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

// queueScan will add a host to the queue to be scanned.
func (hdb *HostDB) queueScan(entry modules.HostDBEntry) {
	// Add the entry to a waitlist, then check if any thread is currently
	// emptying the waitlist. If not, spawn a thread to empty the waitlist.
	hdb.scanList = append(hdb.scanList, entry)
	if hdb.scanWait {
		// Another thread is emptying the scan list, nothing to worry about.
		return
	}

	// Nobody is emptying the scan list, volunteer.
	if hdb.tg.Add() != nil {
		// Hostdb is shutting down, don't spin up another thread.
		return
	}
	hdb.scanWait = true
	go func() {
		defer hdb.tg.Done()

		for {
			hdb.mu.Lock()
			if len(hdb.scanList) == 0 {
				// Scan list is empty, can exit. Let the world know that nobody
				// is emptying the scan list anymore.
				hdb.scanWait = false
				hdb.mu.Unlock()
				return
			}
			// Get the next host, shrink the scan list.
			entry := hdb.scanList[0]
			hdb.scanList = hdb.scanList[1:]
			hdb.mu.Unlock()

			// Block while waiting for an opening in the scan pool.
			select {
			case hdb.scanPool <- entry:
				// iterate again
			case <-hdb.tg.StopChan():
				// quit
				return
			}
		}
	}()
}

// managedUpdateEntry updates an entry in the hostdb after a scan has taken
// place.
//
// CAUTION: This function will automatically add multiple entries to a new host
// to give that host some base uptime. This makes this function co-dependent
// with the host weight functions. Adjustment of the host weight functions need
// to keep this function in mind, and vice-versa.
func (hdb *HostDB) managedUpdateEntry(entry modules.HostDBEntry, netErr error) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// If the host is not online, toss out this update.
	if !hdb.online {
		return
	}

	// Grab the host from the host tree.
	newEntry, exists := hdb.hostTree.Select(entry.PublicKey)
	if exists {
		newEntry.HostExternalSettings = entry.HostExternalSettings
	} else {
		newEntry = entry
	}

	// Add the datapoints for the scan.
	if len(newEntry.ScanHistory) < 2 {
		// Add two scans to the scan history. Two are needed because the scans
		// are forward looking, but we want this first scan to represent as
		// much as one week of uptime or downtime.
		earliestStartTime := time.Now().Add(time.Hour * 7 * 24 * -1) // Permit up to a week of starting uptime or downtime.
		suggestedStartTime := time.Now().Add(time.Minute * 10 * time.Duration(hdb.blockHeight-entry.FirstSeen) * -1)
		if suggestedStartTime.Before(earliestStartTime) {
			suggestedStartTime = earliestStartTime
		}
		newEntry.ScanHistory = modules.HostDBScans{
			{Timestamp: suggestedStartTime, Success: netErr == nil},
			{Timestamp: time.Now(), Success: netErr == nil},
		}
	} else {
		newEntry.ScanHistory = append(newEntry.ScanHistory, modules.HostDBScan{Timestamp: time.Now(), Success: netErr == nil})
	}

	// Add the updated entry
	if !exists {
		err := hdb.hostTree.Insert(newEntry)
		if err != nil {
			hdb.log.Println("ERROR: unable to insert entry which is was thought to be new:", err)
		}
	} else {
		err := hdb.hostTree.Modify(newEntry)
		if err != nil {
			hdb.log.Println("ERROR: unable to modify entry which is thought to exist:", err)
		}
	}

	hdb.save()
}

// managedScanHost will connect to a host and grab the settings, verifying
// uptime and updating to the host's preferences.
func (hdb *HostDB) managedScanHost(entry modules.HostDBEntry) {
	// Request settings from the queued host entry.
	netAddr := entry.NetAddress
	pubKey := entry.PublicKey
	hdb.log.Debugf("Scanning host at %v: key %v", netAddr, pubKey)

	var settings modules.HostExternalSettings
	err := func() error {
		dialer := &net.Dialer{
			Cancel:  hdb.tg.StopChan(),
			Timeout: hostRequestTimeout,
		}
		conn, err := dialer.Dial("tcp", string(netAddr))
		if err != nil {
			return err
		}
		connCloseChan := make(chan struct{})
		go func() {
			select {
			case <-hdb.tg.StopChan():
			case <-connCloseChan:
			}
			conn.Close()
		}()
		defer close(connCloseChan)
		conn.SetDeadline(time.Now().Add(hostScanDeadline))

		err = encoding.WriteObject(conn, modules.RPCSettings)
		if err != nil {
			return err
		}
		var pubkey crypto.PublicKey
		copy(pubkey[:], pubKey.Key)
		return crypto.ReadSignedObject(conn, &settings, maxSettingsLen, pubkey)
	}()
	if err != nil {
		hdb.log.Debugf("Scan of host at %v failed: %v", netAddr, err)
	} else {
		hdb.log.Debugf("Scan of host at %v succeeded.", netAddr)
		entry.HostExternalSettings = settings
	}
	// Update the host tree to have a new entry, including the new error.
	hdb.managedUpdateEntry(entry, err)
}

// threadedProbeHosts pulls hosts from the thread pool and runs a scan on them.
func (hdb *HostDB) threadedProbeHosts() {
	err := hdb.tg.Add()
	if err != nil {
		return
	}
	defer hdb.tg.Done()

	for {
		select {
		case <-hdb.tg.StopChan():
			return
		case hostEntry := <-hdb.scanPool:
			// Block the scan until the host is online.
			for {
				hdb.mu.RLock()
				online := hdb.online
				hdb.mu.RUnlock()
				if online {
					break
				}

				// Check again in 30 seconds.
				select {
				case <-time.After(time.Second * 30):
					continue
				case <-hdb.tg.StopChan():
					return
				}
			}

			// There appears to be internet connectivity, continue with the
			// scan.
			hdb.managedScanHost(hostEntry)
		}
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	err := hdb.tg.Add()
	if err != nil {
		return
	}
	defer hdb.tg.Done()

	for {
		// Set up a scan for the hostCheckupQuanity most valuable hosts in the
		// hostdb. Hosts that fail their scans will be docked significantly,
		// pushing them further back in the hierarchy, ensuring that for the
		// most part only online hosts are getting scanned unless there are
		// fewer than hostCheckupQuantity of them.
		//
		// TODO: Cannot use SelectRandom (despite it being faster) because
		// SelectRandom only returns active/online hosts. Need to be scanning
		// the offline ones as well, otherwise a little bit of downtime is a
		// death sentence for a host.
		hdb.mu.Lock()
		checkups := hdb.hostTree.All()
		if len(checkups) > hostCheckupQuantity {
			checkups = checkups[len(checkups)-hostCheckupQuantity:]
		}
		hdb.log.Println("Performing scan on", len(checkups), "hosts")
		for _, host := range checkups {
			hdb.queueScan(host)
		}
		hdb.mu.Unlock()

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(maxScanSleep))
		minBig := big.NewInt(int64(minScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			build.Critical(err)
			// If there's an error, sleep for the default amount of time.
			defaultBig := big.NewInt(int64(defaultScanSleep))
			randSleep = defaultBig.Sub(defaultBig, minBig)
		}

		// Sleep until it's time for the next scan cycle.
		select {
		case <-hdb.tg.StopChan():
			return
		case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
		}
	}
}
<commit_msg>resolve hostdb deadlock<commit_after>package hostdb

// scan.go contains the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

// queueScan will add a host to the queue to be scanned.
func (hdb *HostDB) queueScan(entry modules.HostDBEntry) {
	// Add the entry to a waitlist, then check if any thread is currently
	// emptying the waitlist. If not, spawn a thread to empty the waitlist.
	hdb.scanList = append(hdb.scanList, entry)
	if hdb.scanWait {
		// Another thread is emptying the scan list, nothing to worry about.
		return
	}

	hdb.scanWait = true
	go func() {
		// Nobody is emptying the scan list, volunteer.
		if hdb.tg.Add() != nil {
			// Hostdb is shutting down, don't spin up another thread.  It is
			// okay to leave scanWait set to true as that will not affect
			// shutdown.
			return
		}
		defer hdb.tg.Done()

		for {
			hdb.mu.Lock()
			if len(hdb.scanList) == 0 {
				// Scan list is empty, can exit. Let the world know that nobody
				// is emptying the scan list anymore.
				hdb.scanWait = false
				hdb.mu.Unlock()
				return
			}
			// Get the next host, shrink the scan list.
			entry := hdb.scanList[0]
			hdb.scanList = hdb.scanList[1:]
			hdb.mu.Unlock()

			// Block while waiting for an opening in the scan pool.
			select {
			case hdb.scanPool <- entry:
				// iterate again
			case <-hdb.tg.StopChan():
				// quit
				return
			}
		}
	}()
}

// managedUpdateEntry updates an entry in the hostdb after a scan has taken
// place.
//
// CAUTION: This function will automatically add multiple entries to a new host
// to give that host some base uptime. This makes this function co-dependent
// with the host weight functions. Adjustment of the host weight functions need
// to keep this function in mind, and vice-versa.
func (hdb *HostDB) managedUpdateEntry(entry modules.HostDBEntry, netErr error) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// If the host is not online, toss out this update.
	if !hdb.online {
		return
	}

	// Grab the host from the host tree.
	newEntry, exists := hdb.hostTree.Select(entry.PublicKey)
	if exists {
		newEntry.HostExternalSettings = entry.HostExternalSettings
	} else {
		newEntry = entry
	}

	// Add the datapoints for the scan.
	if len(newEntry.ScanHistory) < 2 {
		// Add two scans to the scan history. Two are needed because the scans
		// are forward looking, but we want this first scan to represent as
		// much as one week of uptime or downtime.
		earliestStartTime := time.Now().Add(time.Hour * 7 * 24 * -1) // Permit up to a week of starting uptime or downtime.
		suggestedStartTime := time.Now().Add(time.Minute * 10 * time.Duration(hdb.blockHeight-entry.FirstSeen) * -1)
		if suggestedStartTime.Before(earliestStartTime) {
			suggestedStartTime = earliestStartTime
		}
		newEntry.ScanHistory = modules.HostDBScans{
			{Timestamp: suggestedStartTime, Success: netErr == nil},
			{Timestamp: time.Now(), Success: netErr == nil},
		}
	} else {
		newEntry.ScanHistory = append(newEntry.ScanHistory, modules.HostDBScan{Timestamp: time.Now(), Success: netErr == nil})
	}

	// Add the updated entry
	if !exists {
		err := hdb.hostTree.Insert(newEntry)
		if err != nil {
			hdb.log.Println("ERROR: unable to insert entry which is was thought to be new:", err)
		}
	} else {
		err := hdb.hostTree.Modify(newEntry)
		if err != nil {
			hdb.log.Println("ERROR: unable to modify entry which is thought to exist:", err)
		}
	}

	hdb.save()
}

// managedScanHost will connect to a host and grab the settings, verifying
// uptime and updating to the host's preferences.
func (hdb *HostDB) managedScanHost(entry modules.HostDBEntry) {
	// Request settings from the queued host entry.
	netAddr := entry.NetAddress
	pubKey := entry.PublicKey
	hdb.log.Debugf("Scanning host at %v: key %v", netAddr, pubKey)

	var settings modules.HostExternalSettings
	err := func() error {
		dialer := &net.Dialer{
			Cancel:  hdb.tg.StopChan(),
			Timeout: hostRequestTimeout,
		}
		conn, err := dialer.Dial("tcp", string(netAddr))
		if err != nil {
			return err
		}
		connCloseChan := make(chan struct{})
		go func() {
			select {
			case <-hdb.tg.StopChan():
			case <-connCloseChan:
			}
			conn.Close()
		}()
		defer close(connCloseChan)
		conn.SetDeadline(time.Now().Add(hostScanDeadline))

		err = encoding.WriteObject(conn, modules.RPCSettings)
		if err != nil {
			return err
		}
		var pubkey crypto.PublicKey
		copy(pubkey[:], pubKey.Key)
		return crypto.ReadSignedObject(conn, &settings, maxSettingsLen, pubkey)
	}()
	if err != nil {
		hdb.log.Debugf("Scan of host at %v failed: %v", netAddr, err)
	} else {
		hdb.log.Debugf("Scan of host at %v succeeded.", netAddr)
		entry.HostExternalSettings = settings
	}
	// Update the host tree to have a new entry, including the new error.
	hdb.managedUpdateEntry(entry, err)
}

// threadedProbeHosts pulls hosts from the thread pool and runs a scan on them.
func (hdb *HostDB) threadedProbeHosts() {
	err := hdb.tg.Add()
	if err != nil {
		return
	}
	defer hdb.tg.Done()

	for {
		select {
		case <-hdb.tg.StopChan():
			return
		case hostEntry := <-hdb.scanPool:
			// Block the scan until the host is online.
			for {
				hdb.mu.RLock()
				online := hdb.online
				hdb.mu.RUnlock()
				if online {
					break
				}

				// Check again in 30 seconds.
				select {
				case <-time.After(time.Second * 30):
					continue
				case <-hdb.tg.StopChan():
					return
				}
			}

			// There appears to be internet connectivity, continue with the
			// scan.
			hdb.managedScanHost(hostEntry)
		}
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	err := hdb.tg.Add()
	if err != nil {
		return
	}
	defer hdb.tg.Done()

	for {
		// Set up a scan for the hostCheckupQuanity most valuable hosts in the
		// hostdb. Hosts that fail their scans will be docked significantly,
		// pushing them further back in the hierarchy, ensuring that for the
		// most part only online hosts are getting scanned unless there are
		// fewer than hostCheckupQuantity of them.
		//
		// TODO: Cannot use SelectRandom (despite it being faster) because
		// SelectRandom only returns active/online hosts. Need to be scanning
		// the offline ones as well, otherwise a little bit of downtime is a
		// death sentence for a host.
		hdb.mu.Lock()
		checkups := hdb.hostTree.All()
		if len(checkups) > hostCheckupQuantity {
			checkups = checkups[len(checkups)-hostCheckupQuantity:]
		}
		hdb.log.Println("Performing scan on", len(checkups), "hosts")
		for _, host := range checkups {
			hdb.queueScan(host)
		}
		hdb.mu.Unlock()

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(maxScanSleep))
		minBig := big.NewInt(int64(minScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			build.Critical(err)
			// If there's an error, sleep for the default amount of time.
			defaultBig := big.NewInt(int64(defaultScanSleep))
			randSleep = defaultBig.Sub(defaultBig, minBig)
		}

		// Sleep until it's time for the next scan cycle.
		select {
		case <-hdb.tg.StopChan():
			return
		case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
		}
	}
}
<|endoftext|>


<commit_before>package scripts

import (
	"fmt"
	"net/url"
	"path/filepath"
	"strings"

	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"
	"github.com/openshift/source-to-image/pkg/api"
	"github.com/openshift/source-to-image/pkg/docker"
	"github.com/openshift/source-to-image/pkg/errors"
	"github.com/openshift/source-to-image/pkg/util"
)

// Installer interface is responsible for installing scripts needed to run the
// build.
type Installer interface {
	InstallRequired(scripts []string, dstDir string) ([]api.InstallResult, error)
	InstallOptional(scripts []string, dstDir string) []api.InstallResult
}

// ScriptHandler provides an interface for various scripts source handlers.
type ScriptHandler interface {
	Get(script string) *api.InstallResult
	Install(*api.InstallResult) error
	SetDestinationDir(string)
	String() string
}

// UrlScriptHandler handles script download using URL.
type UrlScriptHandler struct {
	URL            string
	DestinationDir string
	download       Downloader
	fs             util.FileSystem
}

const sourcesRootAbbrev = "<source-dir>/"

// SetDestinationDir sets the destination where the scripts should be
// downloaded.
func (s *UrlScriptHandler) SetDestinationDir(baseDir string) {
	s.DestinationDir = filepath.Join(baseDir, api.UploadScripts)
}

// String implements the String() function.
func (s *UrlScriptHandler) String() string {
	return "url handler"
}

// Get parses the provided URL and the script name.
func (s *UrlScriptHandler) Get(script string) *api.InstallResult {
	if len(s.URL) == 0 {
		return nil
	}
	scriptURL, err := url.ParseRequestURI(s.URL + "/" + script)
	if err != nil {
		glog.Infof("invalid script url %q: %v", s.URL, err)
		return nil
	}
	return &api.InstallResult{
		Script: script,
		URL:    scriptURL.String(),
	}
}

// Install downloads the script and fix its permissions.
func (s *UrlScriptHandler) Install(r *api.InstallResult) error {
	downloadURL, err := url.Parse(r.URL)
	if err != nil {
		return err
	}
	dst := filepath.Join(s.DestinationDir, r.Script)
	if _, err := s.download.Download(downloadURL, dst); err != nil {
		if e, ok := err.(errors.Error); ok {
			if e.ErrorCode == errors.ScriptsInsideImageError {
				r.Installed = true
				return nil
			}
		}
		return err
	}
	if err := s.fs.Chmod(dst, 0755); err != nil {
		return err
	}
	r.Installed = true
	r.Downloaded = true
	return nil
}

// SourceScriptHandler handles the case when the scripts are contained in the
// source code directory.
type SourceScriptHandler struct {
	DestinationDir string
	fs             util.FileSystem
}

// Get verifies if the script is present in the source directory and get the
// installation result.
func (s *SourceScriptHandler) Get(script string) *api.InstallResult {
	location := filepath.Join(s.DestinationDir, api.SourceScripts, script)
	if s.fs.Exists(location) {
		return &api.InstallResult{Script: script, URL: location}
	}
	// TODO: The '.sti/bin' path inside the source code directory is deprecated
	// and this should (and will) be removed soon.
	location = strings.Replace(location, "s2i", "sti", 1)
	if s.fs.Exists(location) {
		glog.Infof("DEPRECATED: Use .s2i/bin instead of .sti/bin")
		return &api.InstallResult{Script: script, URL: location}
	}
	return nil
}

// String implements the String() function.
func (s *SourceScriptHandler) String() string {
	return "source handler"
}

// Install copies the script into upload directory and fix its permissions.
func (s *SourceScriptHandler) Install(r *api.InstallResult) error {
	dst := filepath.Join(s.DestinationDir, api.UploadScripts, r.Script)
	if err := s.fs.Rename(r.URL, dst); err != nil {
		return err
	}
	if err := s.fs.Chmod(dst, 0755); err != nil {
		return err
	}
	// Make the path to scripts nicer in logs
	parts := strings.Split(r.URL, "/")
	if len(parts) > 3 {
		r.URL = sourcesRootAbbrev + strings.Join(parts[len(parts)-3:], "/")
	}
	r.Installed = true
	r.Downloaded = true
	return nil
}

// SetDestinationDir sets the directory where the scripts should be uploaded.
// In case of SourceScriptHandler this is a source directory root.
func (s *SourceScriptHandler) SetDestinationDir(baseDir string) {
	s.DestinationDir = filepath.Join(baseDir)
}

// ScriptSourceManager manages various script handlers.
type ScriptSourceManager interface {
	Add(ScriptHandler)
	SetDownloader(Downloader)
	Installer
}

// DefaultScriptSourceManager manages the default script lookup and installation
// for source-to-image.
type DefaultScriptSourceManager struct {
	Image      string
	ScriptsURL string
	download   Downloader
	docker     docker.Docker
	dockerAuth dockerClient.AuthConfiguration
	sources    []ScriptHandler
	fs         util.FileSystem
}

// Add registers a new script source handler.
func (m *DefaultScriptSourceManager) Add(s ScriptHandler) {
	if len(m.sources) == 0 {
		m.sources = []ScriptHandler{}
	}
	m.sources = append(m.sources, s)
}

// NewInstaller returns a new instance of the default Installer implementation
func NewInstaller(image string, scriptsURL string, docker docker.Docker, auth dockerClient.AuthConfiguration) Installer {
	m := DefaultScriptSourceManager{
		Image:      image,
		ScriptsURL: scriptsURL,
		dockerAuth: auth,
		docker:     docker,
		fs:         util.NewFileSystem(),
		download:   NewDownloader(),
	}
	// Order is important here, first we try to get the scripts from provided URL,
	// then we look into sources and check for .s2i/bin scripts.
	if len(m.ScriptsURL) > 0 {
		m.Add(&UrlScriptHandler{URL: m.ScriptsURL, download: m.download, fs: m.fs})
	}

	m.Add(&SourceScriptHandler{fs: m.fs})

	// If the detection handlers above fail, try to get the script url from the
	// docker image itself.
	defaultURL, err := m.docker.GetScriptsURL(m.Image)
	if err == nil && defaultURL != "" {
		m.Add(&UrlScriptHandler{URL: defaultURL, download: m.download, fs: m.fs})
	}
	return &m
}

// InstallRequired Downloads and installs required scripts into dstDir, the result is a
// map of scripts with detailed information about each of the scripts install process
// with error if installing some of them failed
func (i *DefaultScriptSourceManager) InstallRequired(scripts []string, dstDir string) ([]api.InstallResult, error) {
	result := i.InstallOptional(scripts, dstDir)
	failedScripts := []string{}
	var err error
	for _, r := range result {
		if r.Error != nil {
			failedScripts = append(failedScripts, r.Script)
		}
	}
	if len(failedScripts) > 0 {
		err = errors.NewInstallRequiredError(failedScripts, docker.ScriptsURLLabel)
	}
	return result, err
}

// InstallOptional downloads and installs a set of scripts into dstDir, the result is a
// map of scripts with detailed information about each of the scripts install process
func (i *DefaultScriptSourceManager) InstallOptional(scripts []string, dstDir string) []api.InstallResult {
	result := []api.InstallResult{}
	for _, script := range scripts {
		installed := false
		for _, e := range i.sources {
			detected := false
			h := e.(ScriptHandler)
			h.SetDestinationDir(dstDir)
			if r := h.Get(script); r != nil {
				if err := h.Install(r); err != nil {
					glog.Errorf("script %q found by the %s, but failed to install: %v", script, h, err)
				} else {
					result = append(result, *r)
					installed = true
					detected = true
					glog.Infof("Using %q installed from %q", script, r.URL)
				}
			}
			if detected {
				break
			}
		}
		if !installed {
			result = append(result, api.InstallResult{
				Script: script,
				Error:  fmt.Errorf("script %q not installed", script),
			})
		}
	}
	return result
}
<commit_msg>Fix .sti/bin detection<commit_after>package scripts

import (
	"fmt"
	"net/url"
	"path/filepath"
	"strings"

	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"
	"github.com/openshift/source-to-image/pkg/api"
	"github.com/openshift/source-to-image/pkg/docker"
	"github.com/openshift/source-to-image/pkg/errors"
	"github.com/openshift/source-to-image/pkg/util"
)

// Installer interface is responsible for installing scripts needed to run the
// build.
type Installer interface {
	InstallRequired(scripts []string, dstDir string) ([]api.InstallResult, error)
	InstallOptional(scripts []string, dstDir string) []api.InstallResult
}

// ScriptHandler provides an interface for various scripts source handlers.
type ScriptHandler interface {
	Get(script string) *api.InstallResult
	Install(*api.InstallResult) error
	SetDestinationDir(string)
	String() string
}

// UrlScriptHandler handles script download using URL.
type UrlScriptHandler struct {
	URL            string
	DestinationDir string
	download       Downloader
	fs             util.FileSystem
}

const sourcesRootAbbrev = "<source-dir>/"

// SetDestinationDir sets the destination where the scripts should be
// downloaded.
func (s *UrlScriptHandler) SetDestinationDir(baseDir string) {
	s.DestinationDir = filepath.Join(baseDir, api.UploadScripts)
}

// String implements the String() function.
func (s *UrlScriptHandler) String() string {
	return "url handler"
}

// Get parses the provided URL and the script name.
func (s *UrlScriptHandler) Get(script string) *api.InstallResult {
	if len(s.URL) == 0 {
		return nil
	}
	scriptURL, err := url.ParseRequestURI(s.URL + "/" + script)
	if err != nil {
		glog.Infof("invalid script url %q: %v", s.URL, err)
		return nil
	}
	return &api.InstallResult{
		Script: script,
		URL:    scriptURL.String(),
	}
}

// Install downloads the script and fix its permissions.
func (s *UrlScriptHandler) Install(r *api.InstallResult) error {
	downloadURL, err := url.Parse(r.URL)
	if err != nil {
		return err
	}
	dst := filepath.Join(s.DestinationDir, r.Script)
	if _, err := s.download.Download(downloadURL, dst); err != nil {
		if e, ok := err.(errors.Error); ok {
			if e.ErrorCode == errors.ScriptsInsideImageError {
				r.Installed = true
				return nil
			}
		}
		return err
	}
	if err := s.fs.Chmod(dst, 0755); err != nil {
		return err
	}
	r.Installed = true
	r.Downloaded = true
	return nil
}

// SourceScriptHandler handles the case when the scripts are contained in the
// source code directory.
type SourceScriptHandler struct {
	DestinationDir string
	fs             util.FileSystem
}

// Get verifies if the script is present in the source directory and get the
// installation result.
func (s *SourceScriptHandler) Get(script string) *api.InstallResult {
	location := filepath.Join(s.DestinationDir, api.SourceScripts, script)
	if s.fs.Exists(location) {
		return &api.InstallResult{Script: script, URL: location}
	}
	// TODO: The '.sti/bin' path inside the source code directory is deprecated
	// and this should (and will) be removed soon.
	location = strings.Replace(location, "s2i/bin", "sti/bin", 1)
	if s.fs.Exists(location) {
		glog.Infof("DEPRECATED: Use .s2i/bin instead of .sti/bin")
		return &api.InstallResult{Script: script, URL: location}
	}
	return nil
}

// String implements the String() function.
func (s *SourceScriptHandler) String() string {
	return "source handler"
}

// Install copies the script into upload directory and fix its permissions.
func (s *SourceScriptHandler) Install(r *api.InstallResult) error {
	dst := filepath.Join(s.DestinationDir, api.UploadScripts, r.Script)
	if err := s.fs.Rename(r.URL, dst); err != nil {
		return err
	}
	if err := s.fs.Chmod(dst, 0755); err != nil {
		return err
	}
	// Make the path to scripts nicer in logs
	parts := strings.Split(r.URL, "/")
	if len(parts) > 3 {
		r.URL = sourcesRootAbbrev + strings.Join(parts[len(parts)-3:], "/")
	}
	r.Installed = true
	r.Downloaded = true
	return nil
}

// SetDestinationDir sets the directory where the scripts should be uploaded.
// In case of SourceScriptHandler this is a source directory root.
func (s *SourceScriptHandler) SetDestinationDir(baseDir string) {
	s.DestinationDir = filepath.Join(baseDir)
}

// ScriptSourceManager manages various script handlers.
type ScriptSourceManager interface {
	Add(ScriptHandler)
	SetDownloader(Downloader)
	Installer
}

// DefaultScriptSourceManager manages the default script lookup and installation
// for source-to-image.
type DefaultScriptSourceManager struct {
	Image      string
	ScriptsURL string
	download   Downloader
	docker     docker.Docker
	dockerAuth dockerClient.AuthConfiguration
	sources    []ScriptHandler
	fs         util.FileSystem
}

// Add registers a new script source handler.
func (m *DefaultScriptSourceManager) Add(s ScriptHandler) {
	if len(m.sources) == 0 {
		m.sources = []ScriptHandler{}
	}
	m.sources = append(m.sources, s)
}

// NewInstaller returns a new instance of the default Installer implementation
func NewInstaller(image string, scriptsURL string, docker docker.Docker, auth dockerClient.AuthConfiguration) Installer {
	m := DefaultScriptSourceManager{
		Image:      image,
		ScriptsURL: scriptsURL,
		dockerAuth: auth,
		docker:     docker,
		fs:         util.NewFileSystem(),
		download:   NewDownloader(),
	}
	// Order is important here, first we try to get the scripts from provided URL,
	// then we look into sources and check for .s2i/bin scripts.
	if len(m.ScriptsURL) > 0 {
		m.Add(&UrlScriptHandler{URL: m.ScriptsURL, download: m.download, fs: m.fs})
	}

	m.Add(&SourceScriptHandler{fs: m.fs})

	// If the detection handlers above fail, try to get the script url from the
	// docker image itself.
	defaultURL, err := m.docker.GetScriptsURL(m.Image)
	if err == nil && defaultURL != "" {
		m.Add(&UrlScriptHandler{URL: defaultURL, download: m.download, fs: m.fs})
	}
	return &m
}

// InstallRequired Downloads and installs required scripts into dstDir, the result is a
// map of scripts with detailed information about each of the scripts install process
// with error if installing some of them failed
func (i *DefaultScriptSourceManager) InstallRequired(scripts []string, dstDir string) ([]api.InstallResult, error) {
	result := i.InstallOptional(scripts, dstDir)
	failedScripts := []string{}
	var err error
	for _, r := range result {
		if r.Error != nil {
			failedScripts = append(failedScripts, r.Script)
		}
	}
	if len(failedScripts) > 0 {
		err = errors.NewInstallRequiredError(failedScripts, docker.ScriptsURLLabel)
	}
	return result, err
}

// InstallOptional downloads and installs a set of scripts into dstDir, the result is a
// map of scripts with detailed information about each of the scripts install process
func (i *DefaultScriptSourceManager) InstallOptional(scripts []string, dstDir string) []api.InstallResult {
	result := []api.InstallResult{}
	for _, script := range scripts {
		installed := false
		for _, e := range i.sources {
			detected := false
			h := e.(ScriptHandler)
			h.SetDestinationDir(dstDir)
			if r := h.Get(script); r != nil {
				if err := h.Install(r); err != nil {
					glog.Errorf("script %q found by the %s, but failed to install: %v", script, h, err)
				} else {
					result = append(result, *r)
					installed = true
					detected = true
					glog.Infof("Using %q installed from %q", script, r.URL)
				}
			}
			if detected {
				break
			}
		}
		if !installed {
			result = append(result, api.InstallResult{
				Script: script,
				Error:  fmt.Errorf("script %q not installed", script),
			})
		}
	}
	return result
}
<|endoftext|>


<commit_before>//TODO make it a key value storage
package server

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"net/rpc"
	"os"
	"sync"
	"time"

	"github.com/mateusbraga/freestore/pkg/view"
)

var register Value

//  ---------- RPC Requests -------------
type ClientRequest int

func (r *ClientRequest) Read(clientView *view.View, reply *Value) error {
	if !clientView.Equal(currentView.View()) {
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.RLock()
	defer register.mu.RUnlock()

	reply.Value = register.Value
	reply.Timestamp = register.Timestamp

	throughput++

	return nil
}

func (r *ClientRequest) Write(value Value, reply *Value) error {
	if !value.View.Equal(currentView.View()) {
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.Lock()
	defer register.mu.Unlock()

	// Two writes with the same timestamp -> give preference to first one. This makes the Write operation idempotent and still read/write coherent.
	if value.Timestamp > register.Timestamp {
		register.Value = value.Value
		register.Timestamp = value.Timestamp
	}

	return nil
}

func (r *ClientRequest) GetCurrentView(value int, reply *view.View) error {
	*reply = *currentView.View()
	log.Println("Done GetCurrentView request")
	return nil
}

// --------- Init ---------
func init() {
	register.mu.Lock() // The register starts locked
	register.Value = nil
	register.Timestamp = 0
}

func init() {
	rpc.Register(new(ClientRequest))
}

// --------- Types ---------
type Value struct {
	Value     interface{}
	Timestamp int

	View *view.View
	Err  error

	mu sync.RWMutex
}

var throughput uint64
var throughputBuffer = make(map[time.Time]uint64, 70)

func collectThroughputWorker() {
	writeLength := rand.Intn(60)
	var lastThroughput uint64

	for now := range time.Tick(time.Second) {
		aux := throughput
		throughputBuffer[now] = aux - lastThroughput
		lastThroughput = aux

		if len(throughputBuffer) == writeLength {
			writeLength = rand.Intn(60)
			saveThroughput()
		}
	}
}

func saveThroughput() {
	file, err := os.OpenFile("/proj/freestore/throughputs.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		//log.Println(err)
		for loopTime, loopThroughput := range throughputBuffer {
			fmt.Printf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339))
			delete(throughputBuffer, loopTime)
		}
		return
	}
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	for loopTime, loopThroughput := range throughputBuffer {
		if _, err = w.Write([]byte(fmt.Sprintf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339)))); err != nil {
			log.Fatalln(err)
		}
		delete(throughputBuffer, loopTime)
	}
}

func init() {
	go collectThroughputWorker()
}
<commit_msg>Reduce time before writing to throughputs file<commit_after>//TODO make it a key value storage
package server

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"net/rpc"
	"os"
	"sync"
	"time"

	"github.com/mateusbraga/freestore/pkg/view"
)

var register Value

//  ---------- RPC Requests -------------
type ClientRequest int

func (r *ClientRequest) Read(clientView *view.View, reply *Value) error {
	if !clientView.Equal(currentView.View()) {
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.RLock()
	defer register.mu.RUnlock()

	reply.Value = register.Value
	reply.Timestamp = register.Timestamp

	throughput++

	return nil
}

func (r *ClientRequest) Write(value Value, reply *Value) error {
	if !value.View.Equal(currentView.View()) {
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.Lock()
	defer register.mu.Unlock()

	// Two writes with the same timestamp -> give preference to first one. This makes the Write operation idempotent and still read/write coherent.
	if value.Timestamp > register.Timestamp {
		register.Value = value.Value
		register.Timestamp = value.Timestamp
	}

	return nil
}

func (r *ClientRequest) GetCurrentView(value int, reply *view.View) error {
	*reply = *currentView.View()
	log.Println("Done GetCurrentView request")
	return nil
}

// --------- Init ---------
func init() {
	register.mu.Lock() // The register starts locked
	register.Value = nil
	register.Timestamp = 0
}

func init() {
	rpc.Register(new(ClientRequest))
}

// --------- Types ---------
type Value struct {
	Value     interface{}
	Timestamp int

	View *view.View
	Err  error

	mu sync.RWMutex
}

var throughput uint64
var throughputBuffer = make(map[time.Time]uint64, 70)

func collectThroughputWorker() {
	writeLength := rand.Intn(20)
	var lastThroughput uint64

	for now := range time.Tick(time.Second) {
		aux := throughput
		throughputBuffer[now] = aux - lastThroughput
		lastThroughput = aux

		if len(throughputBuffer) == writeLength {
			writeLength = rand.Intn(20)
			saveThroughput()
		}
	}
}

func saveThroughput() {
	file, err := os.OpenFile("/proj/freestore/throughputs.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		//log.Println(err)
		for loopTime, loopThroughput := range throughputBuffer {
			fmt.Printf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339))
			delete(throughputBuffer, loopTime)
		}
		return
	}
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	for loopTime, loopThroughput := range throughputBuffer {
		if _, err = w.Write([]byte(fmt.Sprintf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339)))); err != nil {
			log.Fatalln(err)
		}
		delete(throughputBuffer, loopTime)
	}
}

func init() {
	go collectThroughputWorker()
}
<|endoftext|>


<commit_before>package sharings

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"

	"github.com/cozy/cozy-stack/client/auth"
	"github.com/cozy/cozy-stack/client/request"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/oauth"
	"github.com/labstack/echo"
)

// RevokeSharing revokes the sharing and deletes all the OAuth client and
// triggers associated with it.
//
// Revoking a sharing consists of setting the field `Revoked` to `true`.
// When the sharing is of type "two-way" both recipients and sharer have
// trigger(s) and OAuth client(s) to delete.
// In every other cases only the sharer has trigger(s) to delete and only the
// recipients have an OAuth client to delete.
//
// When this function is called it needs to call either `RevokerSharing` or
// `RevokeRecipient` depending on who initiated the revocation. This is
// represented by the `recursive` boolean parameter. The first call has this set
// to `true` while the subsequent call has it set to `false`.
func RevokeSharing(ins *instance.Instance, sharing *Sharing, recursive bool) error {
	var err error
	if sharing.Owner {
		for _, rs := range sharing.Recipients {
			if recursive {
				err = askToRevokeSharing(ins, sharing, &rs)
				if err != nil {
					continue
				}
			}

			if sharing.SharingType == consts.TwoWaySharing {
				err = DeleteOAuthClient(ins, &rs)
				if err != nil {
					continue
				}
			}
		}

		err = removeSharingTriggers(ins, sharing.SID)
		if err != nil {
			return err
		}

	} else {
		if recursive {
			err = askToRevokeRecipient(ins, sharing, &sharing.Sharer)
			if err != nil {
				return err
			}
		}

		err = DeleteOAuthClient(ins, &sharing.Sharer)
		if err != nil {
			return err
		}

		if sharing.SharingType == consts.TwoWaySharing {
			err = removeSharingTriggers(ins, sharing.SID)
			if err != nil {
				return err
			}
		}
	}
	sharing.Revoked = true
	ins.Logger().Debugf("[sharings] Setting status of sharing %s to revoked",
		sharing.SID)
	return couchdb.UpdateDoc(ins, sharing)
}

// RevokeRecipientByContactID revokes a recipient from the given sharing. Only the sharer
// can make this action.
func RevokeRecipientByContactID(ins *instance.Instance, sharing *Sharing, contactID string) error {
	if !sharing.Owner {
		return ErrOnlySharerCanRevokeRecipient
	}

	for i := range sharing.Recipients {
		rs := &sharing.Recipients[i]
		c := rs.Contact(ins)
		if c != nil && c.DocID == contactID {
			// TODO check how askToRevokeRecipient behave when no recipient has accepted the sharing
			if err := askToRevokeSharing(ins, sharing, rs); err != nil {
				return err
			}
			return RevokeRecipient(ins, sharing, rs)
		}
	}

	return nil
}

// RevokeRecipientByClientID revokes a recipient from the given sharing. Only the sharer
// can make this action.
func RevokeRecipientByClientID(ins *instance.Instance, sharing *Sharing, clientID string) error {
	if !sharing.Owner {
		return ErrOnlySharerCanRevokeRecipient
	}

	for i := range sharing.Recipients {
		rs := &sharing.Recipients[i]
		if rs.Client.ClientID == clientID {
			return RevokeRecipient(ins, sharing, rs)
		}
	}

	return nil
}

// RevokeRecipient revokes a recipient from the given sharing. Only the sharer
// can make this action.
//
// If the sharing is of type "two-way" the sharer also has to remove the
// recipient's OAuth client.
//
// If there are no more recipients the sharing is revoked and the corresponding
// trigger is deleted.
func RevokeRecipient(ins *instance.Instance, sharing *Sharing, recipient *Member) error {
	if sharing.SharingType == consts.TwoWaySharing {
		if err := DeleteOAuthClient(ins, recipient); err != nil {
			return err
		}
		recipient.InboundClientID = ""
	}

	recipient.Client = auth.Client{}
	recipient.AccessToken = auth.AccessToken{}
	recipient.Status = consts.SharingStatusRevoked

	toRevoke := true
	for _, recipient := range sharing.Recipients {
		if recipient.Status != consts.SharingStatusRevoked {
			toRevoke = false
		}
	}

	if toRevoke {
		// TODO check how removeSharingTriggers behave when no recipient has accepted the sharing
		if err := removeSharingTriggers(ins, sharing.SID); err != nil {
			ins.Logger().Errorf("[sharings] RevokeRecipient: Could not remove "+
				"triggers for sharing %s: %s", sharing.SID, err)
		}
		sharing.Revoked = true
	}

	return couchdb.UpdateDoc(ins, sharing)
}

// DeleteOAuthClient deletes the OAuth client used by the member for
// authentifying its requests to this cozy.
func DeleteOAuthClient(ins *instance.Instance, rs *Member) error {
	client, err := oauth.FindClient(ins, rs.InboundClientID)
	if err != nil {
		ins.Logger().Errorf("[sharings] DeleteOAuthClient: Could not "+
			"find OAuth client %s: %s", rs.InboundClientID, err)
		return err
	}
	crErr := client.Delete(ins)
	if crErr != nil {
		ins.Logger().Errorf("[sharings] DeleteOAuthClient: Could not "+
			"delete OAuth client %s: %s", rs.InboundClientID, err)
		return errors.New(crErr.Error)
	}

	ins.Logger().Debugf("[sharings] OAuth client %s deleted", rs.InboundClientID)
	rs.InboundClientID = ""
	return nil
}

func askToRevokeSharing(ins *instance.Instance, sharing *Sharing, rs *Member) error {
	return askToRevoke(ins, sharing, rs, "")
}

func askToRevokeRecipient(ins *instance.Instance, sharing *Sharing, rs *Member) error {
	// TODO: If the recipient revoke a one-way sharing, he  cannot request
	// the sharer yet, as he have no credentials
	if rs.RefContact.ID != "" {
		return askToRevoke(ins, sharing, rs, rs.Client.ClientID)
	}
	return nil

}

// TODO Once we will handle error properly (recipient is disconnected and
// what not) analyze the error returned and take proper actions every time this
// function is called.
func askToRevoke(ins *instance.Instance, sharing *Sharing, rs *Member, recipientClientID string) error {
	sharingID := sharing.SID
	c := rs.Contact(ins)
	if c == nil {
		ins.Logger().Errorf("[sharings] askToRevoke: Could not fetch "+
			"recipient %s from database", rs.RefContact.ID)
		return ErrRecipientDoesNotExist
	}

	var path string
	if recipientClientID == "" {
		path = fmt.Sprintf("/sharings/%s", sharingID)
	} else {
		// From the recipient point of view, only a two-way sharing
		// grants him the rights to request the sharer, as he doesn't have
		// any credentials otherwise.
		if sharing.SharingType == consts.TwoWaySharing {
			path = fmt.Sprintf("/sharings/%s/%s", sharingID, rs.InboundClientID)
		} else {
			return nil
		}
	}

	if rs.URL == "" {
		return ErrRecipientHasNoURL
	}
	u, err := url.Parse(rs.URL)
	if err != nil {
		return err
	}

	reqOpts := &request.Options{
		Domain:  u.Host,
		Scheme:  u.Scheme,
		Path:    path,
		Method:  http.MethodDelete,
		Queries: url.Values{consts.QueryParamRecursive: {"false"}},
		Headers: request.Headers{
			echo.HeaderAuthorization: fmt.Sprintf("Bearer %s",
				rs.AccessToken.AccessToken),
		},
		Body:       nil,
		NoResponse: true,
	}

	_, err = request.Req(reqOpts)

	if err != nil {
		if IsAuthError(err) {
			recInfo, errInfo := ExtractRecipientInfo(rs)
			if errInfo != nil {
				return errInfo
			}
			_, err = RefreshTokenAndRetry(ins, sharingID, recInfo, reqOpts)
		}
		if err != nil {
			ins.Logger().Errorf("[sharings] askToRevoke: Could not ask recipient "+
				"%s to revoke sharing %s: %v", rs.contact.Cozy[0].URL, sharingID, err)
		}
		return err
	}

	rs.Client = auth.Client{}
	rs.AccessToken = auth.AccessToken{}
	return nil
}

// RefreshTokenAndRetry is called after an authentication failure.
// It tries to renew the access_token and request again
func RefreshTokenAndRetry(ins *instance.Instance, sharingID string, rec *RecipientInfo, opts *request.Options) (*http.Response, error) {
	ins.Logger().Errorf("[sharing] The request is not authorized. "+
		"Trying to renew the token for %v", rec.Domain)

	req := &auth.Request{
		Domain: opts.Domain,
		Scheme: opts.Scheme,
	}
	// TODO should we be able to refresh a token for recipient->owner
	sharing, recStatus, err := FindSharingMember(ins, sharingID, rec.Client.ClientID)
	if err != nil {
		return nil, err
	}
	refreshToken := rec.AccessToken.RefreshToken
	access, err := req.RefreshToken(&rec.Client, &rec.AccessToken)
	if err != nil {
		ins.Logger().Errorf("[sharing] Refresh token request failed: %v", err)
		return nil, err
	}
	access.RefreshToken = refreshToken
	recStatus.AccessToken = *access
	if err = couchdb.UpdateDoc(ins, sharing); err != nil {
		return nil, err
	}
	opts.Headers["Authorization"] = "Bearer " + access.AccessToken
	res, err := request.Req(opts)
	return res, err
}

// IsAuthError returns true if the given error is an authentication one
func IsAuthError(err error) bool {
	if v, ok := err.(*request.Error); ok {
		return v.Title == "Bad Request" || v.Title == "Unauthorized"
	}
	return false
}
<commit_msg>Refactor a bit revoke.go<commit_after>package sharings

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"

	"github.com/cozy/cozy-stack/client/auth"
	"github.com/cozy/cozy-stack/client/request"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/oauth"
	"github.com/labstack/echo"
)

// RevokeSharing revokes the sharing and deletes all the OAuth client and
// triggers associated with it.
//
// Revoking a sharing consists of setting the field `Revoked` to `true`.
// When the sharing is of type "two-way" both recipients and sharer have
// trigger(s) and OAuth client(s) to delete.
// In every other cases only the sharer has trigger(s) to delete and only the
// recipients have an OAuth client to delete.
func RevokeSharing(ins *instance.Instance, sharing *Sharing, recursive bool) error {
	if sharing.Owner {
		for _, rs := range sharing.Recipients {
			if recursive {
				if err := askToRevokeSharing(ins, sharing, &rs); err != nil {
					ins.Logger().Infof("[sharings] Can't revoke a recipient for %s: %s", sharing.SID, err)
					continue
				}
			}

			if sharing.SharingType == consts.TwoWaySharing {
				if err := DeleteOAuthClient(ins, &rs); err != nil {
					continue
				}
			}
		}

		if err := removeSharingTriggers(ins, sharing.SID); err != nil {
			return err
		}

	} else {
		if recursive {
			if err := askToRevokeRecipient(ins, sharing, &sharing.Sharer); err != nil {
				ins.Logger().Infof("[sharings] Can't revoke the sharer for %s: %s", sharing.SID, err)
			}
		}

		_ = DeleteOAuthClient(ins, &sharing.Sharer)

		if sharing.SharingType == consts.TwoWaySharing {
			if err := removeSharingTriggers(ins, sharing.SID); err != nil {
				return err
			}
		}
	}

	sharing.Revoked = true
	ins.Logger().Debugf("[sharings] Setting status of sharing %s to revoked", sharing.SID)
	return couchdb.UpdateDoc(ins, sharing)
}

// RevokeRecipientByContactID revokes a recipient from the given sharing. Only the sharer
// can make this action.
func RevokeRecipientByContactID(ins *instance.Instance, sharing *Sharing, contactID string) error {
	if !sharing.Owner {
		return ErrOnlySharerCanRevokeRecipient
	}

	for i := range sharing.Recipients {
		rs := &sharing.Recipients[i]
		c := rs.Contact(ins)
		if c != nil && c.DocID == contactID {
			// TODO check how askToRevokeRecipient behave when no recipient has accepted the sharing
			if err := askToRevokeSharing(ins, sharing, rs); err != nil {
				return err
			}
			return RevokeRecipient(ins, sharing, rs)
		}
	}

	return nil
}

// RevokeRecipientByClientID revokes a recipient from the given sharing on the
// Cozy of the sharer. The recipient has already revoked the sharing on its
// Cozy.
func RevokeRecipientByClientID(ins *instance.Instance, sharing *Sharing, clientID string) error {
	if !sharing.Owner {
		return ErrOnlySharerCanRevokeRecipient
	}

	for i := range sharing.Recipients {
		rs := &sharing.Recipients[i]
		if rs.Client.ClientID == clientID {
			return RevokeRecipient(ins, sharing, rs)
		}
	}

	return nil
}

// RevokeRecipient revokes a recipient from the given sharing. Only the sharer
// can make this action.
//
// If the sharing is of type "two-way" the sharer also has to remove the
// recipient's OAuth client.
//
// If there are no more recipients the sharing is revoked and the corresponding
// trigger is deleted.
func RevokeRecipient(ins *instance.Instance, sharing *Sharing, recipient *Member) error {
	if sharing.SharingType == consts.TwoWaySharing {
		_ = DeleteOAuthClient(ins, recipient)
	}

	recipient.Client = auth.Client{}
	recipient.AccessToken = auth.AccessToken{}
	recipient.Status = consts.SharingStatusRevoked

	toRevoke := true
	for _, recipient := range sharing.Recipients {
		if recipient.Status != consts.SharingStatusRevoked {
			toRevoke = false
		}
	}

	if toRevoke {
		// TODO check how removeSharingTriggers behave when no recipient has accepted the sharing
		if err := removeSharingTriggers(ins, sharing.SID); err != nil {
			ins.Logger().Errorf("[sharings] RevokeRecipient: Could not remove "+
				"triggers for sharing %s: %s", sharing.SID, err)
		}
		sharing.Revoked = true
	}

	return couchdb.UpdateDoc(ins, sharing)
}

// DeleteOAuthClient deletes the OAuth client used by the member for
// authentifying its requests to this cozy.
func DeleteOAuthClient(ins *instance.Instance, rs *Member) error {
	if rs.InboundClientID == "" {
		return nil
	}
	client, err := oauth.FindClient(ins, rs.InboundClientID)
	if err != nil {
		ins.Logger().Errorf("[sharings] DeleteOAuthClient: Could not "+
			"find OAuth client %s: %s", rs.InboundClientID, err)
		return err
	}
	if crErr := client.Delete(ins); crErr != nil {
		ins.Logger().Errorf("[sharings] DeleteOAuthClient: Could not "+
			"delete OAuth client %s: %s", rs.InboundClientID, err)
		return errors.New(crErr.Error)
	}

	ins.Logger().Debugf("[sharings] OAuth client %s deleted", rs.InboundClientID)
	rs.InboundClientID = ""
	return nil
}

func askToRevokeSharing(ins *instance.Instance, sharing *Sharing, rs *Member) error {
	return askToRevoke(ins, sharing, rs, "")
}

func askToRevokeRecipient(ins *instance.Instance, sharing *Sharing, rs *Member) error {
	// TODO: If the recipient revoke a one-way sharing, he  cannot request
	// the sharer yet, as he have no credentials
	if rs.RefContact.ID != "" {
		return askToRevoke(ins, sharing, rs, rs.Client.ClientID)
	}
	return nil

}

// TODO Once we will handle error properly (recipient is disconnected and
// what not) analyze the error returned and take proper actions every time this
// function is called.
func askToRevoke(ins *instance.Instance, sharing *Sharing, rs *Member, recipientClientID string) error {
	sharingID := sharing.SID
	c := rs.Contact(ins)
	if c == nil {
		ins.Logger().Errorf("[sharings] askToRevoke: Could not fetch "+
			"recipient %s from database", rs.RefContact.ID)
		return ErrRecipientDoesNotExist
	}

	var path string
	if recipientClientID == "" {
		path = fmt.Sprintf("/sharings/%s", sharingID)
	} else {
		// From the recipient point of view, only a two-way sharing
		// grants him the rights to request the sharer, as he doesn't have
		// any credentials otherwise.
		if sharing.SharingType == consts.TwoWaySharing {
			path = fmt.Sprintf("/sharings/%s/%s", sharingID, rs.InboundClientID)
		} else {
			return nil
		}
	}

	if rs.URL == "" {
		return ErrRecipientHasNoURL
	}
	u, err := url.Parse(rs.URL)
	if err != nil {
		return err
	}

	reqOpts := &request.Options{
		Domain:  u.Host,
		Scheme:  u.Scheme,
		Path:    path,
		Method:  http.MethodDelete,
		Queries: url.Values{consts.QueryParamRecursive: {"false"}},
		Headers: request.Headers{
			echo.HeaderAuthorization: fmt.Sprintf("Bearer %s",
				rs.AccessToken.AccessToken),
		},
		Body:       nil,
		NoResponse: true,
	}

	_, err = request.Req(reqOpts)

	if err != nil {
		if IsAuthError(err) {
			recInfo, errInfo := ExtractRecipientInfo(rs)
			if errInfo != nil {
				return errInfo
			}
			_, err = RefreshTokenAndRetry(ins, sharingID, recInfo, reqOpts)
		}
		if err != nil {
			ins.Logger().Errorf("[sharings] askToRevoke: Could not ask recipient "+
				"%s to revoke sharing %s: %v", rs.contact.Cozy[0].URL, sharingID, err)
		}
		return err
	}

	rs.Client = auth.Client{}
	rs.AccessToken = auth.AccessToken{}
	return nil
}

// RefreshTokenAndRetry is called after an authentication failure.
// It tries to renew the access_token and request again
// TODO move this function outside of revoke.go
// TODO we should just refresh the token, the retry should be outside of this function
func RefreshTokenAndRetry(ins *instance.Instance, sharingID string, rec *RecipientInfo, opts *request.Options) (*http.Response, error) {
	ins.Logger().Errorf("[sharing] The request is not authorized. "+
		"Trying to renew the token for %v", rec.Domain)

	req := &auth.Request{
		Domain: opts.Domain,
		Scheme: opts.Scheme,
	}
	// TODO we should be able to refresh a token for recipient->owner
	sharing, recStatus, err := FindSharingMember(ins, sharingID, rec.Client.ClientID)
	if err != nil {
		return nil, err
	}
	refreshToken := rec.AccessToken.RefreshToken
	access, err := req.RefreshToken(&rec.Client, &rec.AccessToken)
	if err != nil {
		ins.Logger().Errorf("[sharing] Refresh token request failed: %v", err)
		return nil, err
	}
	access.RefreshToken = refreshToken
	recStatus.AccessToken = *access
	if err = couchdb.UpdateDoc(ins, sharing); err != nil {
		return nil, err
	}
	opts.Headers["Authorization"] = "Bearer " + access.AccessToken
	res, err := request.Req(opts)
	return res, err
}

// IsAuthError returns true if the given error is an authentication one
func IsAuthError(err error) bool {
	if v, ok := err.(*request.Error); ok {
		return v.Title == "Bad Request" || v.Title == "Unauthorized"
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2017-2022 The sacloud/usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"fmt"
	"runtime"
)

var (
	// Version app version
	Version = "1.9.0"
	// Revision git commit short commithash
	Revision = "xxxxxx" // set on build time

	// CopyrightYear .
	CopyrightYear = "2017-2022"
)

// FullVersion return usacloud full version text
func FullVersion() string {
	return fmt.Sprintf("%s %s/%s, build %s", Version, runtime.GOOS, runtime.GOARCH, Revision)
}
<commit_msg>Bump to v1.9.1<commit_after>// Copyright 2017-2022 The sacloud/usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"fmt"
	"runtime"
)

var (
	// Version app version
	Version = "1.9.1"
	// Revision git commit short commithash
	Revision = "xxxxxx" // set on build time

	// CopyrightYear .
	CopyrightYear = "2017-2022"
)

// FullVersion return usacloud full version text
func FullVersion() string {
	return fmt.Sprintf("%s %s/%s, build %s", Version, runtime.GOOS, runtime.GOARCH, Revision)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package version represents the current version of the project.
package version // import "k8s.io/helm/pkg/version"

// Version is the current version of the Helm.
// Update this whenever making a new release.
// The version is of the format Major.Minor.Patch
// Increment major number for new feature additions and behavioral changes.
// Increment minor number for bug fixes and performance enhancements.
// Increment patch number for critical fixes to existing releases.
var Version = "v2.0.0-alpha.3"
<commit_msg>chore(*): bump version to v2.0.0-alpha.4<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package version represents the current version of the project.
package version // import "k8s.io/helm/pkg/version"

// Version is the current version of the Helm.
// Update this whenever making a new release.
// The version is of the format Major.Minor.Patch
// Increment major number for new feature additions and behavioral changes.
// Increment minor number for bug fixes and performance enhancements.
// Increment patch number for critical fixes to existing releases.
var Version = "v2.0.0-alpha.4"
<|endoftext|>


<commit_before>package rest

import (
	"github.com/emicklei/go-restful"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/ws"
)

// httpError is the error and message to respond with.
type httpError struct {
	statusCode int
	message    string
}

// Write writes an httpError as the response.
func (e *httpError) Write(response *restful.Response) {
	response.WriteErrorString(e.statusCode, e.message)
}

// RouteFunc represents the routes function
type RouteFunc func(request *restful.Request, response *restful.Response, user schema.User) (interface{}, error)

// RouteFunc1 is a route function that only returns an error, but no value
type RouteFunc1 func(request *restful.Request, response *restful.Response, user schema.User) error

// Handler represents the way a route function should actually be written.
type Handler func(request *restful.Request, response *restful.Response)

// RouteHandler creates a wrapper function for route methods. This allows route
// methods to return errors and have them handled correctly.
func RouteHandler(f RouteFunc) restful.RouteFunction {
	return func(request *restful.Request, response *restful.Response) {
		user := request.Attribute("user").(schema.User)
		val, err := f(request, response, user)
		switch {
		case err != nil:
			httpErr := ws.ErrorToHTTPError(err)
			httpErr.Write(response)
		case val != nil:
			err = response.WriteEntity(val)
			if err != nil {
				app.Log.Errorf("response.WriteEntity failed: %s", err)
			}
		default:
			// Nothing to do
		}
	}
}

// RouteHandler1 creates a wrapper function for route methods that only return an error. See
// RouteHandler for details.
func RouteHandler1(f RouteFunc1) restful.RouteFunction {
	// Create a function that looks like a RouteFunc but always returns null for its second return value
	f2 := func(request *restful.Request, response *restful.Response, user schema.User) (interface{}, error) {
		err := f(request, response, user)
		return nil, err
	}
	return RouteHandler(f2)
}
<commit_msg>Make comment clearer.<commit_after>package rest

import (
	"github.com/emicklei/go-restful"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/ws"
)

// httpError is the error and message to respond with.
type httpError struct {
	statusCode int
	message    string
}

// Write writes an httpError as the response.
func (e *httpError) Write(response *restful.Response) {
	response.WriteErrorString(e.statusCode, e.message)
}

// RouteFunc represents the routes function
type RouteFunc func(request *restful.Request, response *restful.Response, user schema.User) (interface{}, error)

// RouteFunc1 is a route function that only returns an error, but no value
type RouteFunc1 func(request *restful.Request, response *restful.Response, user schema.User) error

// Handler represents the way a route function should actually be written.
type Handler func(request *restful.Request, response *restful.Response)

// RouteHandler creates a wrapper function for route methods. This allows route
// methods to return errors and have them handled correctly.
func RouteHandler(f RouteFunc) restful.RouteFunction {
	return func(request *restful.Request, response *restful.Response) {
		user := request.Attribute("user").(schema.User)
		val, err := f(request, response, user)
		switch {
		case err != nil:
			httpErr := ws.ErrorToHTTPError(err)
			httpErr.Write(response)
		case val != nil:
			err = response.WriteEntity(val)
			if err != nil {
				app.Log.Errorf("response.WriteEntity failed: %s", err)
			}
		default:
			// No error and no value to write - nothing to do.
		}
	}
}

// RouteHandler1 creates a wrapper function for route methods that only return an error. See
// RouteHandler for details.
func RouteHandler1(f RouteFunc1) restful.RouteFunction {
	// Create a function that looks like a RouteFunc but always returns null for its second return value
	f2 := func(request *restful.Request, response *restful.Response, user schema.User) (interface{}, error) {
		err := f(request, response, user)
		return nil, err
	}
	return RouteHandler(f2)
}
<|endoftext|>


<commit_before>package credhub

import (
	"fmt"
	"io/ioutil"
	"net/url"

	"code.cloudfoundry.org/lager"

	"github.com/cloudfoundry-incubator/credhub-cli/credhub"
	"github.com/cloudfoundry-incubator/credhub-cli/credhub/auth"
	"github.com/concourse/atc/creds"
)

type CredHubManager struct {
	URL          string   `long:"url" description:"CredHub server address used to access secrets."`
	PathPrefix   string   `long:"path-prefix" default:"/concourse" description:"Path under which to namespace credential lookup."`
	CACerts      []string `long:"ca-cert" description:"Paths to PEM-encoded CA cert files to use to verify the CredHub server SSL cert."`
	Insecure     bool     `long:"insecure-skip-verify" description:"Enable insecure SSL verification."`
	ClientId     string   `long:"client-id" description:"Client ID for CredHub authorization."`
	ClientSecret string   `long:"client-secret" description:"Client secret for CredHub authorization."`
	caCerts      []string `no-flag:true`
}

func (manager CredHubManager) IsConfigured() bool {
	return manager.URL != "" || manager.ClientId != "" || manager.ClientSecret != "" || len(manager.CACerts) != 0
}

func (manager CredHubManager) Validate() error {
	parsedUrl, err := url.Parse(manager.URL)
	if err != nil {
		return fmt.Errorf("invalid URL: %s", err)
	}
	// "foo" will parse without error (as a Path, with an empty Host)
	// so we'll do a few additional sanity checks that this is a valid URL
	if parsedUrl.Host == "" || !(parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https") {
		return fmt.Errorf("invalid URL")
	}

	if parsedUrl.Scheme == "https" {
		if len(manager.CACerts) < 1 && !manager.Insecure {
			return fmt.Errorf("CACerts or insecure needs to be set for secure urls")
		}
	}

	if manager.ClientId == "" || manager.ClientSecret == "" {
		return fmt.Errorf("--credhub-client-id and --credhub-client-secret must be set to use CredHub")
	}

	for _, cert := range manager.CACerts {
		contents, err := ioutil.ReadFile(cert)
		if err != nil {
			return fmt.Errorf("Could not read CaCert at path %s", cert)
		}
		manager.caCerts = append(manager.caCerts, string(contents))
	}

	return nil
}

func (manager CredHubManager) NewVariablesFactory(logger lager.Logger) (creds.VariablesFactory, error) {
	var skipTls credhub.Option

	if manager.Insecure {
		skipTls = credhub.SkipTLSValidation(true)
	}

	ch, err := credhub.New(manager.URL,
		skipTls,
		credhub.Auth(auth.UaaClientCredentials(manager.ClientId, manager.ClientSecret)))

	return NewCredHubFactory(logger, ch, manager.PathPrefix), err
}
<commit_msg>Fix panic that occurred when "insecure-skip-verify" was not set.<commit_after>package credhub

import (
	"fmt"
	"io/ioutil"
	"net/url"

	"code.cloudfoundry.org/lager"

	"github.com/cloudfoundry-incubator/credhub-cli/credhub"
	"github.com/cloudfoundry-incubator/credhub-cli/credhub/auth"
	"github.com/concourse/atc/creds"
)

type CredHubManager struct {
	URL          string   `long:"url" description:"CredHub server address used to access secrets."`
	PathPrefix   string   `long:"path-prefix" default:"/concourse" description:"Path under which to namespace credential lookup."`
	CACerts      []string `long:"ca-cert" description:"Paths to PEM-encoded CA cert files to use to verify the CredHub server SSL cert."`
	Insecure     bool     `long:"insecure-skip-verify" description:"Enable insecure SSL verification."`
	ClientId     string   `long:"client-id" description:"Client ID for CredHub authorization."`
	ClientSecret string   `long:"client-secret" description:"Client secret for CredHub authorization."`
	caCerts      []string `no-flag:true`
}

func (manager CredHubManager) IsConfigured() bool {
	return manager.URL != "" || manager.ClientId != "" || manager.ClientSecret != "" || len(manager.CACerts) != 0
}

func (manager CredHubManager) Validate() error {
	parsedUrl, err := url.Parse(manager.URL)
	if err != nil {
		return fmt.Errorf("invalid URL: %s", err)
	}
	// "foo" will parse without error (as a Path, with an empty Host)
	// so we'll do a few additional sanity checks that this is a valid URL
	if parsedUrl.Host == "" || !(parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https") {
		return fmt.Errorf("invalid URL")
	}

	if parsedUrl.Scheme == "https" {
		if len(manager.CACerts) < 1 && !manager.Insecure {
			return fmt.Errorf("CACerts or insecure needs to be set for secure urls")
		}
	}

	if manager.ClientId == "" || manager.ClientSecret == "" {
		return fmt.Errorf("--credhub-client-id and --credhub-client-secret must be set to use CredHub")
	}

	for _, cert := range manager.CACerts {
		contents, err := ioutil.ReadFile(cert)
		if err != nil {
			return fmt.Errorf("Could not read CaCert at path %s", cert)
		}
		manager.caCerts = append(manager.caCerts, string(contents))
	}

	return nil
}

func (manager CredHubManager) NewVariablesFactory(logger lager.Logger) (creds.VariablesFactory, error) {
	var options []credhub.Option

	if manager.Insecure {
		options = append(options, credhub.SkipTLSValidation(true))
	}

	options = append(options, credhub.Auth(auth.UaaClientCredentials(manager.ClientId, manager.ClientSecret)))

	ch, err := credhub.New(manager.URL, options...)
	if err != nil {
		return nil, err
	}

	return NewCredHubFactory(logger, ch, manager.PathPrefix), nil
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import "context"

// Collection provides access to the documents in a single collection.
type Collection interface {
	// Name returns the name of the collection.
	Name() string

	// ReadDocument reads a single document with given key from the collection.
	// The document data is stored into result, the document meta data is returned.
	// If no document exists with given key, a NotFoundError is returned.
	ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error)

	// UpdateDocument updates a single document with given key in the collection.
	// The document data is stored into result, the document meta data is returned.
	// If no document exists with given key, a NotFoundError is returned.
	UpdateDocument(ctx context.Context, key string, update map[string]interface{}, resultOld, resultNew interface{}) (DocumentMeta, error)
}
<commit_msg>docs update<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import "context"

// Collection provides access to the documents in a single collection.
type Collection interface {
	// Name returns the name of the collection.
	Name() string

	// ReadDocument reads a single document with given key from the collection.
	// The document data is stored into result, the document meta data is returned.
	// If no document exists with given key, a NotFoundError is returned.
	ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error)

	// UpdateDocument updates a single document with given key in the collection.
	// If `resultOld` is set to non-nil, the OLD document data is stored in there.
	// If `resultNew` is set to non-nil, the NEW document data is stored in there.
	// The document meta data is returned.
	// If no document exists with given key, a NotFoundError is returned.
	UpdateDocument(ctx context.Context, key string, update map[string]interface{}, resultOld, resultNew interface{}) (DocumentMeta, error)
}
<|endoftext|>


<commit_before>package flags

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"unicode/utf8"
)

// Completion is a type containing information of a completion.
type Completion struct {
	// The completed item
	Item string

	// A description of the completed item (optional)
	Description string
}

type completions []Completion

func (c completions) Len() int {
	return len(c)
}

func (c completions) Less(i, j int) bool {
	return c[i].Item < c[j].Item
}

func (c completions) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Completer is an interface which can be implemented by types
// to provide custom command line argument completion.
type Completer interface {
	// Complete receives a prefix representing a (partial) value
	// for its type and should provide a list of possible valid
	// completions.
	Complete(match string) []Completion
}

type completion struct {
	parser *Parser
}

// Filename is a string alias which provides filename completion.
type Filename string

func completionsWithoutDescriptions(items []string) []Completion {
	ret := make([]Completion, len(items))

	for i, v := range items {
		ret[i].Item = v
	}

	return ret
}

// Complete returns a list of existing files with the given
// prefix.
func (f *Filename) Complete(match string) []Completion {
	ret, _ := filepath.Glob(match + "*")
	if len(ret) == 1 {
		if info, err := os.Stat(ret[0]); err == nil && info.IsDir() {
			ret, _ = filepath.Glob(ret[0] + "/*")
		}
	}
	return completionsWithoutDescriptions(ret)
}

func (c *completion) skipPositional(s *parseState, n int) {
	if n >= len(s.positional) {
		s.positional = nil
	} else {
		s.positional = s.positional[n:]
	}
}

func (c *completion) completeOptionNames(s *parseState, prefix string, match string, short bool) []Completion {
	if short && len(match) != 0 {
		return []Completion{
			Completion{
				Item: prefix + match,
			},
		}
	}

	var results []Completion
	repeats := map[string]bool{}

	for name, opt := range s.lookup.longNames {
		if strings.HasPrefix(name, match) && !opt.Hidden {
			results = append(results, Completion{
				Item:        defaultLongOptDelimiter + name,
				Description: opt.Description,
			})

			if short {
				repeats[string(opt.ShortName)] = true
			}
		}
	}

	if short {
		for name, opt := range s.lookup.shortNames {
			if _, exist := repeats[name]; !exist && strings.HasPrefix(name, match) && !opt.Hidden {
				results = append(results, Completion{
					Item:        string(defaultShortOptDelimiter) + name,
					Description: opt.Description,
				})
			}
		}
	}

	return results
}

func (c *completion) completeNamesForLongPrefix(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s, prefix, match, false)
}

func (c *completion) completeNamesForShortPrefix(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s, prefix, match, true)
}

func (c *completion) completeCommands(s *parseState, match string) []Completion {
	n := make([]Completion, 0, len(s.command.commands))

	for _, cmd := range s.command.commands {
		if cmd.data != c && strings.HasPrefix(cmd.Name, match) {
			n = append(n, Completion{
				Item:        cmd.Name,
				Description: cmd.ShortDescription,
			})
		}
	}

	return n
}

func (c *completion) completeValue(value reflect.Value, prefix string, match string) []Completion {
	if value.Kind() == reflect.Slice {
		value = reflect.New(value.Type().Elem())
	}
	i := value.Interface()

	var ret []Completion

	if cmp, ok := i.(Completer); ok {
		ret = cmp.Complete(match)
	} else if value.CanAddr() {
		if cmp, ok = value.Addr().Interface().(Completer); ok {
			ret = cmp.Complete(match)
		}
	}

	for i, v := range ret {
		ret[i].Item = prefix + v.Item
	}

	return ret
}

func (c *completion) complete(args []string) []Completion {
	if len(args) == 0 {
		args = []string{""}
	}

	s := &parseState{
		args: args,
	}

	c.parser.fillParseState(s)

	var opt *Option

	for len(s.args) > 1 {
		arg := s.pop()

		if (c.parser.Options&PassDoubleDash) != None && arg == "--" {
			opt = nil
			c.skipPositional(s, len(s.args)-1)

			break
		}

		if argumentIsOption(arg) {
			prefix, optname, islong := stripOptionPrefix(arg)
			optname, _, argument := splitOption(prefix, optname, islong)

			if argument == nil {
				var o *Option
				canarg := true

				if islong {
					o = s.lookup.longNames[optname]
				} else {
					for i, r := range optname {
						sname := string(r)
						o = s.lookup.shortNames[sname]

						if o == nil {
							break
						}

						if i == 0 && o.canArgument() && len(optname) != len(sname) {
							canarg = false
							break
						}
					}
				}

				if o == nil && (c.parser.Options&PassAfterNonOption) != None {
					opt = nil
					c.skipPositional(s, len(s.args)-1)

					break
				} else if o != nil && o.canArgument() && !o.OptionalArgument && canarg {
					if len(s.args) > 1 {
						s.pop()
					} else {
						opt = o
					}
				}
			}
		} else {
			if len(s.positional) > 0 {
				if !s.positional[0].isRemaining() {
					// Don't advance beyond a remaining positional arg (because
					// it consumes all subsequent args).
					s.positional = s.positional[1:]
				}
			} else if cmd, ok := s.lookup.commands[arg]; ok {
				cmd.fillParseState(s)
			}

			opt = nil
		}
	}

	lastarg := s.args[len(s.args)-1]
	var ret []Completion

	if opt != nil {
		// Completion for the argument of 'opt'
		ret = c.completeValue(opt.value, "", lastarg)
	} else if argumentStartsOption(lastarg) {
		// Complete the option
		prefix, optname, islong := stripOptionPrefix(lastarg)
		optname, split, argument := splitOption(prefix, optname, islong)

		if argument == nil && !islong {
			rname, n := utf8.DecodeRuneInString(optname)
			sname := string(rname)

			if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() {
				ret = c.completeValue(opt.value, prefix+sname, optname[n:])
			} else {
				ret = c.completeNamesForShortPrefix(s, prefix, optname)
			}
		} else if argument != nil {
			if islong {
				opt = s.lookup.longNames[optname]
			} else {
				opt = s.lookup.shortNames[optname]
			}

			if opt != nil {
				ret = c.completeValue(opt.value, prefix+optname+split, *argument)
			}
		} else if islong {
			ret = c.completeNamesForLongPrefix(s, prefix, optname)
		} else {
			ret = c.completeNamesForShortPrefix(s, prefix, optname)
		}
	} else if len(s.positional) > 0 {
		// Complete for positional argument
		ret = c.completeValue(s.positional[0].value, "", lastarg)
	} else if len(s.command.commands) > 0 {
		// Complete for command
		ret = c.completeCommands(s, lastarg)
	}

	sort.Sort(completions(ret))
	return ret
}

func (c *completion) print(items []Completion, showDescriptions bool) {
	if showDescriptions && len(items) > 1 {
		maxl := 0

		for _, v := range items {
			if len(v.Item) > maxl {
				maxl = len(v.Item)
			}
		}

		for _, v := range items {
			fmt.Printf("%s", v.Item)

			if len(v.Description) > 0 {
				fmt.Printf("%s  # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description)
			}

			fmt.Printf("\n")
		}
	} else {
		for _, v := range items {
			fmt.Println(v.Item)
		}
	}
}
<commit_msg>Recurse completion of single directory down<commit_after>package flags

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"unicode/utf8"
)

// Completion is a type containing information of a completion.
type Completion struct {
	// The completed item
	Item string

	// A description of the completed item (optional)
	Description string
}

type completions []Completion

func (c completions) Len() int {
	return len(c)
}

func (c completions) Less(i, j int) bool {
	return c[i].Item < c[j].Item
}

func (c completions) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Completer is an interface which can be implemented by types
// to provide custom command line argument completion.
type Completer interface {
	// Complete receives a prefix representing a (partial) value
	// for its type and should provide a list of possible valid
	// completions.
	Complete(match string) []Completion
}

type completion struct {
	parser *Parser
}

// Filename is a string alias which provides filename completion.
type Filename string

func completionsWithoutDescriptions(items []string) []Completion {
	ret := make([]Completion, len(items))

	for i, v := range items {
		ret[i].Item = v
	}

	return ret
}

// Complete returns a list of existing files with the given
// prefix.
func (f *Filename) Complete(match string) []Completion {
	ret, _ := filepath.Glob(match + "*")
	if len(ret) == 1 {
		if info, err := os.Stat(ret[0]); err == nil && info.IsDir() {
			return f.Complete(ret[0] + "/")
		}
	}
	return completionsWithoutDescriptions(ret)
}

func (c *completion) skipPositional(s *parseState, n int) {
	if n >= len(s.positional) {
		s.positional = nil
	} else {
		s.positional = s.positional[n:]
	}
}

func (c *completion) completeOptionNames(s *parseState, prefix string, match string, short bool) []Completion {
	if short && len(match) != 0 {
		return []Completion{
			Completion{
				Item: prefix + match,
			},
		}
	}

	var results []Completion
	repeats := map[string]bool{}

	for name, opt := range s.lookup.longNames {
		if strings.HasPrefix(name, match) && !opt.Hidden {
			results = append(results, Completion{
				Item:        defaultLongOptDelimiter + name,
				Description: opt.Description,
			})

			if short {
				repeats[string(opt.ShortName)] = true
			}
		}
	}

	if short {
		for name, opt := range s.lookup.shortNames {
			if _, exist := repeats[name]; !exist && strings.HasPrefix(name, match) && !opt.Hidden {
				results = append(results, Completion{
					Item:        string(defaultShortOptDelimiter) + name,
					Description: opt.Description,
				})
			}
		}
	}

	return results
}

func (c *completion) completeNamesForLongPrefix(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s, prefix, match, false)
}

func (c *completion) completeNamesForShortPrefix(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s, prefix, match, true)
}

func (c *completion) completeCommands(s *parseState, match string) []Completion {
	n := make([]Completion, 0, len(s.command.commands))

	for _, cmd := range s.command.commands {
		if cmd.data != c && strings.HasPrefix(cmd.Name, match) {
			n = append(n, Completion{
				Item:        cmd.Name,
				Description: cmd.ShortDescription,
			})
		}
	}

	return n
}

func (c *completion) completeValue(value reflect.Value, prefix string, match string) []Completion {
	if value.Kind() == reflect.Slice {
		value = reflect.New(value.Type().Elem())
	}
	i := value.Interface()

	var ret []Completion

	if cmp, ok := i.(Completer); ok {
		ret = cmp.Complete(match)
	} else if value.CanAddr() {
		if cmp, ok = value.Addr().Interface().(Completer); ok {
			ret = cmp.Complete(match)
		}
	}

	for i, v := range ret {
		ret[i].Item = prefix + v.Item
	}

	return ret
}

func (c *completion) complete(args []string) []Completion {
	if len(args) == 0 {
		args = []string{""}
	}

	s := &parseState{
		args: args,
	}

	c.parser.fillParseState(s)

	var opt *Option

	for len(s.args) > 1 {
		arg := s.pop()

		if (c.parser.Options&PassDoubleDash) != None && arg == "--" {
			opt = nil
			c.skipPositional(s, len(s.args)-1)

			break
		}

		if argumentIsOption(arg) {
			prefix, optname, islong := stripOptionPrefix(arg)
			optname, _, argument := splitOption(prefix, optname, islong)

			if argument == nil {
				var o *Option
				canarg := true

				if islong {
					o = s.lookup.longNames[optname]
				} else {
					for i, r := range optname {
						sname := string(r)
						o = s.lookup.shortNames[sname]

						if o == nil {
							break
						}

						if i == 0 && o.canArgument() && len(optname) != len(sname) {
							canarg = false
							break
						}
					}
				}

				if o == nil && (c.parser.Options&PassAfterNonOption) != None {
					opt = nil
					c.skipPositional(s, len(s.args)-1)

					break
				} else if o != nil && o.canArgument() && !o.OptionalArgument && canarg {
					if len(s.args) > 1 {
						s.pop()
					} else {
						opt = o
					}
				}
			}
		} else {
			if len(s.positional) > 0 {
				if !s.positional[0].isRemaining() {
					// Don't advance beyond a remaining positional arg (because
					// it consumes all subsequent args).
					s.positional = s.positional[1:]
				}
			} else if cmd, ok := s.lookup.commands[arg]; ok {
				cmd.fillParseState(s)
			}

			opt = nil
		}
	}

	lastarg := s.args[len(s.args)-1]
	var ret []Completion

	if opt != nil {
		// Completion for the argument of 'opt'
		ret = c.completeValue(opt.value, "", lastarg)
	} else if argumentStartsOption(lastarg) {
		// Complete the option
		prefix, optname, islong := stripOptionPrefix(lastarg)
		optname, split, argument := splitOption(prefix, optname, islong)

		if argument == nil && !islong {
			rname, n := utf8.DecodeRuneInString(optname)
			sname := string(rname)

			if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() {
				ret = c.completeValue(opt.value, prefix+sname, optname[n:])
			} else {
				ret = c.completeNamesForShortPrefix(s, prefix, optname)
			}
		} else if argument != nil {
			if islong {
				opt = s.lookup.longNames[optname]
			} else {
				opt = s.lookup.shortNames[optname]
			}

			if opt != nil {
				ret = c.completeValue(opt.value, prefix+optname+split, *argument)
			}
		} else if islong {
			ret = c.completeNamesForLongPrefix(s, prefix, optname)
		} else {
			ret = c.completeNamesForShortPrefix(s, prefix, optname)
		}
	} else if len(s.positional) > 0 {
		// Complete for positional argument
		ret = c.completeValue(s.positional[0].value, "", lastarg)
	} else if len(s.command.commands) > 0 {
		// Complete for command
		ret = c.completeCommands(s, lastarg)
	}

	sort.Sort(completions(ret))
	return ret
}

func (c *completion) print(items []Completion, showDescriptions bool) {
	if showDescriptions && len(items) > 1 {
		maxl := 0

		for _, v := range items {
			if len(v.Item) > maxl {
				maxl = len(v.Item)
			}
		}

		for _, v := range items {
			fmt.Printf("%s", v.Item)

			if len(v.Description) > 0 {
				fmt.Printf("%s  # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description)
			}

			fmt.Printf("\n")
		}
	} else {
		for _, v := range items {
			fmt.Println(v.Item)
		}
	}
}
<|endoftext|>