<commit_before>package shell

import (
	"bytes"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"io"
	"math"
)

func init() {
	Commands = append(Commands, &commandS3BucketQuotaEnforce{})
}

type commandS3BucketQuotaEnforce struct {
}

func (c *commandS3BucketQuotaEnforce) Name() string {
	return "s3.bucket.quota.enforce"
}

func (c *commandS3BucketQuotaEnforce) Help() string {
	return `check quota for all buckets, make the bucket read only if over the limit

	Example:
		s3.bucket.quota.enforce -apply
`
}

func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	applyQuotaLimit := bucketCommand.Bool("apply", false, "actually change the buckets readonly attribute")
	if err = bucketCommand.Parse(args); err != nil {
		return nil
	}

	// collect collection information
	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return err
	}
	collectionInfos := make(map[string]*CollectionInfo)
	collectCollectionInfo(topologyInfo, collectionInfos)

	// read buckets path
	var filerBucketsPath string
	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
	if err != nil {
		return fmt.Errorf("read buckets: %v", err)
	}

	// read existing filer configuration
	fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient)
	if err != nil {
		return err
	}

	// process each bucket
	hasConfChanges := false
	err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
		if !entry.IsDirectory {
			return nil
		}
		collection := entry.Name
		var collectionSize float64
		if collectionInfo, found := collectionInfos[collection]; found {
			collectionSize = collectionInfo.Size
		}
		if c.processEachBucket(fc, filerBucketsPath, entry, writer, collectionSize) {
			hasConfChanges = true
		}
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
	}

	// apply the configuration changes
	if hasConfChanges && *applyQuotaLimit {

		var buf2 bytes.Buffer
		fc.ToText(&buf2)

		if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
			return filer.SaveInsideFiler(client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes())
		}); err != nil && err != filer_pb.ErrNotFound {
			return err
		}
	}

	return err

}

func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize float64) (hasConfChanges bool) {

	locPrefix := filerBucketsPath + "/" + entry.Name + "/"
	locConf := fc.MatchStorageRule(locPrefix)
	locConf.LocationPrefix = locPrefix

	if entry.Quota > 0 {
		if locConf.ReadOnly {
			if collectionSize < float64(entry.Quota) {
				locConf.ReadOnly = false
				hasConfChanges = true
			}
		} else {
			if collectionSize > float64(entry.Quota) {
				locConf.ReadOnly = true
				hasConfChanges = true
			}
		}
	} else {
		if locConf.ReadOnly {
			locConf.ReadOnly = false
			hasConfChanges = true
		}
	}

	if hasConfChanges {
		fmt.Fprintf(writer, "  %s\tsize:%d", entry.Name, collectionSize)
		fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, collectionSize*100/float64(entry.Quota))
		fmt.Fprintln(writer)
		if locConf.ReadOnly {
			fmt.Fprintf(writer, "    changing bucket %s to read only!\n", entry.Name)
		} else {
			fmt.Fprintf(writer, "    changing bucket %s to writable.\n", entry.Name)
		}
		fc.AddLocationConf(locConf)
	}

	return
}
<commit_msg>fix test<commit_after>package shell

import (
	"bytes"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"io"
	"math"
)

func init() {
	Commands = append(Commands, &commandS3BucketQuotaEnforce{})
}

type commandS3BucketQuotaEnforce struct {
}

func (c *commandS3BucketQuotaEnforce) Name() string {
	return "s3.bucket.quota.enforce"
}

func (c *commandS3BucketQuotaEnforce) Help() string {
	return `check quota for all buckets, make the bucket read only if over the limit

	Example:
		s3.bucket.quota.enforce -apply
`
}

func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	applyQuotaLimit := bucketCommand.Bool("apply", false, "actually change the buckets readonly attribute")
	if err = bucketCommand.Parse(args); err != nil {
		return nil
	}

	// collect collection information
	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return err
	}
	collectionInfos := make(map[string]*CollectionInfo)
	collectCollectionInfo(topologyInfo, collectionInfos)

	// read buckets path
	var filerBucketsPath string
	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
	if err != nil {
		return fmt.Errorf("read buckets: %v", err)
	}

	// read existing filer configuration
	fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient)
	if err != nil {
		return err
	}

	// process each bucket
	hasConfChanges := false
	err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
		if !entry.IsDirectory {
			return nil
		}
		collection := entry.Name
		var collectionSize float64
		if collectionInfo, found := collectionInfos[collection]; found {
			collectionSize = collectionInfo.Size
		}
		if c.processEachBucket(fc, filerBucketsPath, entry, writer, collectionSize) {
			hasConfChanges = true
		}
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
	}

	// apply the configuration changes
	if hasConfChanges && *applyQuotaLimit {

		var buf2 bytes.Buffer
		fc.ToText(&buf2)

		if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
			return filer.SaveInsideFiler(client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes())
		}); err != nil && err != filer_pb.ErrNotFound {
			return err
		}
	}

	return err

}

func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize float64) (hasConfChanges bool) {

	locPrefix := filerBucketsPath + "/" + entry.Name + "/"
	locConf := fc.MatchStorageRule(locPrefix)
	locConf.LocationPrefix = locPrefix

	if entry.Quota > 0 {
		if locConf.ReadOnly {
			if collectionSize < float64(entry.Quota) {
				locConf.ReadOnly = false
				hasConfChanges = true
			}
		} else {
			if collectionSize > float64(entry.Quota) {
				locConf.ReadOnly = true
				hasConfChanges = true
			}
		}
	} else {
		if locConf.ReadOnly {
			locConf.ReadOnly = false
			hasConfChanges = true
		}
	}

	if hasConfChanges {
		fmt.Fprintf(writer, "  %s\tsize:%.0f", entry.Name, collectionSize)
		fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, collectionSize*100/float64(entry.Quota))
		fmt.Fprintln(writer)
		if locConf.ReadOnly {
			fmt.Fprintf(writer, "    changing bucket %s to read only!\n", entry.Name)
		} else {
			fmt.Fprintf(writer, "    changing bucket %s to writable.\n", entry.Name)
		}
		fc.AddLocationConf(locConf)
	}

	return
}
<|endoftext|>


<commit_before>package testutil

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/attempt"
)

func RunDiscoverdServer(t TestingT, raftPort, httpPort string) (string, func()) {
	killCh := make(chan struct{})
	doneCh := make(chan struct{})

	if raftPort == "" {
		port, err := RandomPort()
		if err != nil {
			t.Fatal("error getting random discoverd raft port: ", err)
		}
		raftPort = port
	}

	if httpPort == "" {
		port, err := RandomPort()
		if err != nil {
			t.Fatal("error getting random discoverd http port: ", err)
		}
		httpPort = port
	}

	// Generate a data directory.
	dataDir, _ := ioutil.TempDir("", "testutil-")

	go func() {
		cmd := exec.Command("discoverd",
			"-host", "127.0.0.1",
			"-raft-addr", "127.0.0.1:"+raftPort,
			"-http-addr", "127.0.0.1:"+httpPort,
			"-dns-addr", "127.0.0.1:0",
			"-data-dir", dataDir,
		)

		var stderr, stdout io.Reader
		if os.Getenv("DEBUG") != "" {
			stderr, _ = cmd.StderrPipe()
			stdout, _ = cmd.StdoutPipe()
		}
		if err := cmd.Start(); err != nil {
			t.Fatal("discoverd start failed: ", err)
			return
		}
		cmdDone := make(chan error)

		go func() {
			if stdout != nil {
				LogOutput("discoverd", stderr, stdout)
			}
			cmdDone <- cmd.Wait()
		}()
		defer close(doneCh)

		select {
		case <-killCh:
			if err := cmd.Process.Kill(); err != nil {
				t.Fatal("failed to kill discoverd:", err)
				return
			}
			err := <-cmdDone
			t.Log("discoverd process exited: ", err)
		case err := <-cmdDone:
			t.Log("discoverd process exited unexpectedly: ", err)
		}
	}()

	// Ping server and wait for leadership.
	if err := waitForLeader(t, "127.0.0.1:"+httpPort, 5*time.Second); err != nil {
		t.Fatal("waiting for leader: %s", err)
	}

	return "127.0.0.1:" + httpPort, func() {
		close(killCh)
		os.RemoveAll(dataDir)
		<-doneCh
	}
}

func BootDiscoverd(t TestingT, raftPort, httpPort string) (*discoverd.Client, func()) {
	addr, killDiscoverd := RunDiscoverdServer(t, raftPort, httpPort)

	client := discoverd.NewClientWithURL("http://" + addr)
	if err := Attempts.Run(client.Ping); err != nil {
		t.Fatal("Failed to connect to discoverd: ", err)
	}
	return client, killDiscoverd
}

func SetupDiscoverd(t TestingT) (*discoverd.Client, func()) {
	client, killDiscoverd := BootDiscoverd(t, "", "")
	return client, func() {
		killDiscoverd()
	}
}

func waitForLeader(t TestingT, host string, timeout time.Duration) error {
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-timer.C:
			return errors.New("leadership timeout")

		case <-ticker.C:
		}

		// Ping HTTP API.
		resp, err := http.Get(fmt.Sprintf("http://%s/raft/leader", host))
		if err != nil {
			t.Log("http get error:", err)
			continue
		}
		resp.Body.Close()

		// Return successfully on 200.
		if resp.StatusCode == http.StatusOK {
			t.Log("discoverd leader established")
			return nil
		}

		// Otherwise log message that we're still waiting.
		t.Log("waiting for leader: status=", resp.StatusCode)
	}
}

func RandomPort() (string, error) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", err
	}
	_, port, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return port, err
}

func LogOutput(name string, rs ...io.Reader) {
	var wg sync.WaitGroup
	wg.Add(len(rs))
	for _, r := range rs {
		go func(r io.Reader) {
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				log.Println(name+":", scanner.Text())
			}
			wg.Done()
		}(r)
	}
	wg.Wait()
}

type TestingT interface {
	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Log(...interface{})
}

var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}
<commit_msg>discoverd: Fix test log message<commit_after>package testutil

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/attempt"
)

func RunDiscoverdServer(t TestingT, raftPort, httpPort string) (string, func()) {
	killCh := make(chan struct{})
	doneCh := make(chan struct{})

	if raftPort == "" {
		port, err := RandomPort()
		if err != nil {
			t.Fatal("error getting random discoverd raft port: ", err)
		}
		raftPort = port
	}

	if httpPort == "" {
		port, err := RandomPort()
		if err != nil {
			t.Fatal("error getting random discoverd http port: ", err)
		}
		httpPort = port
	}

	// Generate a data directory.
	dataDir, _ := ioutil.TempDir("", "testutil-")

	go func() {
		cmd := exec.Command("discoverd",
			"-host", "127.0.0.1",
			"-raft-addr", "127.0.0.1:"+raftPort,
			"-http-addr", "127.0.0.1:"+httpPort,
			"-dns-addr", "127.0.0.1:0",
			"-data-dir", dataDir,
		)

		var stderr, stdout io.Reader
		if os.Getenv("DEBUG") != "" {
			stderr, _ = cmd.StderrPipe()
			stdout, _ = cmd.StdoutPipe()
		}
		if err := cmd.Start(); err != nil {
			t.Fatal("discoverd start failed: ", err)
			return
		}
		cmdDone := make(chan error)

		go func() {
			if stdout != nil {
				LogOutput("discoverd", stderr, stdout)
			}
			cmdDone <- cmd.Wait()
		}()
		defer close(doneCh)

		select {
		case <-killCh:
			if err := cmd.Process.Kill(); err != nil {
				t.Fatal("failed to kill discoverd:", err)
				return
			}
			err := <-cmdDone
			t.Log("discoverd process exited: ", err)
		case err := <-cmdDone:
			t.Log("discoverd process exited unexpectedly: ", err)
		}
	}()

	// Ping server and wait for leadership.
	if err := waitForLeader(t, "127.0.0.1:"+httpPort, 5*time.Second); err != nil {
		t.Fatal("waiting for leader: ", err)
	}

	return "127.0.0.1:" + httpPort, func() {
		close(killCh)
		os.RemoveAll(dataDir)
		<-doneCh
	}
}

func BootDiscoverd(t TestingT, raftPort, httpPort string) (*discoverd.Client, func()) {
	addr, killDiscoverd := RunDiscoverdServer(t, raftPort, httpPort)

	client := discoverd.NewClientWithURL("http://" + addr)
	if err := Attempts.Run(client.Ping); err != nil {
		t.Fatal("Failed to connect to discoverd: ", err)
	}
	return client, killDiscoverd
}

func SetupDiscoverd(t TestingT) (*discoverd.Client, func()) {
	client, killDiscoverd := BootDiscoverd(t, "", "")
	return client, func() {
		killDiscoverd()
	}
}

func waitForLeader(t TestingT, host string, timeout time.Duration) error {
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-timer.C:
			return errors.New("leadership timeout")

		case <-ticker.C:
		}

		// Ping HTTP API.
		resp, err := http.Get(fmt.Sprintf("http://%s/raft/leader", host))
		if err != nil {
			t.Log("http get error:", err)
			continue
		}
		resp.Body.Close()

		// Return successfully on 200.
		if resp.StatusCode == http.StatusOK {
			t.Log("discoverd leader established")
			return nil
		}

		// Otherwise log message that we're still waiting.
		t.Log("waiting for leader: status=", resp.StatusCode)
	}
}

func RandomPort() (string, error) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", err
	}
	_, port, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return port, err
}

func LogOutput(name string, rs ...io.Reader) {
	var wg sync.WaitGroup
	wg.Add(len(rs))
	for _, r := range rs {
		go func(r io.Reader) {
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				log.Println(name+":", scanner.Text())
			}
			wg.Done()
		}(r)
	}
	wg.Wait()
}

type TestingT interface {
	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Log(...interface{})
}

var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}
<|endoftext|>


<commit_before>package dockercredentials

import (
	"net/url"
	"strings"

	"k8s.io/klog"

	"github.com/docker/distribution/registry/client/auth"

	"k8s.io/kubernetes/pkg/credentialprovider"

	"github.com/openshift/origin/pkg/image/registryclient"
)

var (
	emptyKeyring = &credentialprovider.BasicDockerKeyring{}
)

// NewLocal creates a new credential store that uses the default
// local configuration to find a valid authentication for registry
// targets.
func NewLocal() auth.CredentialStore {
	return &keyringCredentialStore{
		DockerKeyring:     credentialprovider.NewDockerKeyring(),
		RefreshTokenStore: registryclient.NewRefreshTokenStore(),
	}
}

// NewFromFile creates a new credential store for the provided Docker config.json
// authentication file.
func NewFromFile(path string) (auth.CredentialStore, error) {
	cfg, err := credentialprovider.ReadSpecificDockerConfigJsonFile(path)
	if err != nil {
		return nil, err
	}
	keyring := &credentialprovider.BasicDockerKeyring{}
	keyring.Add(cfg)
	return &keyringCredentialStore{
		DockerKeyring:     keyring,
		RefreshTokenStore: registryclient.NewRefreshTokenStore(),
	}, nil
}

type keyringCredentialStore struct {
	credentialprovider.DockerKeyring
	registryclient.RefreshTokenStore
}

func (s *keyringCredentialStore) Basic(url *url.URL) (string, string) {
	return BasicFromKeyring(s.DockerKeyring, url)
}

// BasicFromKeyring finds Basic authorization credentials from a Docker keyring for the given URL as username and
// password. It returns empty strings if no such URL matches.
func BasicFromKeyring(keyring credentialprovider.DockerKeyring, target *url.URL) (string, string) {
	// TODO: compare this logic to Docker authConfig in v2 configuration
	var value string
	if len(target.Scheme) == 0 || target.Scheme == "https" {
		value = target.Host + target.Path
	} else {
		// always require an explicit port to look up HTTP credentials
		if !strings.Contains(target.Host, ":") {
			value = target.Host + ":80" + target.Path
		} else {
			value = target.Host + target.Path
		}
	}

	// Lookup(...) expects an image (not a URL path).
	// The keyring strips /v1/ and /v2/ version prefixes,
	// so we should also when selecting a valid auth for a URL.
	pathWithSlash := target.Path + "/"
	if strings.HasPrefix(pathWithSlash, "/v1/") || strings.HasPrefix(pathWithSlash, "/v2/") {
		value = target.Host + target.Path[3:]
	}

	configs, found := keyring.Lookup(value)

	if !found || len(configs) == 0 {
		// do a special case check for docker.io to match historical lookups when we respond to a challenge
		if value == "auth.docker.io/token" {
			klog.V(5).Infof("Being asked for %s (%s), trying %s for legacy behavior", target, value, "index.docker.io/v1")
			return BasicFromKeyring(keyring, &url.URL{Host: "index.docker.io", Path: "/v1"})
		}
		// docker 1.9 saves 'docker.io' in config in f23, see https://bugzilla.redhat.com/show_bug.cgi?id=1309739
		if value == "index.docker.io" {
			klog.V(5).Infof("Being asked for %s (%s), trying %s for legacy behavior", target, value, "docker.io")
			return BasicFromKeyring(keyring, &url.URL{Host: "docker.io"})
		}

		// try removing the canonical ports for the given requests
		if (strings.HasSuffix(target.Host, ":443") && target.Scheme == "https") ||
			(strings.HasSuffix(target.Host, ":80") && target.Scheme == "http") {
			host := strings.SplitN(target.Host, ":", 2)[0]
			klog.V(5).Infof("Being asked for %s (%s), trying %s without port", target, value, host)

			return BasicFromKeyring(keyring, &url.URL{Scheme: target.Scheme, Host: host, Path: target.Path})
		}

		klog.V(5).Infof("Unable to find a secret to match %s (%s)", target, value)
		return "", ""
	}
	klog.V(5).Infof("Found secret to match %s (%s): %s", target, value, configs[0].ServerAddress)
	return configs[0].Username, configs[0].Password
}
<commit_msg>Local registry config should only search home dir, not working dir<commit_after>package dockercredentials

import (
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"k8s.io/klog"

	"github.com/docker/distribution/registry/client/auth"

	"k8s.io/kubernetes/pkg/credentialprovider"

	"github.com/openshift/origin/pkg/image/registryclient"
)

var (
	emptyKeyring = &credentialprovider.BasicDockerKeyring{}
)

// NewLocal creates a new credential store that uses the default
// local configuration to find a valid authentication for registry
// targets.
func NewLocal() auth.CredentialStore {
	keyring := &credentialprovider.BasicDockerKeyring{}
	keyring.Add(defaultClientDockerConfig())
	return &keyringCredentialStore{
		DockerKeyring:     keyring,
		RefreshTokenStore: registryclient.NewRefreshTokenStore(),
	}
}

// NewFromFile creates a new credential store for the provided Docker config.json
// authentication file.
func NewFromFile(path string) (auth.CredentialStore, error) {
	cfg, err := credentialprovider.ReadSpecificDockerConfigJsonFile(path)
	if err != nil {
		return nil, err
	}
	keyring := &credentialprovider.BasicDockerKeyring{}
	keyring.Add(cfg)
	return &keyringCredentialStore{
		DockerKeyring:     keyring,
		RefreshTokenStore: registryclient.NewRefreshTokenStore(),
	}, nil
}

type keyringCredentialStore struct {
	credentialprovider.DockerKeyring
	registryclient.RefreshTokenStore
}

func (s *keyringCredentialStore) Basic(url *url.URL) (string, string) {
	return BasicFromKeyring(s.DockerKeyring, url)
}

// BasicFromKeyring finds Basic authorization credentials from a Docker keyring for the given URL as username and
// password. It returns empty strings if no such URL matches.
func BasicFromKeyring(keyring credentialprovider.DockerKeyring, target *url.URL) (string, string) {
	// TODO: compare this logic to Docker authConfig in v2 configuration
	var value string
	if len(target.Scheme) == 0 || target.Scheme == "https" {
		value = target.Host + target.Path
	} else {
		// always require an explicit port to look up HTTP credentials
		if !strings.Contains(target.Host, ":") {
			value = target.Host + ":80" + target.Path
		} else {
			value = target.Host + target.Path
		}
	}

	// Lookup(...) expects an image (not a URL path).
	// The keyring strips /v1/ and /v2/ version prefixes,
	// so we should also when selecting a valid auth for a URL.
	pathWithSlash := target.Path + "/"
	if strings.HasPrefix(pathWithSlash, "/v1/") || strings.HasPrefix(pathWithSlash, "/v2/") {
		value = target.Host + target.Path[3:]
	}

	configs, found := keyring.Lookup(value)

	if !found || len(configs) == 0 {
		// do a special case check for docker.io to match historical lookups when we respond to a challenge
		if value == "auth.docker.io/token" {
			klog.V(5).Infof("Being asked for %s (%s), trying %s for legacy behavior", target, value, "index.docker.io/v1")
			return BasicFromKeyring(keyring, &url.URL{Host: "index.docker.io", Path: "/v1"})
		}
		// docker 1.9 saves 'docker.io' in config in f23, see https://bugzilla.redhat.com/show_bug.cgi?id=1309739
		if value == "index.docker.io" {
			klog.V(5).Infof("Being asked for %s (%s), trying %s for legacy behavior", target, value, "docker.io")
			return BasicFromKeyring(keyring, &url.URL{Host: "docker.io"})
		}

		// try removing the canonical ports for the given requests
		if (strings.HasSuffix(target.Host, ":443") && target.Scheme == "https") ||
			(strings.HasSuffix(target.Host, ":80") && target.Scheme == "http") {
			host := strings.SplitN(target.Host, ":", 2)[0]
			klog.V(5).Infof("Being asked for %s (%s), trying %s without port", target, value, host)

			return BasicFromKeyring(keyring, &url.URL{Scheme: target.Scheme, Host: host, Path: target.Path})
		}

		klog.V(5).Infof("Unable to find a secret to match %s (%s)", target, value)
		return "", ""
	}
	klog.V(5).Infof("Found secret to match %s (%s): %s", target, value, configs[0].ServerAddress)
	return configs[0].Username, configs[0].Password
}

// defaultClientDockerConfig returns the credentials that the docker command line client would
// return.
func defaultClientDockerConfig() credentialprovider.DockerConfig {
	// support the modern config file $HOME/.docker/config.json
	if cfg, err := credentialprovider.ReadDockerConfigJSONFile(defaultPathsForCredentials()); err == nil {
		return cfg
	}
	// support the legacy config file $HOME/.dockercfg
	if cfg, err := credentialprovider.ReadDockercfgFile(defaultPathsForLegacyCredentials()); err == nil {
		return cfg
	}
	return credentialprovider.DockerConfig{}
}

// defaultPathsForCredentials returns the correct search directories for a docker config
//  file
func defaultPathsForCredentials() []string {
	if runtime.GOOS == "windows" { // Windows
		return []string{filepath.Join(os.Getenv("USERPROFILE"), ".docker")}
	}
	return []string{filepath.Join(os.Getenv("HOME"), ".docker")}
}

// defaultPathsForCredentials returns the correct search directories for a docker config
//  file
func defaultPathsForLegacyCredentials() []string {
	if runtime.GOOS == "windows" { // Windows
		return []string{os.Getenv("USERPROFILE")}
	}
	return []string{os.Getenv("HOME")}
}
<|endoftext|>


<commit_before>package series

import "github.com/Cistern/catena"

type Observation catena.Row
<commit_msg>define Observation struct explicitly<commit_after>package series

import "github.com/Cistern/catena"

type Observation struct {
	Source    string
	Metric    string
	Timestamp int64
	Value     float64
}

func (o Observation) toCatenaRow() catena.Row {
	return catena.Row{
		Source: o.Source,
		Metric: o.Metric,
		Point: catena.Point{
			Timestamp: o.Timestamp,
			Value:     o.Value,
		},
	}
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/config"
)

// NodeApplyableResource represents a resource that is "applyable":
// it is ready to be applied and is represented by a diff.
type NodeApplyableResource struct {
	Addr          *ResourceAddress // Addr is the address for this resource
	Config        *config.Resource // Config is the resource in the config
	ResourceState *ResourceState   // ResourceState is the ResourceState for this
}

func (n *NodeApplyableResource) Name() string {
	return n.Addr.String()
}

// GraphNodeSubPath
func (n *NodeApplyableResource) Path() []string {
	return n.Addr.Path
}

// GraphNodeReferenceable
func (n *NodeApplyableResource) ReferenceableName() []string {
	if n.Config == nil {
		return nil
	}

	return []string{n.Config.Id()}
}

// GraphNodeReferencer
func (n *NodeApplyableResource) References() []string {
	// Let's make this a little shorter so it is easier to reference
	c := n.Config
	if c == nil {
		return nil
	}

	// Grab all the references
	var result []string
	result = append(result, c.DependsOn...)
	result = append(result, ReferencesFromConfig(c.RawCount)...)
	result = append(result, ReferencesFromConfig(c.RawConfig)...)
	for _, p := range c.Provisioners {
		result = append(result, ReferencesFromConfig(p.ConnInfo)...)
		result = append(result, ReferencesFromConfig(p.RawConfig)...)
	}

	return result
}

// GraphNodeProviderConsumer
func (n *NodeApplyableResource) ProvidedBy() []string {
	// If we have a config we prefer that above all else
	if n.Config != nil {
		return []string{resourceProvider(n.Config.Type, n.Config.Provider)}
	}

	// If we have state, then we will use the provider from there
	if n.ResourceState != nil {
		return []string{n.ResourceState.Provider}
	}

	// Use our type
	return []string{resourceProvider(n.Addr.Type, "")}
}

// GraphNodeProvisionerConsumer
func (n *NodeApplyableResource) ProvisionedBy() []string {
	// If we have no configuration, then we have no provisioners
	if n.Config == nil {
		return nil
	}

	// Build the list of provisioners we need based on the configuration.
	// It is okay to have duplicates here.
	result := make([]string, len(n.Config.Provisioners))
	for i, p := range n.Config.Provisioners {
		result[i] = p.Type
	}

	return result
}

// GraphNodeAttachResourceState
func (n *NodeApplyableResource) ResourceAddr() *ResourceAddress {
	return n.Addr
}

// GraphNodeAttachResourceState
func (n *NodeApplyableResource) AttachResourceState(s *ResourceState) {
	n.ResourceState = s
}

// GraphNodeEvalable
func (n *NodeApplyableResource) EvalTree() EvalNode {
	// stateId is the ID to put into the state
	stateId := n.Addr.stateId()
	if n.Addr.Index > -1 {
		stateId = fmt.Sprintf("%s.%d", stateId, n.Addr.Index)
	}

	// Build the instance info. More of this will be populated during eval
	info := &InstanceInfo{
		Id:   stateId,
		Type: n.Addr.Type,
	}

	// Build the resource for eval
	resource := &Resource{
		Name:       n.Addr.Name,
		Type:       n.Addr.Type,
		CountIndex: n.Addr.Index,
	}
	if resource.CountIndex < 0 {
		resource.CountIndex = 0
	}

	// Determine the dependencies for the state. We use some older
	// code for this that we've used for a long time.
	var stateDeps []string
	{
		oldN := &graphNodeExpandedResource{Resource: n.Config}
		stateDeps = oldN.StateDependencies()
	}

	// Declare a bunch of variables that are used for state during
	// evaluation. Most of this are written to by-address below.
	var provider ResourceProvider
	var diff *InstanceDiff
	var state *InstanceState
	var resourceConfig *ResourceConfig
	var err error
	var createNew bool
	var createBeforeDestroyEnabled bool

	return &EvalSequence{
		Nodes: []EvalNode{
			// Build the instance info
			&EvalInstanceInfo{
				Info: info,
			},

			// Get the saved diff for apply
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// We don't want to do any destroys
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					if diff == nil {
						return true, EvalEarlyExitError{}
					}

					if diff.GetDestroy() && diff.GetAttributesLen() == 0 {
						return true, EvalEarlyExitError{}
					}

					diff.SetDestroy(false)
					return true, nil
				},
				Then: EvalNoop{},
			},

			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					destroy := false
					if diff != nil {
						destroy = diff.GetDestroy() || diff.RequiresNew()
					}

					createBeforeDestroyEnabled =
						n.Config.Lifecycle.CreateBeforeDestroy &&
							destroy

					return createBeforeDestroyEnabled, nil
				},
				Then: &EvalDeposeState{
					Name: stateId,
				},
			},

			&EvalInterpolate{
				Config:   n.Config.RawConfig.Copy(),
				Resource: resource,
				Output:   &resourceConfig,
			},
			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			// Re-run validation to catch any errors we missed, e.g. type
			// mismatches on computed values.
			&EvalValidateResource{
				Provider:       &provider,
				Config:         &resourceConfig,
				ResourceName:   n.Config.Name,
				ResourceType:   n.Config.Type,
				ResourceMode:   n.Config.Mode,
				IgnoreWarnings: true,
			},
			&EvalDiff{
				Info:       info,
				Config:     &resourceConfig,
				Resource:   n.Config,
				Provider:   &provider,
				Diff:       &diff,
				State:      &state,
				OutputDiff: &diff,
			},

			// Get the saved diff
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// Compare the diffs
			&EvalCompareDiff{
				Info: info,
				One:  &diff,
				Two:  &diff,
			},

			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			&EvalApply{
				Info:      info,
				State:     &state,
				Diff:      &diff,
				Provider:  &provider,
				Output:    &state,
				Error:     &err,
				CreateNew: &createNew,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.Config.Provider,
				Dependencies: stateDeps,
				State:        &state,
			},
			&EvalApplyProvisioners{
				Info:           info,
				State:          &state,
				Resource:       n.Config,
				InterpResource: resource,
				CreateNew:      &createNew,
				Error:          &err,
			},
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					return createBeforeDestroyEnabled && err != nil, nil
				},
				Then: &EvalUndeposeState{
					Name:  stateId,
					State: &state,
				},
				Else: &EvalWriteState{
					Name:         stateId,
					ResourceType: n.Config.Type,
					Provider:     n.Config.Provider,
					Dependencies: stateDeps,
					State:        &state,
				},
			},

			// We clear the diff out here so that future nodes
			// don't see a diff that is already complete. There
			// is no longer a diff!
			&EvalWriteDiff{
				Name: stateId,
				Diff: nil,
			},

			&EvalApplyPost{
				Info:  info,
				State: &state,
				Error: &err,
			},
			&EvalUpdateStateHook{},
		},
	}
}
<commit_msg>terraform: properly compare bad diffs<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/config"
)

// NodeApplyableResource represents a resource that is "applyable":
// it is ready to be applied and is represented by a diff.
type NodeApplyableResource struct {
	Addr          *ResourceAddress // Addr is the address for this resource
	Config        *config.Resource // Config is the resource in the config
	ResourceState *ResourceState   // ResourceState is the ResourceState for this
}

func (n *NodeApplyableResource) Name() string {
	return n.Addr.String()
}

// GraphNodeSubPath
func (n *NodeApplyableResource) Path() []string {
	return n.Addr.Path
}

// GraphNodeReferenceable
func (n *NodeApplyableResource) ReferenceableName() []string {
	if n.Config == nil {
		return nil
	}

	return []string{n.Config.Id()}
}

// GraphNodeReferencer
func (n *NodeApplyableResource) References() []string {
	// Let's make this a little shorter so it is easier to reference
	c := n.Config
	if c == nil {
		return nil
	}

	// Grab all the references
	var result []string
	result = append(result, c.DependsOn...)
	result = append(result, ReferencesFromConfig(c.RawCount)...)
	result = append(result, ReferencesFromConfig(c.RawConfig)...)
	for _, p := range c.Provisioners {
		result = append(result, ReferencesFromConfig(p.ConnInfo)...)
		result = append(result, ReferencesFromConfig(p.RawConfig)...)
	}

	return result
}

// GraphNodeProviderConsumer
func (n *NodeApplyableResource) ProvidedBy() []string {
	// If we have a config we prefer that above all else
	if n.Config != nil {
		return []string{resourceProvider(n.Config.Type, n.Config.Provider)}
	}

	// If we have state, then we will use the provider from there
	if n.ResourceState != nil {
		return []string{n.ResourceState.Provider}
	}

	// Use our type
	return []string{resourceProvider(n.Addr.Type, "")}
}

// GraphNodeProvisionerConsumer
func (n *NodeApplyableResource) ProvisionedBy() []string {
	// If we have no configuration, then we have no provisioners
	if n.Config == nil {
		return nil
	}

	// Build the list of provisioners we need based on the configuration.
	// It is okay to have duplicates here.
	result := make([]string, len(n.Config.Provisioners))
	for i, p := range n.Config.Provisioners {
		result[i] = p.Type
	}

	return result
}

// GraphNodeAttachResourceState
func (n *NodeApplyableResource) ResourceAddr() *ResourceAddress {
	return n.Addr
}

// GraphNodeAttachResourceState
func (n *NodeApplyableResource) AttachResourceState(s *ResourceState) {
	n.ResourceState = s
}

// GraphNodeEvalable
func (n *NodeApplyableResource) EvalTree() EvalNode {
	// stateId is the ID to put into the state
	stateId := n.Addr.stateId()
	if n.Addr.Index > -1 {
		stateId = fmt.Sprintf("%s.%d", stateId, n.Addr.Index)
	}

	// Build the instance info. More of this will be populated during eval
	info := &InstanceInfo{
		Id:   stateId,
		Type: n.Addr.Type,
	}

	// Build the resource for eval
	resource := &Resource{
		Name:       n.Addr.Name,
		Type:       n.Addr.Type,
		CountIndex: n.Addr.Index,
	}
	if resource.CountIndex < 0 {
		resource.CountIndex = 0
	}

	// Determine the dependencies for the state. We use some older
	// code for this that we've used for a long time.
	var stateDeps []string
	{
		oldN := &graphNodeExpandedResource{Resource: n.Config}
		stateDeps = oldN.StateDependencies()
	}

	// Declare a bunch of variables that are used for state during
	// evaluation. Most of this are written to by-address below.
	var provider ResourceProvider
	var diff, diffApply *InstanceDiff
	var state *InstanceState
	var resourceConfig *ResourceConfig
	var err error
	var createNew bool
	var createBeforeDestroyEnabled bool

	return &EvalSequence{
		Nodes: []EvalNode{
			// Build the instance info
			&EvalInstanceInfo{
				Info: info,
			},

			// Get the saved diff for apply
			&EvalReadDiff{
				Name: stateId,
				Diff: &diffApply,
			},

			// We don't want to do any destroys
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					if diffApply == nil {
						return true, EvalEarlyExitError{}
					}

					if diffApply.GetDestroy() && diffApply.GetAttributesLen() == 0 {
						return true, EvalEarlyExitError{}
					}

					diffApply.SetDestroy(false)
					return true, nil
				},
				Then: EvalNoop{},
			},

			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					destroy := false
					if diffApply != nil {
						destroy = diffApply.GetDestroy() || diffApply.RequiresNew()
					}

					createBeforeDestroyEnabled =
						n.Config.Lifecycle.CreateBeforeDestroy &&
							destroy

					return createBeforeDestroyEnabled, nil
				},
				Then: &EvalDeposeState{
					Name: stateId,
				},
			},

			&EvalInterpolate{
				Config:   n.Config.RawConfig.Copy(),
				Resource: resource,
				Output:   &resourceConfig,
			},
			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			// Re-run validation to catch any errors we missed, e.g. type
			// mismatches on computed values.
			&EvalValidateResource{
				Provider:       &provider,
				Config:         &resourceConfig,
				ResourceName:   n.Config.Name,
				ResourceType:   n.Config.Type,
				ResourceMode:   n.Config.Mode,
				IgnoreWarnings: true,
			},
			&EvalDiff{
				Info:       info,
				Config:     &resourceConfig,
				Resource:   n.Config,
				Provider:   &provider,
				Diff:       &diffApply,
				State:      &state,
				OutputDiff: &diffApply,
			},

			// Get the saved diff
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// Compare the diffs
			&EvalCompareDiff{
				Info: info,
				One:  &diff,
				Two:  &diffApply,
			},

			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			&EvalApply{
				Info:      info,
				State:     &state,
				Diff:      &diffApply,
				Provider:  &provider,
				Output:    &state,
				Error:     &err,
				CreateNew: &createNew,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.Config.Provider,
				Dependencies: stateDeps,
				State:        &state,
			},
			&EvalApplyProvisioners{
				Info:           info,
				State:          &state,
				Resource:       n.Config,
				InterpResource: resource,
				CreateNew:      &createNew,
				Error:          &err,
			},
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					return createBeforeDestroyEnabled && err != nil, nil
				},
				Then: &EvalUndeposeState{
					Name:  stateId,
					State: &state,
				},
				Else: &EvalWriteState{
					Name:         stateId,
					ResourceType: n.Config.Type,
					Provider:     n.Config.Provider,
					Dependencies: stateDeps,
					State:        &state,
				},
			},

			// We clear the diff out here so that future nodes
			// don't see a diff that is already complete. There
			// is no longer a diff!
			&EvalWriteDiff{
				Name: stateId,
				Diff: nil,
			},

			&EvalApplyPost{
				Info:  info,
				State: &state,
				Error: &err,
			},
			&EvalUpdateStateHook{},
		},
	}
}
<|endoftext|>


<commit_before>// Notes: the program does not care the JSON doc
// is from who and where.
// This program will be expanded to:
// (1) include more JSON specs/formats.
// (2) one JSON doc has multiple JSON structures
// (3) Log output to separate file per process using Go’s logging framework
// (4) watchdog timer for spawned processes
// (5) recovery in case of process crashes
// (6) deal with repeated failures of startup
//     due to resoure constraints or other reasons
//     like network connection failures, etc.
// (7) a REST API support to stop/suspend/kill/cleanup
//     the spawned processes manually by the admin
// (8) more stuff to be added
//
// For now, the workflow is simply as follows:
// (a) read the JSON doc
// (b) parse the JSON
// (c) spawn processes
// (d) logging the output
// (e) starts watchdog timer if needed for each process
// (f) restart the process if it exits with error
// (g) ...

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"
)

type Watchdog struct {
	resets   chan int64
	timeouts chan bool
}

//create a new watchdog
func newWatchdog() *Watchdog {
	wd := &Watchdog{
		resets: make(chan int64, 100),

		timeouts: make(chan bool, 2),
	}
	//init. as false to coordinate events
	wd.timeouts <- false
	go wd.loop()
	return wd
}

//check if there are new resets
func (wd *Watchdog) pump(t0 int64) (t1 int64) {
	t1 = t0
	for {
		select {
		case t := <-wd.resets:
			if t > t0 {
				t1 = t
			}
		default:
			return
		}
	}
	panic("unreachable")
}

//if there is no reset, it is in idle state
//if there is reset, decrement the timer
//till it times out and set the timeout
//channel value to true
func (wd *Watchdog) loop() {
	var t0 int64
idle:
	t0 = <-wd.resets
	t0 = wd.pump(t0)
loop:
	t0 = t0 - time.Now().UnixNano()
	time.Sleep(time.Second * time.Duration(t0))
	now := time.Now().UnixNano()
	t0 = wd.pump(now)
	if t0 == now {
		wd.timeouts <- true
		goto idle
	}
	goto loop
}

//check errors
func check(e error) {
	if e != nil {
		fmt.Println("error:", e)
		panic(e)
	}
}

//this is to set the timeout in Seconds
//use absolute time value to avoid miscalculations
func (wd *Watchdog) reset(timeoutSecs int64) {
	wd.resets <- timeoutSecs + time.Now().UnixNano()
}

//check timeouts, note that we may need to add additional
//channel to coordinate
func check_timeouts(wd *Watchdog,
	logFile *os.File, pre int) {
	tmo := <-wd.timeouts
	if tmo == true {
		fmt.Fprintf(logFile, "Timed out!")
		// TO ADD: we may need to terminate
		// the spawned process forcefully here
		// if it is still running
	}
	//no need to continue the timer if process already ends
	if pre == 1 {
		wd.reset(0)
	}
}

//run procs, if it fails, we need to retry
//with configuratable delay time and maximum
//number of tries
func run_procs(cmd *exec.Cmd, wd *Watchdog,
	logFile *os.File, delaySecs int64,
	numRetries int, tmo int64) {
	//check the timeout with a goroutine
	go check_timeouts(wd, logFile, 0)
restart:
	//run the process and returns the output
	cmdOut, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
		//restart if it errors out
		numRetries--
		if numRetries >= 0 {
			//delay for a while
			time.Sleep(time.Second * time.Duration(delaySecs))
			//reset the watchdog timer
			wd.reset(tmo)
			//retry
			goto restart
		}
	}
	go check_timeouts(wd, logFile, 1)

	//deal with the output
	log.Println(string(cmdOut))
	fmt.Fprintf(logFile, string(cmdOut))
	//wait until operation on logFile is done, close it
	defer logFile.Close()
}

func main() {

	type Process struct {
		Name    string   `json:"name"`
		Path    string   `json:"path"`
		Cmdargs string   `json:"cmdargs"`
		Env     []string `json:"env"`
	}

	type Processes struct {
		Processes []Process `json:"processes"`
	}

	//read the JSON doc
	jsonBlob, err := ioutil.ReadFile("JSONdoc.txt")
	check(err)
	fmt.Print(string(jsonBlob))

	//JSON parser
	var processes Processes
	var logFile *os.File

	err = json.Unmarshal(jsonBlob, &processes)
	check(err)
	fmt.Printf("%+v", processes)
	fmt.Println()

	//watchdog for each process
	var tmo int64
	var delaySecs int64
	var numRetries int
	var wdg []*Watchdog
	wdg = make([]*Watchdog, len(processes.Processes))

	//spawning processes
	for i := 0; i <= len(processes.Processes)-1; i++ {
		fileName := "logfile"
		//log file name is based on date/time
		cmd := exec.Command("date")
		cmdOut, err := cmd.Output()
		check(err)
		fileName += string(cmdOut) + "_" + string(48+i)
		logFile, err = os.Create(fileName)
		check(err)

		//test only for the watchdogs
		wdg[i] = newWatchdog()
		//set the timeout as 5 Seconds in the test, it should change later
		//based on the application scenarios
		tmo = 5
		delaySecs = 1
		numRetries = 3
		wdg[i].reset(tmo)

        //spawn and run the processes
		cmd = exec.Command(processes.Processes[i].Name, processes.Processes[i].Cmdargs)
		run_procs(cmd, wdg[i], logFile, delaySecs, numRetries, tmo)
	}
}
<commit_msg>timeout checking and process completion<commit_after>// Notes: the program does not care the JSON doc
// is from who and where.
// This program will be expanded to:
// (1) include more JSON specs/formats.
// (2) one JSON doc has multiple JSON structures
// (3) Log output to separate file per process using Go’s logging framework
// (4) watchdog timer for spawned processes
// (5) recovery in case of process crashes
// (6) deal with repeated failures of startup
//     due to resoure constraints or other reasons
//     like network connection failures, etc.
// (7) a REST API support to stop/suspend/kill/cleanup
//     the spawned processes manually by the admin
// (8) more stuff to be added
//
// For now, the workflow is simply as follows:
// (a) read the JSON doc
// (b) parse the JSON
// (c) spawn processes
// (d) logging the output
// (e) starts watchdog timer if needed for each process
// (f) restart the process if it exits with error
// (g) ...

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"
)

type Watchdog struct {
	resets   chan int64
	timeouts chan bool
}

//create a new watchdog
func newWatchdog() *Watchdog {
	wd := &Watchdog{
		resets: make(chan int64, 100),

		timeouts: make(chan bool, 2),
	}
	//init. as false to coordinate events
	wd.timeouts <- false
	go wd.loop()
	return wd
}

//check if there are new resets
func (wd *Watchdog) pump(t0 int64) (t1 int64) {
	t1 = t0
	for {
		select {
		case t := <-wd.resets:
			if t > t0 {
				t1 = t
			}
		default:
			return
		}
	}
	panic("unreachable")
}

//if there is no reset, it is in idle state
//if there is reset, decrement the timer
//till it times out and set the timeout
//channel value to true
func (wd *Watchdog) loop() {
	var t0 int64
idle:
	t0 = <-wd.resets
	t0 = wd.pump(t0)
loop:
	t0 = t0 - time.Now().UnixNano()
	time.Sleep(time.Second * time.Duration(t0))
	now := time.Now().UnixNano()
	t0 = wd.pump(now)
	if t0 == now {
		wd.timeouts <- true
		goto idle
	}
	goto loop
}

//check errors
func check(e error) {
	if e != nil {
		fmt.Println("error:", e)
		panic(e)
	}
}

//this is to set the timeout in Seconds
//use absolute time value to avoid miscalculations
func (wd *Watchdog) reset(timeoutSecs int64) {
	wd.resets <- timeoutSecs + time.Now().UnixNano()
}

//check timeouts, note that we may need to add additional
//channel to coordinate
func check_timeouts(wd *Watchdog,
	logFile *os.File, pre int) {
	tmo := <-wd.timeouts
	if tmo == true {
		fmt.Fprintf(logFile, "Timed out!")
		// TO ADD: we may need to terminate
		// the spawned process forcefully here
		// if it is still running
	}
	//no need to continue the timer if process already ends
    // TO CHANGE: later on to use additional channel for
    // the coordination between timeout_checking and
    // process completion
	if pre == 1 {
		wd.reset(0)
	}
}

//run procs, if it fails, we need to retry
//with configuratable delay time and maximum
//number of tries
func run_procs(cmd *exec.Cmd, wd *Watchdog,
	logFile *os.File, delaySecs int64,
	numRetries int, tmo int64) {
	//check the timeout with a goroutine
	go check_timeouts(wd, logFile, 0)
restart:
	//run the process and returns the output
    // TO CHANGE: just call cmd.Start() without waiting to
    //complete and use additional channel to coordinate
    // with timeout_checking
	cmdOut, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
		//restart if it errors out
		numRetries--
		if numRetries >= 0 {
			//delay for a while
			time.Sleep(time.Second * time.Duration(delaySecs))
			//reset the watchdog timer
			wd.reset(tmo)
			//retry
			goto restart
		}
	}
	go check_timeouts(wd, logFile, 1)

	//deal with the output
	log.Println(string(cmdOut))
	fmt.Fprintf(logFile, string(cmdOut))
	//wait until operation on logFile is done, close it
	defer logFile.Close()
}

func main() {

	type Process struct {
		Name    string   `json:"name"`
		Path    string   `json:"path"`
		Cmdargs string   `json:"cmdargs"`
		Env     []string `json:"env"`
	}

	type Processes struct {
		Processes []Process `json:"processes"`
	}

	//read the JSON doc
	jsonBlob, err := ioutil.ReadFile("JSONdoc.txt")
	check(err)
	fmt.Print(string(jsonBlob))

	//JSON parser
	var processes Processes
	var logFile *os.File

	err = json.Unmarshal(jsonBlob, &processes)
	check(err)
	fmt.Printf("%+v", processes)
	fmt.Println()

	//watchdog for each process
	var tmo int64
	var delaySecs int64
	var numRetries int
	var wdg []*Watchdog
	wdg = make([]*Watchdog, len(processes.Processes))

	//spawning processes
	for i := 0; i <= len(processes.Processes)-1; i++ {
		fileName := "logfile"
		//log file name is based on date/time
		cmd := exec.Command("date")
		cmdOut, err := cmd.Output()
		check(err)
		fileName += string(cmdOut) + "_" + string(48+i)
		logFile, err = os.Create(fileName)
		check(err)

		//test only for the watchdogs
		wdg[i] = newWatchdog()
		//set the timeout as 5 Seconds in the test, it should change later
		//based on the application scenarios
		tmo = 5
		delaySecs = 1
		numRetries = 3
		wdg[i].reset(tmo)

        //spawn and run the processes
		cmd = exec.Command(processes.Processes[i].Name, processes.Processes[i].Cmdargs)
		run_procs(cmd, wdg[i], logFile, delaySecs, numRetries, tmo)
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"sync"

	"github.com/prometheus/client_golang/prometheus"
)

type maxGauge struct {
	mtx   sync.Mutex
	value float64
	prometheus.Gauge
}

func (m *maxGauge) Set(value float64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	if value > m.value {
		m.value = value
		m.Gauge.Set(value)
	}
}

func (m *maxGauge) Get() float64 {
	return m.value
}
<commit_msg>resolve race condition in maxGauge (#5647)<commit_after>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"sync"

	"github.com/prometheus/client_golang/prometheus"
)

type maxGauge struct {
	mtx   sync.Mutex
	value float64
	prometheus.Gauge
}

func (m *maxGauge) Set(value float64) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	if value > m.value {
		m.value = value
		m.Gauge.Set(value)
	}
}

func (m *maxGauge) Get() float64 {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	return m.value
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"
	"b00lduck/datalogger/display/framebuffer"
	"b00lduck/datalogger/display/touchscreen"
	"image"
	"image/jpeg"
	"strings"
	"image/png"
	"image/gif"
	"b00lduck/datalogger/display/gui"
	"b00lduck/datalogger/display/tools"
	"b00lduck/datalogger/display/i2c"
	"fmt"
)

var displayBuffer* image.RGBA

func drawDisplay(data []byte) {

	srcCount := 0
	targetCount := 0

	for srcCount < 240*320*4 {
		r := displayBuffer.Pix[srcCount]
		srcCount++;
		g := displayBuffer.Pix[srcCount]
		srcCount++;
		b := displayBuffer.Pix[srcCount]
		srcCount++;
		srcCount++;

		r8 := uint16(r >> 3)
		g8 := uint16(g >> 2)
		b8 := uint16(b >> 3)

		out := r8 << 11 + g8 << 5 + b8

		outl := uint8(out >> 8)
		outh := uint8(out & 0xff)

		data[targetCount] = outh
		targetCount++
		data[targetCount] = outl
		targetCount++
	}

}

func main() {

	fb := framebuffer.Framebuffer{}
	fb.Open(os.Args[1])
	defer fb.Close()
	data := fb.Data()

	ts := touchscreen.Touchscreen{}
	ts.Open(os.Args[2])
	defer ts.Close()
	go ts.Run()

	mm, err := i2c.New()
	tools.ErrorCheck(err)

	displayBuffer = image.NewRGBA(image.Rect(0, 0, 320, 240))

	cat := loadImage("cats-q-c-320-240-3.jpg")
	arrowUp := loadImage("arrow_up.gif")
	arrowDown := loadImage("arrow_down.gif")

	gui := gui.NewGui()

	gui.SetBackground(cat)
	for i := 0; i < 8; i ++ {
		gui.AddButton(arrowUp, 20 + i * 35, 60 )
		gui.AddButton(arrowDown, 20 + i * 35, 140 )
	}

	//go gui.Run(displayBuffer, &ts.Event)

	gui.Draw(displayBuffer)
	drawDisplay(data)

	for {
		//drawDisplay(data)
		time.Sleep(100 * time.Millisecond)
		mm.Write(2,1)
		fmt.Println(mm.Read(8))
		fmt.Println(mm.Read(9))
	}

}

func loadImage(filename string) image.Image {

	f, err := os.Open("images/" + filename)
	tools.ErrorCheck(err)

	var img image.Image = nil

	lowerFilename := strings.ToLower(filename)
	switch {
		case strings.HasSuffix(lowerFilename, ".jpg"):
			img, err = jpeg.Decode(f)
		case strings.HasSuffix(lowerFilename, ".png"):
			img, err = png.Decode(f)
		case strings.HasSuffix(lowerFilename, ".gif"):
			img, err = gif.Decode(f)
	}

	tools.ErrorCheck(err)

	return img
}
<commit_msg>DISPLAY: other magnetometer integration<commit_after>package main

import (
	"os"
	"time"
	"b00lduck/datalogger/display/framebuffer"
	"b00lduck/datalogger/display/touchscreen"
	"image"
	"image/jpeg"
	"strings"
	"image/png"
	"image/gif"
	"b00lduck/datalogger/display/gui"
	"b00lduck/datalogger/display/tools"
	"b00lduck/datalogger/display/i2c"
	"fmt"
)

var displayBuffer* image.RGBA

func drawDisplay(data []byte) {

	srcCount := 0
	targetCount := 0

	for srcCount < 240*320*4 {
		r := displayBuffer.Pix[srcCount]
		srcCount++;
		g := displayBuffer.Pix[srcCount]
		srcCount++;
		b := displayBuffer.Pix[srcCount]
		srcCount++;
		srcCount++;

		r8 := uint16(r >> 3)
		g8 := uint16(g >> 2)
		b8 := uint16(b >> 3)

		out := r8 << 11 + g8 << 5 + b8

		outl := uint8(out >> 8)
		outh := uint8(out & 0xff)

		data[targetCount] = outh
		targetCount++
		data[targetCount] = outl
		targetCount++
	}

}

func main() {

	fb := framebuffer.Framebuffer{}
	fb.Open(os.Args[1])
	defer fb.Close()
	data := fb.Data()

	ts := touchscreen.Touchscreen{}
	ts.Open(os.Args[2])
	defer ts.Close()
	go ts.Run()

	mm, err := i2c.New()
	tools.ErrorCheck(err)

	displayBuffer = image.NewRGBA(image.Rect(0, 0, 320, 240))

	cat := loadImage("cats-q-c-320-240-3.jpg")
	arrowUp := loadImage("arrow_up.gif")
	arrowDown := loadImage("arrow_down.gif")

	gui := gui.NewGui()

	gui.SetBackground(cat)
	for i := 0; i < 8; i ++ {
		gui.AddButton(arrowUp, 20 + i * 35, 60 )
		gui.AddButton(arrowDown, 20 + i * 35, 140 )
	}

	//go gui.Run(displayBuffer, &ts.Event)

	gui.Draw(displayBuffer)
	drawDisplay(data)

	for {
		//drawDisplay(data)
		time.Sleep(100 * time.Millisecond)

		x := uint16(mm.Read(8)) << 8 + mm.Read(9)
		y := uint16(mm.Read(10)) << 8 + mm.Read(11)
		z := uint16(mm.Read(12)) << 8 + mm.Read(13)

		fmt.Printf("X:%d Y:%d Z:%d", x, y, z)
	}

}

func loadImage(filename string) image.Image {

	f, err := os.Open("images/" + filename)
	tools.ErrorCheck(err)

	var img image.Image = nil

	lowerFilename := strings.ToLower(filename)
	switch {
		case strings.HasSuffix(lowerFilename, ".jpg"):
			img, err = jpeg.Decode(f)
		case strings.HasSuffix(lowerFilename, ".png"):
			img, err = png.Decode(f)
		case strings.HasSuffix(lowerFilename, ".gif"):
			img, err = gif.Decode(f)
	}

	tools.ErrorCheck(err)

	return img
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncutil

import (
	"flag"
	"sync"
)

var fCheckInvariants = flag.Bool("syncutil.check_invariants", false, "Crash when registered invariants are violated.")

// A reader/writer mutex like sync.RWMutex that additionally runs a check for
// registered invariants at times when invariants should hold, when enabled.
// This can aid debugging subtle code by crashing early as soon as something
// unexpected happens.
//
// Must be created with NewInvariantMutex. See that function for more details.
//
// A typical use looks like this:
//
//     type myStruct struct {
//       mu syncutil.InvariantMutex
//
//       // INVARIANT: nextGeneration == currentGeneration + 1
//       currentGeneration int // GUARDED_BY(mu)
//       nextGeneration    int // GUARDED_BY(mu)
//     }
//
//     // The constructor function for myStruct sets up the mutex to
//     // call the checkInvariants method.
//     func newMyStruct() *myStruct {
//       s := &myStruct{
//         currentGeneration: 1,
//         nextGeneration:    2,
//       }
//
//       s.mu = syncutil.NewInvariantMutex(s.checkInvariants)
//       return s
//     }
//
//     func (s *myStruct) checkInvariants() {
//       if s.nextGeneration != s.currentGeneration+1 {
//         panic(
//           fmt.Sprintf("%v != %v + 1", s.nextGeneration, s.currentGeneration))
//       }
//     }
//
//     // When the flag is set, invariants will be checked at entry to and exit
//     // from this function.
//     func (s *myStruct) setGeneration(n int) {
//       s.mu.Lock()
//       defer s.mu.Unlock()
//
//       s.currentGeneration = n
//       s.nextGeneration = n + 1
//     }
//
type InvariantMutex struct {
	mu    sync.RWMutex
	check func()
}

func (i *InvariantMutex) Lock() {
	i.mu.Lock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) Unlock() {
	i.checkIfEnabled()
	i.mu.Unlock()
}

func (i *InvariantMutex) RLock() {
	i.mu.RLock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) RUnlock() {
	i.checkIfEnabled()
	i.mu.RUnlock()
}

func (i *InvariantMutex) checkIfEnabled() {
	if *fCheckInvariants {
		i.check()
	}
}

// Create a reader/writer mutex which, when the flag -syncutil.check_invariants
// is set, will call the supplied function at moments when invariants protected
// by the mutex should hold (e.g. just after acquiring the lock). The function
// should crash if an invariant is violated. It should not have side effects,
// as there are no guarantees that it will run.
func NewInvariantMutex(check func()) InvariantMutex {
	if check == nil {
		panic("check must be non-nil.")
	}

	return InvariantMutex{
		check: check,
	}
}
<commit_msg>Check invariants when creating the mutex, too.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncutil

import (
	"flag"
	"sync"
)

var fCheckInvariants = flag.Bool("syncutil.check_invariants", false, "Crash when registered invariants are violated.")

// A reader/writer mutex like sync.RWMutex that additionally runs a check for
// registered invariants at times when invariants should hold, when enabled.
// This can aid debugging subtle code by crashing early as soon as something
// unexpected happens.
//
// Must be created with NewInvariantMutex. See that function for more details.
//
// A typical use looks like this:
//
//     type myStruct struct {
//       mu syncutil.InvariantMutex
//
//       // INVARIANT: nextGeneration == currentGeneration + 1
//       currentGeneration int // GUARDED_BY(mu)
//       nextGeneration    int // GUARDED_BY(mu)
//     }
//
//     // The constructor function for myStruct sets up the mutex to
//     // call the checkInvariants method.
//     func newMyStruct() *myStruct {
//       s := &myStruct{
//         currentGeneration: 1,
//         nextGeneration:    2,
//       }
//
//       s.mu = syncutil.NewInvariantMutex(s.checkInvariants)
//       return s
//     }
//
//     func (s *myStruct) checkInvariants() {
//       if s.nextGeneration != s.currentGeneration+1 {
//         panic(
//           fmt.Sprintf("%v != %v + 1", s.nextGeneration, s.currentGeneration))
//       }
//     }
//
//     // When the flag is set, invariants will be checked at entry to and exit
//     // from this function.
//     func (s *myStruct) setGeneration(n int) {
//       s.mu.Lock()
//       defer s.mu.Unlock()
//
//       s.currentGeneration = n
//       s.nextGeneration = n + 1
//     }
//
type InvariantMutex struct {
	mu    sync.RWMutex
	check func()
}

func (i *InvariantMutex) Lock() {
	i.mu.Lock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) Unlock() {
	i.checkIfEnabled()
	i.mu.Unlock()
}

func (i *InvariantMutex) RLock() {
	i.mu.RLock()
	i.checkIfEnabled()
}

func (i *InvariantMutex) RUnlock() {
	i.checkIfEnabled()
	i.mu.RUnlock()
}

func (i *InvariantMutex) checkIfEnabled() {
	if *fCheckInvariants {
		i.check()
	}
}

// Create a reader/writer mutex which, when the flag -syncutil.check_invariants
// is set, will call the supplied function at moments when invariants protected
// by the mutex should hold (e.g. just after acquiring the lock). The function
// should crash if an invariant is violated. It should not have side effects,
// as there are no guarantees that it will run.
//
// The invariants must hold at the time that NewInvariantMutex is called.
func NewInvariantMutex(check func()) InvariantMutex {
	if check == nil {
		panic("check must be non-nil.")
	}

	// Check now, if enabled.
	if *fCheckInvariants {
		check()
	}

	return InvariantMutex{
		check: check,
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package stats contains utilities that aid gathering statistics of
// system call mismatches in the verified programs.
package stats

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"sort"

	"github.com/google/syzkaller/prog"
)

// Stats encapsulates data for creating statistics related to the results
// of the verification process.
type Stats struct {
	// Calls stores statistics for all supported system calls.
	Calls           map[string]*CallStats
	TotalMismatches int
}

// CallStats stores information used to generate statistics for the
// system call.
type CallStats struct {
	// Name is the system call name.
	Name string
	// Mismatches stores the number of errno mismatches identifed in the
	// verified programs for this system call.
	Mismatches int
	// Occurrences is the number of times the system call appeared in a
	// verified program.
	Occurrences int
	// States stores the kernel return values that caused mismatches.
	States map[int]bool
}

// InitStats creates a stats object that will report verification
// statistics when an os.Interrupt occurs.
func InitStats(calls map[*prog.Syscall]bool, w io.Writer) *Stats {
	s := &Stats{Calls: make(map[string]*CallStats)}
	for c := range calls {
		s.Calls[c.Name] = &CallStats{Name: c.Name, States: make(map[int]bool)}
	}

	c := make(chan os.Signal)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		if s.TotalMismatches < 0 {
			fmt.Fprint(w, "No mismatches occurred until syz-verifier was stopped.")
			os.Exit(0)
		}
		s.ReportGlobalStats(w)
		os.Exit(0)
	}()

	return s
}

// ReportCallStats creates a report with the current statistics for call.
func (s *Stats) ReportCallStats(call string) string {
	cs, ok := s.Calls[call]
	if !ok {
		return ""
	}
	name, m, o := cs.Name, cs.Mismatches, cs.Occurrences
	data := fmt.Sprintf("statistics for %s:\n"+
		"\t↳ mismatches of %s / occurrences of %s: %d / %d (%0.2f %%)\n"+
		"\t↳ mismatches of %s / total number of mismatches: "+
		"%d / %d (%0.2f %%)\n"+
		"\t↳ %d distinct states identified: %v\n", name, name, name, m, o,
		getPercentage(m, o), name, m, s.TotalMismatches,
		getPercentage(m, s.TotalMismatches), len(cs.States), s.getOrderedStates(name))
	return data
}

func getPercentage(value, total int) float64 {
	return float64(value) / float64(total) * 100
}

// ReportGlobalStats creates a report with statistics about all the
// supported system calls for which errno mismatches were identified in
// the verified programs, shown in decreasing order.
func (s *Stats) ReportGlobalStats(w io.Writer) {
	tc := s.totalCallsExecuted()
	fmt.Fprintf(w, "total number of mismatches / total number of calls "+
		"executed: %d / %d (%0.2f %%)\n\n", s.TotalMismatches, tc, getPercentage(s.TotalMismatches, tc))
	cs := s.getOrderedStats()
	for _, c := range cs {
		fmt.Fprintf(w, "%s\n", s.ReportCallStats(c.Name))
	}
}

func (s *Stats) totalCallsExecuted() int {
	t := 0
	for _, cs := range s.Calls {
		t += cs.Occurrences
	}
	return t
}

func (s *Stats) getOrderedStats() []*CallStats {
	css := make([]*CallStats, 0)
	for _, cs := range s.Calls {
		if cs.Mismatches > 0 {
			css = append(css, cs)
		}
	}

	sort.Slice(css, func(i, j int) bool {
		return css[i].Mismatches > css[j].Mismatches
	})

	return css
}

func (s *Stats) getOrderedStates(call string) []int {
	states := s.Calls[call].States
	ss := make([]int, 0, len(states))
	for s := range states {
		ss = append(ss, s)
	}
	sort.Ints(ss)
	return ss
}
<commit_msg>syz-verifier/stats: order CallStats by percentage of mismatches out of call occurences<commit_after>// Copyright 2021 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package stats contains utilities that aid gathering statistics of
// system call mismatches in the verified programs.
package stats

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"sort"

	"github.com/google/syzkaller/prog"
)

// Stats encapsulates data for creating statistics related to the results
// of the verification process.
type Stats struct {
	// Calls stores statistics for all supported system calls.
	Calls           map[string]*CallStats
	TotalMismatches int
}

// CallStats stores information used to generate statistics for the
// system call.
type CallStats struct {
	// Name is the system call name.
	Name string
	// Mismatches stores the number of errno mismatches identifed in the
	// verified programs for this system call.
	Mismatches int
	// Occurrences is the number of times the system call appeared in a
	// verified program.
	Occurrences int
	// States stores the kernel return values that caused mismatches.
	States map[int]bool
}

// InitStats creates a stats object that will report verification
// statistics when an os.Interrupt occurs.
func InitStats(calls map[*prog.Syscall]bool, w io.Writer) *Stats {
	s := &Stats{Calls: make(map[string]*CallStats)}
	for c := range calls {
		s.Calls[c.Name] = &CallStats{Name: c.Name, States: make(map[int]bool)}
	}

	c := make(chan os.Signal)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		if s.TotalMismatches < 0 {
			fmt.Fprint(w, "No mismatches occurred until syz-verifier was stopped.")
			os.Exit(0)
		}
		s.ReportGlobalStats(w)
		os.Exit(0)
	}()

	return s
}

// ReportCallStats creates a report with the current statistics for call.
func (s *Stats) ReportCallStats(call string) string {
	cs, ok := s.Calls[call]
	if !ok {
		return ""
	}
	name, m, o := cs.Name, cs.Mismatches, cs.Occurrences
	data := fmt.Sprintf("statistics for %s:\n"+
		"\t↳ mismatches of %s / occurrences of %s: %d / %d (%0.2f %%)\n"+
		"\t↳ mismatches of %s / total number of mismatches: "+
		"%d / %d (%0.2f %%)\n"+
		"\t↳ %d distinct states identified: %v\n", name, name, name, m, o,
		getPercentage(m, o), name, m, s.TotalMismatches,
		getPercentage(m, s.TotalMismatches), len(cs.States), s.getOrderedStates(name))
	return data
}

func getPercentage(value, total int) float64 {
	return float64(value) / float64(total) * 100
}

// ReportGlobalStats creates a report with statistics about all the
// supported system calls for which errno mismatches were identified in
// the verified programs, shown in decreasing order.
func (s *Stats) ReportGlobalStats(w io.Writer) {
	tc := s.totalCallsExecuted()
	fmt.Fprintf(w, "total number of mismatches / total number of calls "+
		"executed: %d / %d (%0.2f %%)\n\n", s.TotalMismatches, tc, getPercentage(s.TotalMismatches, tc))
	cs := s.getOrderedStats()
	for _, c := range cs {
		fmt.Fprintf(w, "%s\n", s.ReportCallStats(c.Name))
	}
}

func (s *Stats) totalCallsExecuted() int {
	t := 0
	for _, cs := range s.Calls {
		t += cs.Occurrences
	}
	return t
}

func (s *Stats) getOrderedStats() []*CallStats {
	css := make([]*CallStats, 0)
	for _, cs := range s.Calls {
		if cs.Mismatches > 0 {
			css = append(css, cs)
		}
	}

	sort.Slice(css, func(i, j int) bool {
		return getPercentage(css[i].Mismatches, css[i].Occurrences) > getPercentage(css[j].Mismatches, css[j].Occurrences)
	})

	return css
}

func (s *Stats) getOrderedStates(call string) []int {
	states := s.Calls[call].States
	ss := make([]int, 0, len(states))
	for s := range states {
		ss = append(ss, s)
	}
	sort.Ints(ss)
	return ss
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package telemetry_test

import (
	"context"
	"runtime"
	"testing"

	"github.com/Jeffail/gabs/v2"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/session"
	"github.com/pingcap/tidb/store/mockstore"
	"github.com/pingcap/tidb/telemetry"
	"github.com/stretchr/testify/require"
	"go.etcd.io/etcd/integration"
)

func TestTrackingID(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	t.Parallel()

	s := newSuite(t)
	defer s.close()

	id, err := telemetry.GetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", id)

	id2, err := telemetry.ResetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.NotEqual(t, "", id2)

	id3, err := telemetry.GetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, id2, id3)
}

func TestPreview(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	s := newSuite(t)
	defer s.close()

	config.GetGlobalConfig().EnableTelemetry = false
	r, err := telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)

	trackingID, err := telemetry.ResetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)

	config.GetGlobalConfig().EnableTelemetry = true
	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)

	jsonParsed, err := gabs.ParseJSON([]byte(r))
	require.NoError(t, err)
	require.Equal(t, trackingID, jsonParsed.Path("trackingId").Data().(string))
	// Apple M1 doesn't contain cpuFlags
	if runtime.GOARCH != "arm" && runtime.GOOS != "darwin" {
		require.True(t, jsonParsed.ExistsP("hostExtra.cpuFlags"))
	}
	require.True(t, jsonParsed.ExistsP("hostExtra.os"))
	require.Len(t, jsonParsed.Path("instances").Children(), 2)
	require.Equal(t, "tidb", jsonParsed.Path("instances.0.instanceType").Data().(string))
	require.Equal(t, "tikv", jsonParsed.Path("instances.1.instanceType").Data().(string))
	require.True(t, jsonParsed.ExistsP("hardware"))

	_, err = s.se.Execute(context.Background(), "SET @@global.tidb_enable_telemetry = 0")
	require.NoError(t, err)
	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)

	_, err = s.se.Execute(context.Background(), "SET @@global.tidb_enable_telemetry = 1")
	config.GetGlobalConfig().EnableTelemetry = false
	require.NoError(t, err)

	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)
}

func TestReport(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	s := newSuite(t)
	defer s.close()

	config.GetGlobalConfig().EnableTelemetry = false
	err := telemetry.ReportUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)

	status, err := telemetry.GetTelemetryStatus(s.etcdCluster.RandClient())
	require.NoError(t, err)

	jsonParsed, err := gabs.ParseJSON([]byte(status))
	require.NoError(t, err)
	require.True(t, jsonParsed.Path("is_error").Data().(bool))
	require.Equal(t, "telemetry is disabled", jsonParsed.Path("error_msg").Data().(string))
	require.False(t, jsonParsed.Path("is_request_sent").Data().(bool))
}

type testSuite struct {
	store       kv.Storage
	dom         *domain.Domain
	etcdCluster *integration.ClusterV3
	se          session.Session
	close       func()
}

func newSuite(t *testing.T) *testSuite {
	suite := new(testSuite)

	store, err := mockstore.NewMockStore()
	require.NoError(t, err)
	suite.store = store

	session.SetSchemaLease(0)
	session.DisableStats4Test()

	dom, err := session.BootstrapSession(store)
	require.NoError(t, err)
	suite.dom = dom

	etcdCluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
	suite.etcdCluster = etcdCluster

	se, err := session.CreateSession4Test(store)
	require.NoError(t, err)
	suite.se = se

	suite.close = func() {
		suite.se.Close()
		suite.etcdCluster.Terminate(t)
		suite.dom.Close()
		err = suite.store.Close()
		require.NoError(t, err)
	}

	return suite
}
<commit_msg>telemetry: wrong skip test when apple intel or linux arm (#26826)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package telemetry_test

import (
	"context"
	"runtime"
	"testing"

	"github.com/Jeffail/gabs/v2"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/session"
	"github.com/pingcap/tidb/store/mockstore"
	"github.com/pingcap/tidb/telemetry"
	"github.com/stretchr/testify/require"
	"go.etcd.io/etcd/integration"
)

func TestTrackingID(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	t.Parallel()

	s := newSuite(t)
	defer s.close()

	id, err := telemetry.GetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", id)

	id2, err := telemetry.ResetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.NotEqual(t, "", id2)

	id3, err := telemetry.GetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, id2, id3)
}

func TestPreview(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	s := newSuite(t)
	defer s.close()

	config.GetGlobalConfig().EnableTelemetry = false
	r, err := telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)

	trackingID, err := telemetry.ResetTrackingID(s.etcdCluster.RandClient())
	require.NoError(t, err)

	config.GetGlobalConfig().EnableTelemetry = true
	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)

	jsonParsed, err := gabs.ParseJSON([]byte(r))
	require.NoError(t, err)
	require.Equal(t, trackingID, jsonParsed.Path("trackingId").Data().(string))
	// Apple M1 doesn't contain cpuFlags
	if !(runtime.GOARCH == "arm" && runtime.GOOS == "darwin") {
		require.True(t, jsonParsed.ExistsP("hostExtra.cpuFlags"))
	}
	require.True(t, jsonParsed.ExistsP("hostExtra.os"))
	require.Len(t, jsonParsed.Path("instances").Children(), 2)
	require.Equal(t, "tidb", jsonParsed.Path("instances.0.instanceType").Data().(string))
	require.Equal(t, "tikv", jsonParsed.Path("instances.1.instanceType").Data().(string))
	require.True(t, jsonParsed.ExistsP("hardware"))

	_, err = s.se.Execute(context.Background(), "SET @@global.tidb_enable_telemetry = 0")
	require.NoError(t, err)
	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)

	_, err = s.se.Execute(context.Background(), "SET @@global.tidb_enable_telemetry = 1")
	config.GetGlobalConfig().EnableTelemetry = false
	require.NoError(t, err)

	r, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)
	require.Equal(t, "", r)
}

func TestReport(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
	}

	s := newSuite(t)
	defer s.close()

	config.GetGlobalConfig().EnableTelemetry = false
	err := telemetry.ReportUsageData(s.se, s.etcdCluster.RandClient())
	require.NoError(t, err)

	status, err := telemetry.GetTelemetryStatus(s.etcdCluster.RandClient())
	require.NoError(t, err)

	jsonParsed, err := gabs.ParseJSON([]byte(status))
	require.NoError(t, err)
	require.True(t, jsonParsed.Path("is_error").Data().(bool))
	require.Equal(t, "telemetry is disabled", jsonParsed.Path("error_msg").Data().(string))
	require.False(t, jsonParsed.Path("is_request_sent").Data().(bool))
}

type testSuite struct {
	store       kv.Storage
	dom         *domain.Domain
	etcdCluster *integration.ClusterV3
	se          session.Session
	close       func()
}

func newSuite(t *testing.T) *testSuite {
	suite := new(testSuite)

	store, err := mockstore.NewMockStore()
	require.NoError(t, err)
	suite.store = store

	session.SetSchemaLease(0)
	session.DisableStats4Test()

	dom, err := session.BootstrapSession(store)
	require.NoError(t, err)
	suite.dom = dom

	etcdCluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
	suite.etcdCluster = etcdCluster

	se, err := session.CreateSession4Test(store)
	require.NoError(t, err)
	suite.se = se

	suite.close = func() {
		suite.se.Close()
		suite.etcdCluster.Terminate(t)
		suite.dom.Close()
		err = suite.store.Close()
		require.NoError(t, err)
	}

	return suite
}
<|endoftext|>