<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcutil"
	flags "github.com/btcsuite/go-flags"
)

const (
	defaultConfigFilename = "lnd.conf"
	defaultDataDirname    = "data"
	defaultLogLevel       = "info"
	defaultLogDirname     = "logs"
	defaultLogFilename    = "btcd.log"
	defaultRPCPort        = 10009
	defaultSPVMode        = false
	defaultPeerPort       = 10011
	defaultRPCHost        = "localhost:18334"
	defaultRPCUser        = "user"
	defaultRPCPass        = "passwd"
	defaultSPVHostAdr     = "localhost:18333"
)

var (
	lndHomeDir        = btcutil.AppDataDir("lnd", false)
	defaultConfigFile = filepath.Join(lndHomeDir, defaultConfigFilename)
	defaultDataDir    = filepath.Join(lndHomeDir, defaultDataDirname)
	defaultLogDir     = filepath.Join(lndHomeDir, defaultLogDirname)

	// activeNetParams is a pointer to the parameters specific to the
	// currently active bitcoin network.
	//activeNetParams = &chaincfg.SegNetParams
	activeNetParams = &chaincfg.TestNet3Params

	btcdHomeDir        = btcutil.AppDataDir("btcd", false)
	defaultRPCKeyFile  = filepath.Join(btcdHomeDir, "rpc.key")
	defaultRPCCertFile = filepath.Join(btcdHomeDir, "rpc.cert")
)

// config defines the configuratino options for lnd.
//
// See loadConfig for further details regarding the configuration
// loading+parsing process.
type config struct {
	ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"`

	ConfigFile string `long:"C" long:"configfile" description:"Path to configuration file"`
	DataDir    string `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
	LogDir     string `long:"logdir" description:"Directory to log output."`

	Listeners   []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 8333, testnet: 18333)"`
	ExternalIPs []string `long:"externalip" description:"Add an ip to the list of local addresses we claim to listen on to peers"`

	DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`

	PeerPort int    `long:"peerport" description:"The port to listen on for incoming p2p connections"`
	RPCPort  int    `long:"rpcport" description:"The port for the rpc server"`
	SPVMode  bool   `long:"spv" description:"assert to enter spv wallet mode"`
	RPCHost  string `long:"btcdhost" description:"The btcd rpc listening address. "`
	RPCUser  string `short:"u" long:"rpcuser" description:"Username for RPC connections"`
	RPCPass  string `short:"P" long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`

	RPCCert    string `long:"rpccert" description:"File containing btcd's certificate file"`
	RPCKey     string `long:"rpckey" description:"File containing btcd's certificate key"`
	SPVHostAdr string `long:"spvhostadr" description:"Address of full bitcoin node. It is used in SPV mode."`
	TestNet3   bool   `long:"testnet" description:"Use the test network"`
	SimNet     bool   `long:"simnet" description:"Use the simulation test network"`
	SegNet     bool   `long:"segnet" description:"Use the segragated witness test network"`
}

// loadConfig initializes and parses the config using a config file and command
// line options.
//
// The configuration proceeds as follows:
// 	1) Start with a default config with sane settings
// 	2) Pre-parse the command line to check for an alternative config file
// 	3) Load configuration file overwriting defaults with any specified options
// 	4) Parse CLI options and overwrite/add any specified options
func loadConfig() (*config, error) {
	defaultCfg := config{
		ConfigFile: defaultConfigFile,
		DataDir:    defaultDataDir,
		DebugLevel: defaultLogLevel,
		LogDir:     defaultLogDir,
		PeerPort:   defaultPeerPort,
		RPCPort:    defaultRPCPort,
		SPVMode:    defaultSPVMode,
		RPCHost:    defaultRPCHost,
		RPCUser:    defaultRPCUser,
		RPCPass:    defaultRPCPass,
		RPCCert:    defaultRPCCertFile,
		RPCKey:     defaultRPCKeyFile,
		SPVHostAdr: defaultSPVHostAdr,
	}

	// Pre-parse the command line options to pick up an alternative config
	// file.
	preCfg := defaultCfg
	if _, err := flags.Parse(&preCfg); err != nil {
		return nil, err
	}

	// Show the version and exit if the version flag was specified.
	appName := filepath.Base(os.Args[0])
	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
	usageMessage := fmt.Sprintf("Use %s -h to show usage", appName)
	if preCfg.ShowVersion {
		fmt.Println(appName, "version", version())
		os.Exit(0)
	}

	// Create the home directory if it doesn't already exist.
	funcName := "loadConfig"
	if err := os.MkdirAll(lndHomeDir, 0700); err != nil {
		// Show a nicer error message if it's because a symlink is
		// linked to a directory that does not exist (probably because
		// it's not mounted).
		if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
			if link, lerr := os.Readlink(e.Path); lerr == nil {
				str := "is symlink %s -> %s mounted?"
				err = fmt.Errorf(str, e.Path, link)
			}
		}

		str := "%s: Failed to create home directory: %v"
		err := fmt.Errorf(str, funcName, err)
		fmt.Fprintln(os.Stderr, err)
		return nil, err
	}

	// Next, load any additional configuration options from the file.
	cfg := defaultCfg
	if err := flags.IniParse(preCfg.ConfigFile, &cfg); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// Finally, parse the remaining command line options again to ensure
	// they take precedence.
	if _, err := flags.Parse(&cfg); err != nil {
		return nil, err
	}

	// Multiple networks can't be selected simultaneously.
	// Count number of network flags passed; assign active network params
	// while we're at it
	numNets := 0
	if cfg.TestNet3 {
		numNets++
		activeNetParams = &chaincfg.TestNet3Params
	}
	if cfg.SegNet {
		numNets++
		activeNetParams = &chaincfg.SegNetParams
	}
	if cfg.SimNet {
		numNets++
		activeNetParams = &chaincfg.SimNetParams
	}
	if numNets > 1 {
		str := "%s: The testnet, segnet, and simnet params can't be " +
			"used together -- choose one of the three"
		err := fmt.Errorf(str, funcName)
		return nil, err
	}

	// Append the network type to the data directory so it is "namespaced"
	// per network.  In addition to the block database, there are other
	// pieces of data that are saved to disk such as address manager state.
	// All data is specific to a network, so namespacing the data directory
	// means each individual piece of serialized data does not have to
	// worry about changing names per network and such.
	cfg.DataDir = cleanAndExpandPath(cfg.DataDir)
	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)

	// Append the network type to the log directory so it is "namespaced"
	// per network in the same fashion as the data directory.
	cfg.LogDir = cleanAndExpandPath(cfg.LogDir)
	cfg.LogDir = filepath.Join(cfg.LogDir, activeNetParams.Name)

	// Initialize logging at the default logging level.
	initSeelogLogger(filepath.Join(cfg.LogDir, defaultLogFilename))
	setLogLevels(defaultLogLevel)

	// Parse, validate, and set debug log level(s).
	if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil {
		err := fmt.Errorf("%s: %v", funcName, err.Error())
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, usageMessage)
		return nil, err
	}

	// TODO(roasbeef): logging
	return &cfg, nil
}

// cleanAndExpandPath expands environment variables and leading ~ in the
// passed path, cleans the result, and returns it.
func cleanAndExpandPath(path string) string {
	// Expand initial ~ to OS specific home directory.
	if strings.HasPrefix(path, "~") {
		homeDir := filepath.Dir(lndHomeDir)
		path = strings.Replace(path, "~", homeDir, 1)
	}

	// NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%,
	// but they variables can still be expanded via POSIX-style $VARIABLE.
	return filepath.Clean(os.ExpandEnv(path))
}

// parseAndSetDebugLevels attempts to parse the specified debug level and set
// the levels accordingly.  An appropriate error is returned if anything is
// invalid.
func parseAndSetDebugLevels(debugLevel string) error {
	// When the specified string doesn't have any delimters, treat it as
	// the log level for all subsystems.
	if !strings.Contains(debugLevel, ",") && !strings.Contains(debugLevel, "=") {
		// Validate debug log level.
		if !validLogLevel(debugLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, debugLevel)
		}

		// Change the logging level for all subsystems.
		setLogLevels(debugLevel)

		return nil
	}

	// Split the specified string into subsystem/level pairs while detecting
	// issues and update the log levels accordingly.
	for _, logLevelPair := range strings.Split(debugLevel, ",") {
		if !strings.Contains(logLevelPair, "=") {
			str := "The specified debug level contains an invalid " +
				"subsystem/level pair [%v]"
			return fmt.Errorf(str, logLevelPair)
		}

		// Extract the specified subsystem and log level.
		fields := strings.Split(logLevelPair, "=")
		subsysID, logLevel := fields[0], fields[1]

		// Validate subsystem.
		if _, exists := subsystemLoggers[subsysID]; !exists {
			str := "The specified subsystem [%v] is invalid -- " +
				"supported subsytems %v"
			return fmt.Errorf(str, subsysID, supportedSubsystems())
		}

		// Validate log level.
		if !validLogLevel(logLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, logLevel)
		}

		setLogLevel(subsysID, logLevel)
	}

	return nil
}

// validLogLevel returns whether or not logLevel is a valid debug log level.
func validLogLevel(logLevel string) bool {
	switch logLevel {
	case "trace":
		fallthrough
	case "debug":
		fallthrough
	case "info":
		fallthrough
	case "warn":
		fallthrough
	case "error":
		fallthrough
	case "critical":
		return true
	}
	return false
}

// supportedSubsystems returns a sorted slice of the supported subsystems for
// logging purposes.
func supportedSubsystems() []string {
	// Convert the subsystemLoggers map keys to a slice.
	subsystems := make([]string, 0, len(subsystemLoggers))
	for subsysID := range subsystemLoggers {
		subsystems = append(subsystems, subsysID)
	}

	// Sort the subsystems for stable display.
	sort.Strings(subsystems)
	return subsystems
}
<commit_msg>default net params to segnet4<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcutil"
	flags "github.com/btcsuite/go-flags"
)

const (
	defaultConfigFilename = "lnd.conf"
	defaultDataDirname    = "data"
	defaultLogLevel       = "info"
	defaultLogDirname     = "logs"
	defaultLogFilename    = "btcd.log"
	defaultRPCPort        = 10009
	defaultSPVMode        = false
	defaultPeerPort       = 10011
	defaultRPCHost        = "localhost:18334"
	defaultRPCUser        = "user"
	defaultRPCPass        = "passwd"
	defaultSPVHostAdr     = "localhost:18333"
)

var (
	lndHomeDir        = btcutil.AppDataDir("lnd", false)
	defaultConfigFile = filepath.Join(lndHomeDir, defaultConfigFilename)
	defaultDataDir    = filepath.Join(lndHomeDir, defaultDataDirname)
	defaultLogDir     = filepath.Join(lndHomeDir, defaultLogDirname)

	// activeNetParams is a pointer to the parameters specific to the
	// currently active bitcoin network.
	activeNetParams = &chaincfg.SegNet4Params

	btcdHomeDir        = btcutil.AppDataDir("btcd", false)
	defaultRPCKeyFile  = filepath.Join(btcdHomeDir, "rpc.key")
	defaultRPCCertFile = filepath.Join(btcdHomeDir, "rpc.cert")
)

// config defines the configuratino options for lnd.
//
// See loadConfig for further details regarding the configuration
// loading+parsing process.
type config struct {
	ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"`

	ConfigFile string `long:"C" long:"configfile" description:"Path to configuration file"`
	DataDir    string `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
	LogDir     string `long:"logdir" description:"Directory to log output."`

	Listeners   []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 8333, testnet: 18333)"`
	ExternalIPs []string `long:"externalip" description:"Add an ip to the list of local addresses we claim to listen on to peers"`

	DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`

	PeerPort int    `long:"peerport" description:"The port to listen on for incoming p2p connections"`
	RPCPort  int    `long:"rpcport" description:"The port for the rpc server"`
	SPVMode  bool   `long:"spv" description:"assert to enter spv wallet mode"`
	RPCHost  string `long:"btcdhost" description:"The btcd rpc listening address. "`
	RPCUser  string `short:"u" long:"rpcuser" description:"Username for RPC connections"`
	RPCPass  string `short:"P" long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`

	RPCCert    string `long:"rpccert" description:"File containing btcd's certificate file"`
	RPCKey     string `long:"rpckey" description:"File containing btcd's certificate key"`
	SPVHostAdr string `long:"spvhostadr" description:"Address of full bitcoin node. It is used in SPV mode."`
	TestNet3   bool   `long:"testnet" description:"Use the test network"`
	SimNet     bool   `long:"simnet" description:"Use the simulation test network"`
	SegNet     bool   `long:"segnet" description:"Use the segragated witness test network"`
}

// loadConfig initializes and parses the config using a config file and command
// line options.
//
// The configuration proceeds as follows:
// 	1) Start with a default config with sane settings
// 	2) Pre-parse the command line to check for an alternative config file
// 	3) Load configuration file overwriting defaults with any specified options
// 	4) Parse CLI options and overwrite/add any specified options
func loadConfig() (*config, error) {
	defaultCfg := config{
		ConfigFile: defaultConfigFile,
		DataDir:    defaultDataDir,
		DebugLevel: defaultLogLevel,
		LogDir:     defaultLogDir,
		PeerPort:   defaultPeerPort,
		RPCPort:    defaultRPCPort,
		SPVMode:    defaultSPVMode,
		RPCHost:    defaultRPCHost,
		RPCUser:    defaultRPCUser,
		RPCPass:    defaultRPCPass,
		RPCCert:    defaultRPCCertFile,
		RPCKey:     defaultRPCKeyFile,
		SPVHostAdr: defaultSPVHostAdr,
	}

	// Pre-parse the command line options to pick up an alternative config
	// file.
	preCfg := defaultCfg
	if _, err := flags.Parse(&preCfg); err != nil {
		return nil, err
	}

	// Show the version and exit if the version flag was specified.
	appName := filepath.Base(os.Args[0])
	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
	usageMessage := fmt.Sprintf("Use %s -h to show usage", appName)
	if preCfg.ShowVersion {
		fmt.Println(appName, "version", version())
		os.Exit(0)
	}

	// Create the home directory if it doesn't already exist.
	funcName := "loadConfig"
	if err := os.MkdirAll(lndHomeDir, 0700); err != nil {
		// Show a nicer error message if it's because a symlink is
		// linked to a directory that does not exist (probably because
		// it's not mounted).
		if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
			if link, lerr := os.Readlink(e.Path); lerr == nil {
				str := "is symlink %s -> %s mounted?"
				err = fmt.Errorf(str, e.Path, link)
			}
		}

		str := "%s: Failed to create home directory: %v"
		err := fmt.Errorf(str, funcName, err)
		fmt.Fprintln(os.Stderr, err)
		return nil, err
	}

	// Next, load any additional configuration options from the file.
	cfg := defaultCfg
	if err := flags.IniParse(preCfg.ConfigFile, &cfg); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// Finally, parse the remaining command line options again to ensure
	// they take precedence.
	if _, err := flags.Parse(&cfg); err != nil {
		return nil, err
	}

	// Multiple networks can't be selected simultaneously.
	// Count number of network flags passed; assign active network params
	// while we're at it
	numNets := 0
	if cfg.TestNet3 {
		numNets++
		activeNetParams = &chaincfg.TestNet3Params
	}
	if cfg.SegNet {
		numNets++
		activeNetParams = &chaincfg.SegNet4Params
	}
	if cfg.SimNet {
		numNets++
		activeNetParams = &chaincfg.SimNetParams
	}
	if numNets > 1 {
		str := "%s: The testnet, segnet, and simnet params can't be " +
			"used together -- choose one of the three"
		err := fmt.Errorf(str, funcName)
		return nil, err
	}

	// Append the network type to the data directory so it is "namespaced"
	// per network.  In addition to the block database, there are other
	// pieces of data that are saved to disk such as address manager state.
	// All data is specific to a network, so namespacing the data directory
	// means each individual piece of serialized data does not have to
	// worry about changing names per network and such.
	cfg.DataDir = cleanAndExpandPath(cfg.DataDir)
	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)

	// Append the network type to the log directory so it is "namespaced"
	// per network in the same fashion as the data directory.
	cfg.LogDir = cleanAndExpandPath(cfg.LogDir)
	cfg.LogDir = filepath.Join(cfg.LogDir, activeNetParams.Name)

	// Initialize logging at the default logging level.
	initSeelogLogger(filepath.Join(cfg.LogDir, defaultLogFilename))
	setLogLevels(defaultLogLevel)

	// Parse, validate, and set debug log level(s).
	if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil {
		err := fmt.Errorf("%s: %v", funcName, err.Error())
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, usageMessage)
		return nil, err
	}

	// TODO(roasbeef): logging
	return &cfg, nil
}

// cleanAndExpandPath expands environment variables and leading ~ in the
// passed path, cleans the result, and returns it.
func cleanAndExpandPath(path string) string {
	// Expand initial ~ to OS specific home directory.
	if strings.HasPrefix(path, "~") {
		homeDir := filepath.Dir(lndHomeDir)
		path = strings.Replace(path, "~", homeDir, 1)
	}

	// NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%,
	// but they variables can still be expanded via POSIX-style $VARIABLE.
	return filepath.Clean(os.ExpandEnv(path))
}

// parseAndSetDebugLevels attempts to parse the specified debug level and set
// the levels accordingly.  An appropriate error is returned if anything is
// invalid.
func parseAndSetDebugLevels(debugLevel string) error {
	// When the specified string doesn't have any delimters, treat it as
	// the log level for all subsystems.
	if !strings.Contains(debugLevel, ",") && !strings.Contains(debugLevel, "=") {
		// Validate debug log level.
		if !validLogLevel(debugLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, debugLevel)
		}

		// Change the logging level for all subsystems.
		setLogLevels(debugLevel)

		return nil
	}

	// Split the specified string into subsystem/level pairs while detecting
	// issues and update the log levels accordingly.
	for _, logLevelPair := range strings.Split(debugLevel, ",") {
		if !strings.Contains(logLevelPair, "=") {
			str := "The specified debug level contains an invalid " +
				"subsystem/level pair [%v]"
			return fmt.Errorf(str, logLevelPair)
		}

		// Extract the specified subsystem and log level.
		fields := strings.Split(logLevelPair, "=")
		subsysID, logLevel := fields[0], fields[1]

		// Validate subsystem.
		if _, exists := subsystemLoggers[subsysID]; !exists {
			str := "The specified subsystem [%v] is invalid -- " +
				"supported subsytems %v"
			return fmt.Errorf(str, subsysID, supportedSubsystems())
		}

		// Validate log level.
		if !validLogLevel(logLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, logLevel)
		}

		setLogLevel(subsysID, logLevel)
	}

	return nil
}

// validLogLevel returns whether or not logLevel is a valid debug log level.
func validLogLevel(logLevel string) bool {
	switch logLevel {
	case "trace":
		fallthrough
	case "debug":
		fallthrough
	case "info":
		fallthrough
	case "warn":
		fallthrough
	case "error":
		fallthrough
	case "critical":
		return true
	}
	return false
}

// supportedSubsystems returns a sorted slice of the supported subsystems for
// logging purposes.
func supportedSubsystems() []string {
	// Convert the subsystemLoggers map keys to a slice.
	subsystems := make([]string, 0, len(subsystemLoggers))
	for subsysID := range subsystemLoggers {
		subsystems = append(subsystems, subsysID)
	}

	// Sort the subsystems for stable display.
	sort.Strings(subsystems)
	return subsystems
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 The Jaeger Authors.
// Copyright (c) 2017 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servers

import (
	"context"
	"io"
	"sync"
	"testing"
	"time"

	athrift "github.com/apache/thrift/lib/go/thrift"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/jaeger-lib/metrics/metricstest"
	"go.uber.org/atomic"

	"github.com/jaegertracing/jaeger/cmd/agent/app/customtransport"
	"github.com/jaegertracing/jaeger/cmd/agent/app/servers/thriftudp"
	"github.com/jaegertracing/jaeger/cmd/agent/app/testutils"
	"github.com/jaegertracing/jaeger/thrift-gen/agent"
	"github.com/jaegertracing/jaeger/thrift-gen/zipkincore"
)

func TestTBufferedServer_SendReceive(t *testing.T) {
	metricsFactory := metricstest.NewFactory(0)

	transport, err := thriftudp.NewTUDPServerTransport("127.0.0.1:0")
	require.NoError(t, err)

	maxPacketSize := 65000
	server, err := NewTBufferedServer(transport, 100, maxPacketSize, metricsFactory)
	require.NoError(t, err)
	go server.Serve()
	defer server.Stop()

	hostPort := transport.Addr().String()
	client, clientCloser, err := testutils.NewZipkinThriftUDPClient(hostPort)
	require.NoError(t, err)
	defer clientCloser.Close()

	span := zipkincore.NewSpan()
	span.Name = "span1"

	for i := 0; i < 1000; i++ {
		err := client.EmitZipkinBatch(context.Background(), []*zipkincore.Span{span})
		require.NoError(t, err)

		select {
		case readBuf := <-server.DataChan():
			assert.NotEqual(t, 0, len(readBuf.GetBytes()))

			inMemReporter := testutils.NewInMemoryReporter()
			protoFact := athrift.NewTCompactProtocolFactory()
			trans := &customtransport.TBufferedReadTransport{}
			protocol := protoFact.GetProtocol(trans)

			_, err = protocol.Transport().Write(readBuf.GetBytes())
			require.NoError(t, err)

			server.DataRecd(readBuf) // return to pool

			handler := agent.NewAgentProcessor(inMemReporter)
			_, err = handler.Process(context.Background(), protocol, protocol)
			require.NoError(t, err)

			require.Len(t, inMemReporter.ZipkinSpans(), 1)
			assert.Equal(t, "span1", inMemReporter.ZipkinSpans()[0].Name)

			return // exit test on successful receipt
		default:
			time.Sleep(10 * time.Millisecond)
		}
	}
	t.Fatal("server did not receive packets")
}

// The fakeTransport allows the server to read two packets, one filled with 1's, another with 2's,
// then returns an error, and then blocks on the semaphore. The semaphore is only released when
// the test is exiting.
type fakeTransport struct {
	packet atomic.Int64
	wg     sync.WaitGroup
}

func (t *fakeTransport) Read(p []byte) (n int, err error) {
	packet := t.packet.Inc()
	if packet > 2 {
		if packet > 3 {
			// return error once when packet==3, otherwise block
			t.wg.Wait()
		}
		return 0, io.EOF
	}
	for i := range p {
		p[i] = byte(packet)
	}
	return len(p), nil
}

func (t *fakeTransport) Close() error {
	return nil
}

func TestTBufferedServer_Metrics(t *testing.T) {
	metricsFactory := metricstest.NewFactory(0)

	transport := new(fakeTransport)
	transport.wg.Add(1)
	defer transport.wg.Done()

	maxPacketSize := 65000
	server, err := NewTBufferedServer(transport, 1, maxPacketSize, metricsFactory)
	require.NoError(t, err)
	go server.Serve()
	defer server.Stop()

	// The fakeTransport will allow the server to read exactly two packets and one error.
	// Since we use the server with queue size == 1, the first packet will be
	// sent to channel, and the second one dropped.

	packetDropped := false
	for i := 0; i < 5000; i++ {
		c, _ := metricsFactory.Snapshot()
		if c["thrift.udp.server.packets.dropped"] == 1 {
			packetDropped = true
			break
		}
		time.Sleep(time.Millisecond)
	}
	require.True(t, packetDropped, "packetDropped")

	var readBuf *ReadBuf
	select {
	case readBuf = <-server.DataChan():
		b := readBuf.GetBytes()
		assert.Len(t, b, 65000)
		assert.EqualValues(t, 1, b[0], "first packet must be all 0x01's")
	default:
		t.Fatal("expecting a packet in the channel")
	}

	metricsFactory.AssertCounterMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packets.processed", Value: 1},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packets.dropped", Value: 1},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.read.errors", Value: 1},
	)
	metricsFactory.AssertGaugeMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packet_size", Value: 65000},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.queue_size", Value: 1},
	)

	server.DataRecd(readBuf)
	metricsFactory.AssertGaugeMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.queue_size", Value: 0},
	)
}
<commit_msg>Fix flaky tbuffered server test (#2635)<commit_after>// Copyright (c) 2019 The Jaeger Authors.
// Copyright (c) 2017 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servers

import (
	"context"
	"io"
	"sync"
	"testing"
	"time"

	athrift "github.com/apache/thrift/lib/go/thrift"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/jaeger-lib/metrics/metricstest"
	"go.uber.org/atomic"

	"github.com/jaegertracing/jaeger/cmd/agent/app/customtransport"
	"github.com/jaegertracing/jaeger/cmd/agent/app/servers/thriftudp"
	"github.com/jaegertracing/jaeger/cmd/agent/app/testutils"
	"github.com/jaegertracing/jaeger/thrift-gen/agent"
	"github.com/jaegertracing/jaeger/thrift-gen/zipkincore"
)

func TestTBufferedServer_SendReceive(t *testing.T) {
	metricsFactory := metricstest.NewFactory(0)

	transport, err := thriftudp.NewTUDPServerTransport("127.0.0.1:0")
	require.NoError(t, err)

	maxPacketSize := 65000
	server, err := NewTBufferedServer(transport, 100, maxPacketSize, metricsFactory)
	require.NoError(t, err)
	go server.Serve()
	defer server.Stop()

	hostPort := transport.Addr().String()
	client, clientCloser, err := testutils.NewZipkinThriftUDPClient(hostPort)
	require.NoError(t, err)
	defer clientCloser.Close()

	span := zipkincore.NewSpan()
	span.Name = "span1"

	for i := 0; i < 1000; i++ {
		err := client.EmitZipkinBatch(context.Background(), []*zipkincore.Span{span})
		require.NoError(t, err)

		select {
		case readBuf := <-server.DataChan():
			assert.NotEqual(t, 0, len(readBuf.GetBytes()))

			inMemReporter := testutils.NewInMemoryReporter()
			protoFact := athrift.NewTCompactProtocolFactory()
			trans := &customtransport.TBufferedReadTransport{}
			protocol := protoFact.GetProtocol(trans)

			_, err = protocol.Transport().Write(readBuf.GetBytes())
			require.NoError(t, err)

			server.DataRecd(readBuf) // return to pool

			handler := agent.NewAgentProcessor(inMemReporter)
			_, err = handler.Process(context.Background(), protocol, protocol)
			require.NoError(t, err)

			require.Len(t, inMemReporter.ZipkinSpans(), 1)
			assert.Equal(t, "span1", inMemReporter.ZipkinSpans()[0].Name)

			return // exit test on successful receipt
		default:
			time.Sleep(10 * time.Millisecond)
		}
	}
	t.Fatal("server did not receive packets")
}

// The fakeTransport allows the server to read two packets, one filled with 1's, another with 2's,
// then returns an error, and then blocks on the semaphore. The semaphore is only released when
// the test is exiting.
type fakeTransport struct {
	packet atomic.Int64
	wg     sync.WaitGroup
}

// Read simulates three packets received, then blocks until semaphore is released at the end of the test.
// First packet is returned as normal.
// Second packet is simulated as error.
// Third packet is returned as normal, but will be dropped as overflow by the server whose queue size = 1.
func (t *fakeTransport) Read(p []byte) (n int, err error) {
	packet := t.packet.Inc()
	if packet == 2 {
		// return some error packet, followed by valid one
		return 0, io.ErrNoProgress
	}
	if packet > 3 {
		// block after 3 packets until the server is shutdown and semaphore released
		t.wg.Wait()
		return 0, io.EOF
	}
	for i := range p {
		p[i] = byte(packet)
	}
	return len(p), nil
}

func (t *fakeTransport) Close() error {
	return nil
}

func TestTBufferedServer_Metrics(t *testing.T) {
	metricsFactory := metricstest.NewFactory(0)

	transport := new(fakeTransport)
	transport.wg.Add(1)
	defer transport.wg.Done()

	maxPacketSize := 65000
	server, err := NewTBufferedServer(transport, 1, maxPacketSize, metricsFactory)
	require.NoError(t, err)
	go server.Serve()
	defer server.Stop()

	// The fakeTransport will allow the server to read exactly two packets and one error in between.
	// Since we use the server with queue size == 1, the first packet will be
	// sent to channel, the error will increment the metric, and the second valid packet dropped.

	packetDropped := false
	for i := 0; i < 5000; i++ {
		c, _ := metricsFactory.Snapshot()
		if c["thrift.udp.server.packets.dropped"] == 1 {
			packetDropped = true
			break
		}
		time.Sleep(time.Millisecond)
	}
	require.True(t, packetDropped, "packetDropped")

	var readBuf *ReadBuf
	select {
	case readBuf = <-server.DataChan():
		b := readBuf.GetBytes()
		assert.Len(t, b, 65000)
		assert.EqualValues(t, 1, b[0], "first packet must be all 0x01's")
	default:
		t.Fatal("expecting a packet in the channel")
	}

	metricsFactory.AssertCounterMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packets.processed", Value: 1},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packets.dropped", Value: 1},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.read.errors", Value: 1},
	)
	metricsFactory.AssertGaugeMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.packet_size", Value: 65000},
		metricstest.ExpectedMetric{Name: "thrift.udp.server.queue_size", Value: 1},
	)

	server.DataRecd(readBuf)
	metricsFactory.AssertGaugeMetrics(t,
		metricstest.ExpectedMetric{Name: "thrift.udp.server.queue_size", Value: 0},
	)
}
<|endoftext|>


<commit_before>package golang

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/graph"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/toolchain/golang/gog"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	grapher2.Register(&Package{}, grapher2.DockerGrapher{defaultGoVersion})
}

func (v *goVersion) BuildGrapher(dir string, unit unit.SourceUnit, c *config.Repository) (*container.Command, error) {
	gogBinPath := filepath.Join(os.Getenv("GOBIN"), "gog")

	pkg := unit.(*Package)

	cont, err := v.containerForRepo(dir, unit, c)
	if err != nil {
		return nil, err
	}

	// Install VCS tools in Docker container.
	cont.Dockerfile = append(cont.Dockerfile, []byte("RUN apt-get -yq install git mercurial bzr subversion\n")...)

	cont.AddFiles = append(cont.AddFiles, [2]string{gogBinPath, "/usr/local/bin/gog"})
	cont.Cmd = []string{"bash", "-c", fmt.Sprintf("go get -v -t %s; gog %s", pkg.ImportPath, pkg.ImportPath)}

	cmd := container.Command{
		Container: *cont,
		Transform: func(in []byte) ([]byte, error) {
			var o gog.Output
			err := json.Unmarshal(in, &o)
			if err != nil {
				return nil, err
			}

			o2 := grapher2.Output{
				Symbols: make([]*graph.Symbol, len(o.Symbols)),
				Refs:    make([]*graph.Ref, len(o.Refs)),
				Docs:    make([]*graph.Doc, len(o.Docs)),
			}

			for i, gs := range o.Symbols {
				o2.Symbols[i], err = v.convertGoSymbol(gs, c)
				if err != nil {
					return nil, err
				}
			}
			for i, gr := range o.Refs {
				o2.Refs[i], err = v.convertGoRef(gr, c)
				if err != nil {
					return nil, err
				}
			}
			for i, gd := range o.Docs {
				o2.Docs[i], err = v.convertGoDoc(gd, c)
				if err != nil {
					return nil, err
				}
			}

			return json.Marshal(o2)
		},
	}

	return &cmd, nil
}

// SymbolData is extra Go-specific data about a symbol.
type SymbolData struct {
	gog.SymbolInfo

	// PackageImportPath is the import path of the package containing this
	// symbol (if this symbol is not a package). If this symbol is a package,
	// PackageImportPath is its own import path.
	PackageImportPath string `json:",omitempty"`
}

func (v *goVersion) convertGoSymbol(gs *gog.Symbol, c *config.Repository) (*graph.Symbol, error) {
	resolvedTarget, err := v.resolveGoImportDep(gs.SymbolKey.PackageImportPath, c)
	if err != nil {
		return nil, err
	}

	sym := &graph.Symbol{
		SymbolKey: graph.SymbolKey{
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
			Path:     graph.SymbolPath(pathOrDot(strings.Join(gs.Path, "/"))),
		},

		Name: gs.Name,
		Kind: graph.SymbolKind(gog.GeneralKindMap[gs.Kind]),

		File:     gs.File,
		DefStart: gs.DeclSpan[0],
		DefEnd:   gs.DeclSpan[1],

		Exported: gs.SymbolInfo.Exported,
		Test:     strings.HasSuffix(gs.File, "_test.go"),
	}

	d := SymbolData{
		PackageImportPath: gs.SymbolKey.PackageImportPath,
		SymbolInfo:        gs.SymbolInfo,
	}
	sym.Data, err = json.Marshal(d)
	if err != nil {
		return nil, err
	}

	if sym.Kind == "func" {
		sym.Callable = true
	}

	return sym, nil
}

func (v *goVersion) convertGoRef(gr *gog.Ref, c *config.Repository) (*graph.Ref, error) {
	resolvedTarget, err := v.resolveGoImportDep(gr.Symbol.PackageImportPath, c)
	if err != nil {
		return nil, err
	}
	if resolvedTarget == nil {
		return nil, nil
	}

	return &graph.Ref{
		SymbolRepo:     uriOrEmpty(resolvedTarget.ToRepoCloneURL),
		SymbolPath:     graph.SymbolPath(pathOrDot(strings.Join(gr.Symbol.Path, "/"))),
		SymbolUnit:     resolvedTarget.ToUnit,
		SymbolUnitType: resolvedTarget.ToUnitType,
		Def:            gr.Def,
		File:           gr.File,
		Start:          gr.Span[0],
		End:            gr.Span[1],
	}, nil
}

func (v *goVersion) convertGoDoc(gd *gog.Doc, c *config.Repository) (*graph.Doc, error) {
	resolvedTarget, err := v.resolveGoImportDep(gd.PackageImportPath, c)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		SymbolKey: graph.SymbolKey{
			Path:     graph.SymbolPath(pathOrDot(strings.Join(gd.Path, "/"))),
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
		},
		Format: gd.Format,
		Data:   gd.Data,
		File:   gd.File,
		Start:  gd.Span[0],
		End:    gd.Span[1],
	}, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}

func pathOrDot(path string) string {
	if path == "" {
		return "."
	}
	return path
}
<commit_msg>treepath, modify Go symbol paths to be prefixed with ./<commit_after>package golang

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/graph"
	"sourcegraph.com/sourcegraph/srcgraph/grapher2"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
	"sourcegraph.com/sourcegraph/srcgraph/toolchain/golang/gog"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	grapher2.Register(&Package{}, grapher2.DockerGrapher{defaultGoVersion})
}

func (v *goVersion) BuildGrapher(dir string, unit unit.SourceUnit, c *config.Repository) (*container.Command, error) {
	gogBinPath := filepath.Join(os.Getenv("GOBIN"), "gog")

	pkg := unit.(*Package)

	cont, err := v.containerForRepo(dir, unit, c)
	if err != nil {
		return nil, err
	}

	// Install VCS tools in Docker container.
	cont.Dockerfile = append(cont.Dockerfile, []byte("RUN apt-get -yq install git mercurial bzr subversion\n")...)

	cont.AddFiles = append(cont.AddFiles, [2]string{gogBinPath, "/usr/local/bin/gog"})
	cont.Cmd = []string{"bash", "-c", fmt.Sprintf("go get -v -t %s; gog %s", pkg.ImportPath, pkg.ImportPath)}

	cmd := container.Command{
		Container: *cont,
		Transform: func(in []byte) ([]byte, error) {
			var o gog.Output
			err := json.Unmarshal(in, &o)
			if err != nil {
				return nil, err
			}

			o2 := grapher2.Output{
				Symbols: make([]*graph.Symbol, len(o.Symbols)),
				Refs:    make([]*graph.Ref, len(o.Refs)),
				Docs:    make([]*graph.Doc, len(o.Docs)),
			}

			for i, gs := range o.Symbols {
				o2.Symbols[i], err = v.convertGoSymbol(gs, c)
				if err != nil {
					return nil, err
				}
			}
			for i, gr := range o.Refs {
				o2.Refs[i], err = v.convertGoRef(gr, c)
				if err != nil {
					return nil, err
				}
			}
			for i, gd := range o.Docs {
				o2.Docs[i], err = v.convertGoDoc(gd, c)
				if err != nil {
					return nil, err
				}
			}

			return json.Marshal(o2)
		},
	}

	return &cmd, nil
}

// SymbolData is extra Go-specific data about a symbol.
type SymbolData struct {
	gog.SymbolInfo

	// PackageImportPath is the import path of the package containing this
	// symbol (if this symbol is not a package). If this symbol is a package,
	// PackageImportPath is its own import path.
	PackageImportPath string `json:",omitempty"`
}

func (v *goVersion) convertGoSymbol(gs *gog.Symbol, c *config.Repository) (*graph.Symbol, error) {
	resolvedTarget, err := v.resolveGoImportDep(gs.SymbolKey.PackageImportPath, c)
	if err != nil {
		return nil, err
	}
	path := graph.SymbolPath(pathOrDot(strings.Join(gs.Path, "/")))
	treePath := graph.TreePath(path)
	if !treePath.IsValid() {
		return nil, fmt.Errorf("'%s' is not a valid tree-path")
	}

	sym := &graph.Symbol{
		SymbolKey: graph.SymbolKey{
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
			Path:     path,
		},
		TreePath: treePath,

		Name: gs.Name,
		Kind: graph.SymbolKind(gog.GeneralKindMap[gs.Kind]),

		File:     gs.File,
		DefStart: gs.DeclSpan[0],
		DefEnd:   gs.DeclSpan[1],

		Exported: gs.SymbolInfo.Exported,
		Test:     strings.HasSuffix(gs.File, "_test.go"),
	}

	d := SymbolData{
		PackageImportPath: gs.SymbolKey.PackageImportPath,
		SymbolInfo:        gs.SymbolInfo,
	}
	sym.Data, err = json.Marshal(d)
	if err != nil {
		return nil, err
	}

	if sym.Kind == "func" {
		sym.Callable = true
	}

	return sym, nil
}

func (v *goVersion) convertGoRef(gr *gog.Ref, c *config.Repository) (*graph.Ref, error) {
	resolvedTarget, err := v.resolveGoImportDep(gr.Symbol.PackageImportPath, c)
	if err != nil {
		return nil, err
	}
	if resolvedTarget == nil {
		return nil, nil
	}

	return &graph.Ref{
		SymbolRepo:     uriOrEmpty(resolvedTarget.ToRepoCloneURL),
		SymbolPath:     graph.SymbolPath(pathOrDot(strings.Join(gr.Symbol.Path, "/"))),
		SymbolUnit:     resolvedTarget.ToUnit,
		SymbolUnitType: resolvedTarget.ToUnitType,
		Def:            gr.Def,
		File:           gr.File,
		Start:          gr.Span[0],
		End:            gr.Span[1],
	}, nil
}

func (v *goVersion) convertGoDoc(gd *gog.Doc, c *config.Repository) (*graph.Doc, error) {
	resolvedTarget, err := v.resolveGoImportDep(gd.PackageImportPath, c)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		SymbolKey: graph.SymbolKey{
			Path:     graph.SymbolPath(pathOrDot(strings.Join(gd.Path, "/"))),
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
		},
		Format: gd.Format,
		Data:   gd.Data,
		File:   gd.File,
		Start:  gd.Span[0],
		End:    gd.Span[1],
	}, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}

func pathOrDot(path string) string {
	if path == "" {
		return "."
	}
	return fmt.Sprintf("./%s", path)
}
<|endoftext|>


<commit_before>package jwt

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/pmylund/go-cache"
	"github.com/square/go-jose"
	"github.com/stripe/aws-go/aws"
)

// JWKSet represents JWK Set.
// See https://tools.ietf.org/html/rfc7517#section-5
type JWKSet struct {
	Keys []*jose.JsonWebKey `json:"keys"`
}

// JWKSetResponse represents a response of JWK Set.
// This contains a TTL (Time to Live) for caching purpose.
type JWKSetResponse struct {
	Keys []*jose.JsonWebKey

	TTL time.Duration // This would be used as TTL for caching.
}

// JWKsFetcher is an interface that represents JWKs fetcher.
type JWKsFetcher interface {
	// FetchJWKs retrieves JWKSet from path.
	FetchJWKs(path string) (*JWKSetResponse, error)
}

// JWKsHTTPFetcher fetches JWKs via HTTP.
type JWKsHTTPFetcher struct {
	Client *http.Client
}

// FetchJWKs implements JWKsFetcher interface by using http.Client.
// FetchJWKs tries to retrieve JWKSet from uri.
func (f *JWKsHTTPFetcher) FetchJWKs(uri string) (*JWKSetResponse, error) {
	resp, err := f.Client.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
		TTL:  0,
	}, err
}

// JWKsS3Fetcher fetches JWKs via S3.
type JWKsS3Fetcher struct {
	S3Svc s3iface.S3API
}

// FetchJWKs implements JWKsS3Fetcher by using S3. It tries to retrieve an S3 object from path.
// path must be in s3://<bucket>/<key>.
func (f *JWKsS3Fetcher) FetchJWKs(path string) (*JWKSetResponse, error) {
	s3url, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	params := &s3.GetObjectInput{
		Bucket: aws.String(s3url.Host),
		Key:    aws.String(s3url.Path),
	}
	resp, err := f.S3Svc.GetObject(params)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
		TTL:  0,
	}, err
}

type JWKsCacher struct {
	JWKsFetcher

	Cache *cache.Cache
}

func (c *JWKsCacher) FetchJWKs(cacheKey string) (*JWKSetResponse, error) {
	if keys, found := c.Cache.Get(cacheKey); found {
		return &JWKSetResponse{Keys: keys.([]*jose.JsonWebKey)}, nil
	}
	jwksresp, err := c.FetchJWKs(cacheKey)
	if err != nil {
		return nil, err
	}

	c.Cache.Set(cacheKey, jwksresp.Keys, jwksresp.TTL)
	return jwksresp, nil
}

// DecodeJWKSet decodes the data with reading from r into JWKs.
func DecodeJWKSet(r io.Reader) ([]*jose.JsonWebKey, error) {
	keyset := JWKSet{}
	if err := json.NewDecoder(r).Decode(&keyset); err != nil && err != io.EOF {
		return nil, err
	}

	return keyset.Keys, nil
}
<commit_msg>jwk: cleanup<commit_after>package jwt

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/pmylund/go-cache"
	"github.com/square/go-jose"
	"github.com/stripe/aws-go/aws"
)

// JWKSet represents JWK Set.
// See https://tools.ietf.org/html/rfc7517#section-5
type JWKSet struct {
	Keys []*jose.JsonWebKey `json:"keys"`
}

// JWKSetResponse represents a response of JWK Set.
// This contains a TTL (Time to Live) for caching purpose.
type JWKSetResponse struct {
	Keys []*jose.JsonWebKey

	TTL time.Duration // This would be used as TTL for caching.
}

// JWKsFetcher is an interface that represents JWKs fetcher.
type JWKsFetcher interface {
	// FetchJWKs retrieves JWKSet from path.
	FetchJWKs(path string) (*JWKSetResponse, error)
}

// JWKsHTTPFetcher fetches JWKs via HTTP.
type JWKsHTTPFetcher struct {
	Client *http.Client
}

// FetchJWKs implements JWKsFetcher interface by using http.Client.
// FetchJWKs tries to retrieve JWKSet from uri.
func (f *JWKsHTTPFetcher) FetchJWKs(uri string) (*JWKSetResponse, error) {
	resp, err := f.Client.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsS3Fetcher fetches JWKs via S3.
type JWKsS3Fetcher struct {
	S3Svc s3iface.S3API
}

// FetchJWKs implements JWKsS3Fetcher by using S3. It tries to retrieve an S3 object from path.
// path must be in s3://<bucket>/<key>.
func (f *JWKsS3Fetcher) FetchJWKs(path string) (*JWKSetResponse, error) {
	s3url, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	params := &s3.GetObjectInput{
		Bucket: aws.String(s3url.Host),
		Key:    aws.String(s3url.Path),
	}
	resp, err := f.S3Svc.GetObject(params)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jwks, err := DecodeJWKSet(resp.Body)
	return &JWKSetResponse{
		Keys: jwks,
	}, err
}

// JWKsCacher fetches JWKs via Cache if available.
type JWKsCacher struct {
	Fetcher JWKsFetcher
	Cache   *cache.Cache
}

// FetchJWKs tries to retrieve JWKs from Cache. If the cache is not available,
// it will call Fetcher.FetchJWKs and cache the result for future request.
func (c *JWKsCacher) FetchJWKs(cacheKey string) (*JWKSetResponse, error) {
	if keys, found := c.Cache.Get(cacheKey); found {
		return &JWKSetResponse{Keys: keys.([]*jose.JsonWebKey)}, nil
	}
	jwksresp, err := c.Fetcher.FetchJWKs(cacheKey)
	if err != nil {
		return nil, err
	}

	c.Cache.Set(cacheKey, jwksresp.Keys, jwksresp.TTL)
	return jwksresp, nil
}

// DecodeJWKSet decodes the data with reading from r into JWKs.
func DecodeJWKSet(r io.Reader) ([]*jose.JsonWebKey, error) {
	keyset := JWKSet{}
	if err := json.NewDecoder(r).Decode(&keyset); err != nil && err != io.EOF {
		return nil, err
	}

	return keyset.Keys, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
)

type (
	Vector struct {
		root  []interface{}
		tail  []interface{}
		count int
		shift uint
	}
	VectorSeq struct {
		vector *Vector
		index  int
	}
	IndexError struct {
		index int
		count int
	}
)

func clone(s []interface{}) []interface{} {
	result := make([]interface{}, len(s), cap(s))
	copy(result, s)
	return result
}

func (v *Vector) tailoff() int {
	if v.count < 32 {
		return 0
	}
	return ((v.count - 1) >> 5) << 5
}

func (v *Vector) arrayFor(i int) []interface{} {
	if i >= v.count || i < 0 {
		panic(IndexError{index: i, count: v.count})
	}
	if i >= v.tailoff() {
		return v.tail
	}
	node := v.root
	for level := v.shift; level > 0; level -= 5 {
		node = node[(i>>level)&0x01F].([]interface{})
	}
	return node
}

func (v *Vector) at(i int) Object {
	return v.arrayFor(i)[i&0x01F].(Object)
}

func newPath(level uint, node []interface{}) []interface{} {
	if level == 0 {
		return node
	}
	result := make([]interface{}, 32)
	result[0] = newPath(level-5, node)
	return result
}

func (v *Vector) pushTail(level uint, parent []interface{}, tailNode []interface{}) []interface{} {
	subidx := ((v.count - 1) >> level) & 0x01F
	result := clone(parent)
	var nodeToInsert []interface{}
	if level == 5 {
		nodeToInsert = tailNode
	} else {
		if parent[subidx] != nil {
			nodeToInsert = v.pushTail(level-5, parent[subidx].([]interface{}), tailNode)
		} else {
			nodeToInsert = newPath(level-5, tailNode)
		}
	}
	result[subidx] = nodeToInsert
	return result
}

func (v *Vector) conj(obj Object) *Vector {
	var newTail []interface{}
	if v.count-v.tailoff() < 32 {
		newTail = append(clone(v.tail), obj)
		return &Vector{count: v.count + 1, shift: v.shift, root: v.root, tail: newTail}
	}
	var newRoot []interface{}
	newShift := v.shift
	if (v.count >> 5) > (1 << v.shift) {
		newRoot = make([]interface{}, 32)
		newRoot[0] = v.root
		newRoot[1] = newPath(v.shift, v.tail)
		newShift += 5
	} else {
		newRoot = v.pushTail(v.shift, v.root, v.tail)
	}
	newTail = make([]interface{}, 1, 32)
	newTail[0] = obj
	return &Vector{count: v.count + 1, shift: newShift, root: newRoot, tail: newTail}
}

func (v *Vector) ToString(escape bool) string {
	var b bytes.Buffer
	b.WriteRune('[')
	if v.count > 0 {
		for i := 0; i < v.count-1; i++ {
			b.WriteString(v.at(i).ToString(escape))
			b.WriteRune(' ')
		}
		b.WriteString(v.at(v.count - 1).ToString(escape))
	}
	b.WriteRune(']')
	return b.String()
}

func (v *Vector) Equals(other interface{}) bool {
	switch otherVector := other.(type) {
	case *Vector:
		if v == otherVector {
			return true
		}
		if v.count != otherVector.count {
			return false
		}
		for i := 0; i < v.count-1; i++ {
			if v.at(i) != otherVector.at(i) {
				return false
			}
		}
		return true
	default:
		return false
	}
}

func (vseq *VectorSeq) Equals(other interface{}) bool {
	if vseq == other {
		return true
	}
	switch s := other.(type) {
	case Sequenceable:
		return SeqsEqual(vseq, s.Seq())
	default:
		return false
	}
}

func (vseq *VectorSeq) ToString(escape bool) string {
	return SeqToString(vseq, escape)
}

func (vseq *VectorSeq) First() Object {
	return vseq.vector.at(vseq.index)
}

func (vseq *VectorSeq) Rest() Seq {
	if vseq.index+1 < vseq.vector.count {
		return &VectorSeq{vector: vseq.vector, index: vseq.index + 1}
	}
	return EmptyList
}

func (vseq *VectorSeq) IsEmpty() bool {
	return vseq.index >= vseq.vector.count
}

func (vseq *VectorSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: vseq}
}

func (v *Vector) Seq() Seq {
	return &VectorSeq{vector: v, index: 0}
}

var EmptyVector = &Vector{
	count: 0,
	shift: 5,
	root:  make([]interface{}, 32),
	tail:  make([]interface{}, 0, 32),
}
<commit_msg>Fix vector equality<commit_after>package main

import (
	"bytes"
)

type (
	Vector struct {
		root  []interface{}
		tail  []interface{}
		count int
		shift uint
	}
	VectorSeq struct {
		vector *Vector
		index  int
	}
	IndexError struct {
		index int
		count int
	}
)

func clone(s []interface{}) []interface{} {
	result := make([]interface{}, len(s), cap(s))
	copy(result, s)
	return result
}

func (v *Vector) tailoff() int {
	if v.count < 32 {
		return 0
	}
	return ((v.count - 1) >> 5) << 5
}

func (v *Vector) arrayFor(i int) []interface{} {
	if i >= v.count || i < 0 {
		panic(IndexError{index: i, count: v.count})
	}
	if i >= v.tailoff() {
		return v.tail
	}
	node := v.root
	for level := v.shift; level > 0; level -= 5 {
		node = node[(i>>level)&0x01F].([]interface{})
	}
	return node
}

func (v *Vector) at(i int) Object {
	return v.arrayFor(i)[i&0x01F].(Object)
}

func newPath(level uint, node []interface{}) []interface{} {
	if level == 0 {
		return node
	}
	result := make([]interface{}, 32)
	result[0] = newPath(level-5, node)
	return result
}

func (v *Vector) pushTail(level uint, parent []interface{}, tailNode []interface{}) []interface{} {
	subidx := ((v.count - 1) >> level) & 0x01F
	result := clone(parent)
	var nodeToInsert []interface{}
	if level == 5 {
		nodeToInsert = tailNode
	} else {
		if parent[subidx] != nil {
			nodeToInsert = v.pushTail(level-5, parent[subidx].([]interface{}), tailNode)
		} else {
			nodeToInsert = newPath(level-5, tailNode)
		}
	}
	result[subidx] = nodeToInsert
	return result
}

func (v *Vector) conj(obj Object) *Vector {
	var newTail []interface{}
	if v.count-v.tailoff() < 32 {
		newTail = append(clone(v.tail), obj)
		return &Vector{count: v.count + 1, shift: v.shift, root: v.root, tail: newTail}
	}
	var newRoot []interface{}
	newShift := v.shift
	if (v.count >> 5) > (1 << v.shift) {
		newRoot = make([]interface{}, 32)
		newRoot[0] = v.root
		newRoot[1] = newPath(v.shift, v.tail)
		newShift += 5
	} else {
		newRoot = v.pushTail(v.shift, v.root, v.tail)
	}
	newTail = make([]interface{}, 1, 32)
	newTail[0] = obj
	return &Vector{count: v.count + 1, shift: newShift, root: newRoot, tail: newTail}
}

func (v *Vector) ToString(escape bool) string {
	var b bytes.Buffer
	b.WriteRune('[')
	if v.count > 0 {
		for i := 0; i < v.count-1; i++ {
			b.WriteString(v.at(i).ToString(escape))
			b.WriteRune(' ')
		}
		b.WriteString(v.at(v.count - 1).ToString(escape))
	}
	b.WriteRune(']')
	return b.String()
}

func (v *Vector) Equals(other interface{}) bool {
	if v == other {
		return true
	}
	switch otherVector := other.(type) {
	case *Vector:
		if v.count != otherVector.count {
			return false
		}
		for i := 0; i < v.count; i++ {
			if v.at(i) != otherVector.at(i) {
				return false
			}
		}
		return true
	default:
		return false
	}
}

func (vseq *VectorSeq) Equals(other interface{}) bool {
	if vseq == other {
		return true
	}
	switch s := other.(type) {
	case Sequenceable:
		return SeqsEqual(vseq, s.Seq())
	default:
		return false
	}
}

func (vseq *VectorSeq) ToString(escape bool) string {
	return SeqToString(vseq, escape)
}

func (vseq *VectorSeq) First() Object {
	return vseq.vector.at(vseq.index)
}

func (vseq *VectorSeq) Rest() Seq {
	if vseq.index+1 < vseq.vector.count {
		return &VectorSeq{vector: vseq.vector, index: vseq.index + 1}
	}
	return EmptyList
}

func (vseq *VectorSeq) IsEmpty() bool {
	return vseq.index >= vseq.vector.count
}

func (vseq *VectorSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: vseq}
}

func (v *Vector) Seq() Seq {
	return &VectorSeq{vector: v, index: 0}
}

var EmptyVector = &Vector{
	count: 0,
	shift: 5,
	root:  make([]interface{}, 32),
	tail:  make([]interface{}, 0, 32),
}
<|endoftext|>


<commit_before>package irma

import (
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
)

// ProofStatus is the status of the complete proof
type ProofStatus string

// Status is the proof status of a single attribute
type AttributeProofStatus string

const (
	ProofStatusValid             = ProofStatus("VALID")              // Proof is valid
	ProofStatusInvalid           = ProofStatus("INVALID")            // Proof is invalid
	ProofStatusInvalidTimestamp  = ProofStatus("INVALID_TIMESTAMP")  // Attribute-based signature had invalid timestamp
	ProofStatusUnmatchedRequest  = ProofStatus("UNMATCHED_REQUEST")  // Proof does not correspond to a specified request
	ProofStatusMissingAttributes = ProofStatus("MISSING_ATTRIBUTES") // Proof does not contain all requested attributes
	ProofStatusExpired           = ProofStatus("EXPIRED")            // Attributes were expired at proof creation time (now, or according to timestamp in case of abs)

	AttributeProofStatusPresent      = AttributeProofStatus("PRESENT")       // Attribute is disclosed and matches the value
	AttributeProofStatusExtra        = AttributeProofStatus("EXTRA")         // Attribute is disclosed, but wasn't requested in request
	AttributeProofStatusMissing      = AttributeProofStatus("MISSING")       // Attribute is NOT disclosed, but should be according to request
	AttributeProofStatusInvalidValue = AttributeProofStatus("INVALID_VALUE") // Attribute is disclosed, but has invalid value according to request
)

// DisclosedAttribute represents a disclosed attribute.
type DisclosedAttribute struct {
	Value      TranslatedString        `json:"value"` // Value of the disclosed attribute
	Identifier AttributeTypeIdentifier `json:"id"`
	Status     AttributeProofStatus    `json:"status"`
}

// ProofList is a gabi.ProofList with some extra methods.
type ProofList gabi.ProofList

var ErrorMissingPublicKey = errors.New("Missing public key")

// ExtractPublicKeys returns the public keys of each proof in the proofList, in the same order,
// for later use in verification of the proofList. If one of the proofs is not a ProofD
// an error is returned.
func (pl ProofList) ExtractPublicKeys(configuration *Configuration) ([]*gabi.PublicKey, error) {
	var publicKeys = make([]*gabi.PublicKey, 0, len(pl))

	for _, v := range pl {
		switch v.(type) {
		case *gabi.ProofD:
			proof := v.(*gabi.ProofD)
			metadata := MetadataFromInt(proof.ADisclosed[1], configuration) // index 1 is metadata attribute
			publicKey, err := metadata.PublicKey()
			if err != nil {
				return nil, err
			}
			if publicKey == nil {
				return nil, ErrorMissingPublicKey
			}
			publicKeys = append(publicKeys, publicKey)
		default:
			return nil, errors.New("Cannot extract public key, not a disclosure proofD")
		}
	}
	return publicKeys, nil
}

// VerifyProofs verifies the proofs cryptographically.
func (pl ProofList) VerifyProofs(configuration *Configuration, context *big.Int, nonce *big.Int, publickeys []*gabi.PublicKey, isSig bool) (bool, error) {
	if publickeys == nil {
		var err error
		publickeys, err = pl.ExtractPublicKeys(configuration)
		if err != nil {
			return false, err
		}
	}

	if len(pl) != len(publickeys) {
		return false, errors.New("Insufficient public keys to verify the proofs")
	}

	// Compute slice to inform gabi of which proofs should be verified to share the same secret key
	keyshareServers := make([]string, len(pl))
	for i := range pl {
		schemeID := NewIssuerIdentifier(publickeys[i].Issuer).SchemeManagerIdentifier()
		if !configuration.SchemeManagers[schemeID].Distributed() {
			keyshareServers[i] = "." // dummy value: no IRMA scheme will ever have this name
		} else {
			keyshareServers[i] = schemeID.Name()
		}
	}

	return gabi.ProofList(pl).Verify(publickeys, context, nonce, isSig, keyshareServers), nil
}

// Expired returns true if any of the contained disclosure proofs is specified at the specified time,
// or now, when the specified time is nil.
func (pl ProofList) Expired(configuration *Configuration, t *time.Time) bool {
	if t == nil {
		temp := time.Now()
		t = &temp
	}
	for _, proof := range pl {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		if metadata.Expiry().Before(*t) {
			return true
		}
	}
	return false
}

// DisclosedAttributes returns a slice containing the disclosed attributes that are present in the proof list.
// If a non-empty and non-nil AttributeDisjunctionList is included, then the first attributes in the returned slice match
// with the disjunction list in the disjunction list. If any of the given disjunctions is not matched by one
// of the disclosed attributes, then the corresponding item in the returned slice has status AttributeProofStatusMissing.
// The first return parameter of this function indicates whether or not all disjunctions (if present) are satisfied.
func (d *Disclosure) DisclosedAttributes(configuration *Configuration, disjunctions AttributeDisjunctionList) (bool, []*DisclosedAttribute, error) {
	if d.Indices == nil || len(disjunctions) == 0 {
		return ProofList(d.Proofs).DisclosedAttributes(configuration, disjunctions)
	}

	list := make([]*DisclosedAttribute, len(disjunctions))
	usedAttrs := map[int]map[int]struct{}{} // keep track of attributes that satisfy the disjunctions

	// For each of the disjunctions, lookup the attribute that the user sent to satisfy this disjunction,
	// using the indices specified by the user in d.Indices. Then see if the attribute satisfies the disjunction.
	for i, disjunction := range disjunctions {
		index := d.Indices[i][0]
		proofd, ok := d.Proofs[index.CredentialIndex].(*gabi.ProofD)
		if !ok {
			// If with the index the user told us to look for the required attribute at this specific location,
			// and the proof here is not a disclosure proof, then reject
			return false, nil, errors.New("ProofList contained proof of invalid type")
		}

		if usedAttrs[index.CredentialIndex] == nil {
			usedAttrs[index.CredentialIndex] = map[int]struct{}{}
		}
		usedAttrs[index.CredentialIndex][index.AttributeIndex] = struct{}{}

		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		attr, attrval, err := parseAttribute(index.AttributeIndex, metadata, proofd.ADisclosed[index.AttributeIndex+2])
		if err != nil {
			return false, nil, err
		}

		if disjunction.attemptSatisfy(attr.Identifier, attrval) {
			list[i] = attr
			if disjunction.satisfied() {
				list[i].Status = AttributeProofStatusPresent
			} else {
				list[i].Status = AttributeProofStatusInvalidValue
			}
		} else {
			list[i] = &DisclosedAttribute{Status: AttributeProofStatusMissing}
		}
	}

	// Loop over any extra attributes in d.Proofs not requested in any of the disjunctions
	for i, proof := range d.Proofs {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		for attrIndex, attrInt := range proofd.ADisclosed {
			if attrIndex == 0 || attrIndex == 1 { // Never add secret key or metadata (i.e. no-attribute disclosure) as extra
				continue // Note that the secret should never be disclosed by the client, but we skip it to be sure
			}
			if _, used := usedAttrs[i][attrIndex]; used {
				continue
			}

			attr, _, err := parseAttribute(attrIndex, metadata, attrInt)
			if err != nil {
				return false, nil, err
			}
			attr.Status = AttributeProofStatusExtra
			list = append(list, attr)
		}
	}

	return len(disjunctions) == 0 || disjunctions.satisfied(), list, nil
}

func parseAttribute(index int, metadata *MetadataAttribute, attr *big.Int) (*DisclosedAttribute, *string, error) {
	var attrid AttributeTypeIdentifier
	var attrval *string
	credtype := metadata.CredentialType()
	if credtype == nil {
		return nil, nil, errors.New("ProofList contained a disclosure proof of an unkown credential type")
	}
	if index == 1 {
		attrid = NewAttributeTypeIdentifier(credtype.Identifier().String())
		p := "present"
		attrval = &p
	} else {
		attrid = credtype.AttributeTypes[index-2].GetAttributeTypeIdentifier()
		attrval = decodeAttribute(attr, metadata.Version())
	}
	return &DisclosedAttribute{
		Identifier: attrid,
		Value:      translateAttribute(attrval),
	}, attrval, nil
}

func (pl ProofList) DisclosedAttributes(configuration *Configuration, disjunctions AttributeDisjunctionList) (bool, []*DisclosedAttribute, error) {
	var list []*DisclosedAttribute
	list = make([]*DisclosedAttribute, len(disjunctions))
	for i := range list {
		// Populate list with AttributeProofStatusMissing; if an attribute that satisfies a disjunction
		// is found below, the corresponding entry in the list is overwritten
		list[i] = &DisclosedAttribute{
			Status: AttributeProofStatusMissing,
		}
	}

	// Temp slice for attributes that have not yet been matched to one of the disjunctions of the request
	// When we are done matching disclosed attributes against the request, filling the first few slots of list,
	// we append these to list just before returning
	extraAttrs := map[AttributeTypeIdentifier]*DisclosedAttribute{}

	for _, proof := range pl {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute

		for attrIndex, attrInt := range proofd.ADisclosed {
			if attrIndex == 0 {
				continue // Should never be disclosed, but skip it to be sure
			}

			attr, attrval, err := parseAttribute(attrIndex, metadata, attrInt)
			if err != nil {
				return false, nil, err
			}

			if attrIndex > 1 { // Never add metadata (i.e. no-attribute disclosure) as extra
				extraAttrs[attr.Identifier] = attr
			}
			if len(disjunctions) == 0 {
				continue
			}

			// See if the current attribute satisfies one of the disjunctions, if so, delete it from extraAttrs
			for i, disjunction := range disjunctions {
				if disjunction.attemptSatisfy(attr.Identifier, attrval) {
					if disjunction.satisfied() {
						attr.Status = AttributeProofStatusPresent
					} else {
						attr.Status = AttributeProofStatusInvalidValue
					}
					list[i] = attr
					delete(extraAttrs, attr.Identifier)
				}
			}
		}
	}

	// Any attributes still in here do not satisfy any of the specified disjunctions; append them now
	for _, attr := range extraAttrs {
		list = append(list, attr)
	}

	return len(disjunctions) == 0 || disjunctions.satisfied(), list, nil
}

func (d *Disclosure) VerifyAgainstDisjunctions(
	configuration *Configuration,
	required AttributeDisjunctionList,
	context, nonce *big.Int,
	publickeys []*gabi.PublicKey,
	issig bool,
) ([]*DisclosedAttribute, ProofStatus, error) {
	// Cryptographically verify the IRMA disclosure proofs in the signature
	valid, err := ProofList(d.Proofs).VerifyProofs(configuration, context, nonce, publickeys, issig)
	if !valid || err != nil {
		return nil, ProofStatusInvalid, err
	}

	// Next extract the contained attributes from the proofs, and match them to the signature request if present
	allmatched, list, err := d.DisclosedAttributes(configuration, required)
	if err != nil {
		return nil, ProofStatusInvalid, err
	}

	// Return MISSING_ATTRIBUTES as proofstatus if one of the disjunctions in the request (if present) is not satisfied
	if !allmatched {
		return list, ProofStatusMissingAttributes, nil
	}

	return list, ProofStatusValid, nil
}

func (d *Disclosure) Verify(configuration *Configuration, request *DisclosureRequest) ([]*DisclosedAttribute, ProofStatus, error) {
	list, status, err := d.VerifyAgainstDisjunctions(configuration, request.Content, request.Context, request.Nonce, nil, false)
	if err != nil {
		return list, status, err
	}

	now := time.Now()
	if expired := ProofList(d.Proofs).Expired(configuration, &now); expired {
		return list, ProofStatusExpired, nil
	}

	return list, status, nil
}

// Verify the attribute-based signature, optionally against a corresponding signature request. If the request is present
// (i.e. not nil), then the first attributes in the returned result match with the disjunction list in the request
// (that is, the i'th attribute in the result should satisfy the i'th disjunction in the request). If the request is not
// fully satisfied in this fasion, the Status of the result is ProofStatusMissingAttributes. Any remaining attributes
// (i.e. not asked for by the request) are also included in the result, after the attributes that match disjunctions
// in the request.
//
// The signature request is optional; if it is nil then the attribute-based signature is still verified, and all
// containing attributes returned in the result.
func (sm *SignedMessage) Verify(configuration *Configuration, request *SignatureRequest) ([]*DisclosedAttribute, ProofStatus, error) {
	var message string

	// First check if this signature matches the request
	if request != nil {
		request.Timestamp = sm.Timestamp
		if !sm.MatchesNonceAndContext(request) {
			return nil, ProofStatusUnmatchedRequest, nil
		}
		// If there is a request, then the signed message must be that of the request
		message = request.Message
	} else {
		// If not, we just verify that the signed message is a valid signature over its contained message
		message = sm.Message
	}

	// Verify the timestamp
	if sm.Timestamp != nil {
		if err := sm.VerifyTimestamp(message, configuration); err != nil {
			return nil, ProofStatusInvalidTimestamp, nil
		}
	}

	// Now, cryptographically verify the IRMA disclosure proofs in the signature
	var required AttributeDisjunctionList
	if request != nil {
		required = request.Content
	}
	result, status, err := sm.Disclosure().VerifyAgainstDisjunctions(configuration, required, sm.Context, sm.GetNonce(), nil, true)
	if status != ProofStatusValid || err != nil {
		return result, status, err
	}

	// Check if a credential is expired
	var t time.Time
	if sm.Timestamp != nil {
		t = time.Unix(sm.Timestamp.Time, 0)
	}
	if expired := ProofList(sm.Signature).Expired(configuration, &t); expired {
		// The ABS contains attributes that were expired at the time of creation of the ABS.
		return result, ProofStatusExpired, nil
	}

	// All disjunctions satisfied and nothing expired, proof is valid!
	return result, ProofStatusValid, nil
}
<commit_msg>Fix wrong index offset in verification<commit_after>package irma

import (
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
)

// ProofStatus is the status of the complete proof
type ProofStatus string

// Status is the proof status of a single attribute
type AttributeProofStatus string

const (
	ProofStatusValid             = ProofStatus("VALID")              // Proof is valid
	ProofStatusInvalid           = ProofStatus("INVALID")            // Proof is invalid
	ProofStatusInvalidTimestamp  = ProofStatus("INVALID_TIMESTAMP")  // Attribute-based signature had invalid timestamp
	ProofStatusUnmatchedRequest  = ProofStatus("UNMATCHED_REQUEST")  // Proof does not correspond to a specified request
	ProofStatusMissingAttributes = ProofStatus("MISSING_ATTRIBUTES") // Proof does not contain all requested attributes
	ProofStatusExpired           = ProofStatus("EXPIRED")            // Attributes were expired at proof creation time (now, or according to timestamp in case of abs)

	AttributeProofStatusPresent      = AttributeProofStatus("PRESENT")       // Attribute is disclosed and matches the value
	AttributeProofStatusExtra        = AttributeProofStatus("EXTRA")         // Attribute is disclosed, but wasn't requested in request
	AttributeProofStatusMissing      = AttributeProofStatus("MISSING")       // Attribute is NOT disclosed, but should be according to request
	AttributeProofStatusInvalidValue = AttributeProofStatus("INVALID_VALUE") // Attribute is disclosed, but has invalid value according to request
)

// DisclosedAttribute represents a disclosed attribute.
type DisclosedAttribute struct {
	Value      TranslatedString        `json:"value"` // Value of the disclosed attribute
	Identifier AttributeTypeIdentifier `json:"id"`
	Status     AttributeProofStatus    `json:"status"`
}

// ProofList is a gabi.ProofList with some extra methods.
type ProofList gabi.ProofList

var ErrorMissingPublicKey = errors.New("Missing public key")

// ExtractPublicKeys returns the public keys of each proof in the proofList, in the same order,
// for later use in verification of the proofList. If one of the proofs is not a ProofD
// an error is returned.
func (pl ProofList) ExtractPublicKeys(configuration *Configuration) ([]*gabi.PublicKey, error) {
	var publicKeys = make([]*gabi.PublicKey, 0, len(pl))

	for _, v := range pl {
		switch v.(type) {
		case *gabi.ProofD:
			proof := v.(*gabi.ProofD)
			metadata := MetadataFromInt(proof.ADisclosed[1], configuration) // index 1 is metadata attribute
			publicKey, err := metadata.PublicKey()
			if err != nil {
				return nil, err
			}
			if publicKey == nil {
				return nil, ErrorMissingPublicKey
			}
			publicKeys = append(publicKeys, publicKey)
		default:
			return nil, errors.New("Cannot extract public key, not a disclosure proofD")
		}
	}
	return publicKeys, nil
}

// VerifyProofs verifies the proofs cryptographically.
func (pl ProofList) VerifyProofs(configuration *Configuration, context *big.Int, nonce *big.Int, publickeys []*gabi.PublicKey, isSig bool) (bool, error) {
	if publickeys == nil {
		var err error
		publickeys, err = pl.ExtractPublicKeys(configuration)
		if err != nil {
			return false, err
		}
	}

	if len(pl) != len(publickeys) {
		return false, errors.New("Insufficient public keys to verify the proofs")
	}

	// Compute slice to inform gabi of which proofs should be verified to share the same secret key
	keyshareServers := make([]string, len(pl))
	for i := range pl {
		schemeID := NewIssuerIdentifier(publickeys[i].Issuer).SchemeManagerIdentifier()
		if !configuration.SchemeManagers[schemeID].Distributed() {
			keyshareServers[i] = "." // dummy value: no IRMA scheme will ever have this name
		} else {
			keyshareServers[i] = schemeID.Name()
		}
	}

	return gabi.ProofList(pl).Verify(publickeys, context, nonce, isSig, keyshareServers), nil
}

// Expired returns true if any of the contained disclosure proofs is specified at the specified time,
// or now, when the specified time is nil.
func (pl ProofList) Expired(configuration *Configuration, t *time.Time) bool {
	if t == nil {
		temp := time.Now()
		t = &temp
	}
	for _, proof := range pl {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		if metadata.Expiry().Before(*t) {
			return true
		}
	}
	return false
}

// DisclosedAttributes returns a slice containing the disclosed attributes that are present in the proof list.
// If a non-empty and non-nil AttributeDisjunctionList is included, then the first attributes in the returned slice match
// with the disjunction list in the disjunction list. If any of the given disjunctions is not matched by one
// of the disclosed attributes, then the corresponding item in the returned slice has status AttributeProofStatusMissing.
// The first return parameter of this function indicates whether or not all disjunctions (if present) are satisfied.
func (d *Disclosure) DisclosedAttributes(configuration *Configuration, disjunctions AttributeDisjunctionList) (bool, []*DisclosedAttribute, error) {
	if d.Indices == nil || len(disjunctions) == 0 {
		return ProofList(d.Proofs).DisclosedAttributes(configuration, disjunctions)
	}

	list := make([]*DisclosedAttribute, len(disjunctions))
	usedAttrs := map[int]map[int]struct{}{} // keep track of attributes that satisfy the disjunctions

	// For each of the disjunctions, lookup the attribute that the user sent to satisfy this disjunction,
	// using the indices specified by the user in d.Indices. Then see if the attribute satisfies the disjunction.
	for i, disjunction := range disjunctions {
		index := d.Indices[i][0]
		proofd, ok := d.Proofs[index.CredentialIndex].(*gabi.ProofD)
		if !ok {
			// If with the index the user told us to look for the required attribute at this specific location,
			// and the proof here is not a disclosure proof, then reject
			return false, nil, errors.New("ProofList contained proof of invalid type")
		}

		if usedAttrs[index.CredentialIndex] == nil {
			usedAttrs[index.CredentialIndex] = map[int]struct{}{}
		}
		usedAttrs[index.CredentialIndex][index.AttributeIndex] = struct{}{}

		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		attr, attrval, err := parseAttribute(index.AttributeIndex, metadata, proofd.ADisclosed[index.AttributeIndex])
		if err != nil {
			return false, nil, err
		}

		if disjunction.attemptSatisfy(attr.Identifier, attrval) {
			list[i] = attr
			if disjunction.satisfied() {
				list[i].Status = AttributeProofStatusPresent
			} else {
				list[i].Status = AttributeProofStatusInvalidValue
			}
		} else {
			list[i] = &DisclosedAttribute{Status: AttributeProofStatusMissing}
		}
	}

	// Loop over any extra attributes in d.Proofs not requested in any of the disjunctions
	for i, proof := range d.Proofs {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute
		for attrIndex, attrInt := range proofd.ADisclosed {
			if attrIndex == 0 || attrIndex == 1 { // Never add secret key or metadata (i.e. no-attribute disclosure) as extra
				continue // Note that the secret should never be disclosed by the client, but we skip it to be sure
			}
			if _, used := usedAttrs[i][attrIndex]; used {
				continue
			}

			attr, _, err := parseAttribute(attrIndex, metadata, attrInt)
			if err != nil {
				return false, nil, err
			}
			attr.Status = AttributeProofStatusExtra
			list = append(list, attr)
		}
	}

	return len(disjunctions) == 0 || disjunctions.satisfied(), list, nil
}

func parseAttribute(index int, metadata *MetadataAttribute, attr *big.Int) (*DisclosedAttribute, *string, error) {
	var attrid AttributeTypeIdentifier
	var attrval *string
	credtype := metadata.CredentialType()
	if credtype == nil {
		return nil, nil, errors.New("ProofList contained a disclosure proof of an unkown credential type")
	}
	if index == 1 {
		attrid = NewAttributeTypeIdentifier(credtype.Identifier().String())
		p := "present"
		attrval = &p
	} else {
		attrid = credtype.AttributeTypes[index-2].GetAttributeTypeIdentifier()
		attrval = decodeAttribute(attr, metadata.Version())
	}
	return &DisclosedAttribute{
		Identifier: attrid,
		Value:      translateAttribute(attrval),
	}, attrval, nil
}

func (pl ProofList) DisclosedAttributes(configuration *Configuration, disjunctions AttributeDisjunctionList) (bool, []*DisclosedAttribute, error) {
	var list []*DisclosedAttribute
	list = make([]*DisclosedAttribute, len(disjunctions))
	for i := range list {
		// Populate list with AttributeProofStatusMissing; if an attribute that satisfies a disjunction
		// is found below, the corresponding entry in the list is overwritten
		list[i] = &DisclosedAttribute{
			Status: AttributeProofStatusMissing,
		}
	}

	// Temp slice for attributes that have not yet been matched to one of the disjunctions of the request
	// When we are done matching disclosed attributes against the request, filling the first few slots of list,
	// we append these to list just before returning
	extraAttrs := map[AttributeTypeIdentifier]*DisclosedAttribute{}

	for _, proof := range pl {
		proofd, ok := proof.(*gabi.ProofD)
		if !ok {
			continue
		}
		metadata := MetadataFromInt(proofd.ADisclosed[1], configuration) // index 1 is metadata attribute

		for attrIndex, attrInt := range proofd.ADisclosed {
			if attrIndex == 0 {
				continue // Should never be disclosed, but skip it to be sure
			}

			attr, attrval, err := parseAttribute(attrIndex, metadata, attrInt)
			if err != nil {
				return false, nil, err
			}

			if attrIndex > 1 { // Never add metadata (i.e. no-attribute disclosure) as extra
				extraAttrs[attr.Identifier] = attr
			}
			if len(disjunctions) == 0 {
				continue
			}

			// See if the current attribute satisfies one of the disjunctions, if so, delete it from extraAttrs
			for i, disjunction := range disjunctions {
				if disjunction.attemptSatisfy(attr.Identifier, attrval) {
					if disjunction.satisfied() {
						attr.Status = AttributeProofStatusPresent
					} else {
						attr.Status = AttributeProofStatusInvalidValue
					}
					list[i] = attr
					delete(extraAttrs, attr.Identifier)
				}
			}
		}
	}

	// Any attributes still in here do not satisfy any of the specified disjunctions; append them now
	for _, attr := range extraAttrs {
		list = append(list, attr)
	}

	return len(disjunctions) == 0 || disjunctions.satisfied(), list, nil
}

func (d *Disclosure) VerifyAgainstDisjunctions(
	configuration *Configuration,
	required AttributeDisjunctionList,
	context, nonce *big.Int,
	publickeys []*gabi.PublicKey,
	issig bool,
) ([]*DisclosedAttribute, ProofStatus, error) {
	// Cryptographically verify the IRMA disclosure proofs in the signature
	valid, err := ProofList(d.Proofs).VerifyProofs(configuration, context, nonce, publickeys, issig)
	if !valid || err != nil {
		return nil, ProofStatusInvalid, err
	}

	// Next extract the contained attributes from the proofs, and match them to the signature request if present
	allmatched, list, err := d.DisclosedAttributes(configuration, required)
	if err != nil {
		return nil, ProofStatusInvalid, err
	}

	// Return MISSING_ATTRIBUTES as proofstatus if one of the disjunctions in the request (if present) is not satisfied
	if !allmatched {
		return list, ProofStatusMissingAttributes, nil
	}

	return list, ProofStatusValid, nil
}

func (d *Disclosure) Verify(configuration *Configuration, request *DisclosureRequest) ([]*DisclosedAttribute, ProofStatus, error) {
	list, status, err := d.VerifyAgainstDisjunctions(configuration, request.Content, request.Context, request.Nonce, nil, false)
	if err != nil {
		return list, status, err
	}

	now := time.Now()
	if expired := ProofList(d.Proofs).Expired(configuration, &now); expired {
		return list, ProofStatusExpired, nil
	}

	return list, status, nil
}

// Verify the attribute-based signature, optionally against a corresponding signature request. If the request is present
// (i.e. not nil), then the first attributes in the returned result match with the disjunction list in the request
// (that is, the i'th attribute in the result should satisfy the i'th disjunction in the request). If the request is not
// fully satisfied in this fasion, the Status of the result is ProofStatusMissingAttributes. Any remaining attributes
// (i.e. not asked for by the request) are also included in the result, after the attributes that match disjunctions
// in the request.
//
// The signature request is optional; if it is nil then the attribute-based signature is still verified, and all
// containing attributes returned in the result.
func (sm *SignedMessage) Verify(configuration *Configuration, request *SignatureRequest) ([]*DisclosedAttribute, ProofStatus, error) {
	var message string

	// First check if this signature matches the request
	if request != nil {
		request.Timestamp = sm.Timestamp
		if !sm.MatchesNonceAndContext(request) {
			return nil, ProofStatusUnmatchedRequest, nil
		}
		// If there is a request, then the signed message must be that of the request
		message = request.Message
	} else {
		// If not, we just verify that the signed message is a valid signature over its contained message
		message = sm.Message
	}

	// Verify the timestamp
	if sm.Timestamp != nil {
		if err := sm.VerifyTimestamp(message, configuration); err != nil {
			return nil, ProofStatusInvalidTimestamp, nil
		}
	}

	// Now, cryptographically verify the IRMA disclosure proofs in the signature
	var required AttributeDisjunctionList
	if request != nil {
		required = request.Content
	}
	result, status, err := sm.Disclosure().VerifyAgainstDisjunctions(configuration, required, sm.Context, sm.GetNonce(), nil, true)
	if status != ProofStatusValid || err != nil {
		return result, status, err
	}

	// Check if a credential is expired
	var t time.Time
	if sm.Timestamp != nil {
		t = time.Unix(sm.Timestamp.Time, 0)
	}
	if expired := ProofList(sm.Signature).Expired(configuration, &t); expired {
		// The ABS contains attributes that were expired at the time of creation of the ABS.
		return result, ProofStatusExpired, nil
	}

	// All disjunctions satisfied and nothing expired, proof is valid!
	return result, ProofStatusValid, nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/

package main

import (
    "fmt"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/dynamodb"
)

func main() {
    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create DynamoDB client
    svc := dynamodb.New(sess)

    input := &dynamodb.DeleteItemInput{
        Key: map[string]*dynamodb.AttributeValue{
            "year": {
                N: aws.String("2015"),
            },
            "title": {
                S: aws.String("The Big New Movie"),
            },
        },
        TableName: aws.String("Movies"),
    }

    _, err := svc.DeleteItem(input)

    if err != nil {
        fmt.Println("Got error calling DeleteItem")
        fmt.Println(err.Error())
        return
    }

    fmt.Println("Deleted 'The Big New Movie' (2015)")
}
<commit_msg>Oops, the DynamoDB example in Go was correct<commit_after>/*
   Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/

package main

import (
    "fmt"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/dynamodb"
)

func main() {
    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create DynamoDB client
    svc := dynamodb.New(sess)

    input := &dynamodb.DeleteItemInput{
        Key: map[string]*dynamodb.AttributeValue{
            "year": {
                N: aws.String("2015"),
            },
            "title": {
                S: aws.String("The Big New Movie"),
            },
        },
        TableName: aws.String("Movies"),
    }

    _, err = svc.DeleteItem(input)

    if err != nil {
        fmt.Println("Got error calling DeleteItem")
        fmt.Println(err.Error())
        return
    }

    fmt.Println("Deleted 'The Big New Movie' (2015)")
}
<|endoftext|>