<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
)

var (
	newBuildCrawlerInterval      = 10 * time.Second
	finishedBuildCrawlerInterval = 2 * time.Minute
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *db.DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(newBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", db.Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
			err := stathat.CountOne(repo.LastBuildStartedAt)
			if err != nil {
				c.Logger.Println(err)
			}
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(finishedBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo *Repo
		//query db.Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(nil).Sort("lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			// this build has real error
			// otherwise it's still in progress
			if build == nil {
				c.Logger.Println(err)
			}
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	if err := iter.Close(); err != nil {
		c.Logger.Println(err)
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := build.FinishedAt != nil && build.StartedAt != nil
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, db.Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(db.Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<commit_msg>Increase new build crawler interval to 30 sec<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
)

var (
	newBuildCrawlerInterval      = 30 * time.Second
	finishedBuildCrawlerInterval = 2 * time.Minute
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *db.DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(newBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", db.Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
			err := stathat.CountOne(repo.LastBuildStartedAt)
			if err != nil {
				c.Logger.Println(err)
			}
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *db.DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(finishedBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo *Repo
		//query db.Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(nil).Sort("lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			// this build has real error
			// otherwise it's still in progress
			if build == nil {
				c.Logger.Println(err)
			}
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	if err := iter.Close(); err != nil {
		c.Logger.Println(err)
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := build.FinishedAt != nil && build.StartedAt != nil
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, db.Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(db.Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/ipv4"
	"net"
)

func main() {
	manager, _ := NewUDP_Manager()
	c := []*UDP{}
	for i := 0; i < 20; i++ {
		udp, _ := manager.NewUDP(((uint16)(20000 + i)), ((uint16)(20000 + i)))
		c = append(c, udp)
		udp.write([]byte{((byte)(100 + i))})
		a, _ := udp.read(1)
		fmt.Println(a)
	}
	for _, element := range c {
		element.close()
	}
}

type UDP_manager struct {
    ipAddress string
	pl   net.PacketConn
	open bool
	conn *ipv4.RawConn
	buff map[uint16](chan byte)
}

type UDP struct {
    manager   *UDP_manager
	conn      *ipv4.RawConn
	bytes     chan byte
	src, dest uint16
}

func NewUDP_Manager(ip string) (*UDP_manager, error) {
	p, err := net.ListenPacket("ip4:17", ip)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	x := &UDP_manager{open: true, conn: r, pl: p, buff: make(map[uint16](chan byte)), ipAddress: ip}

	go x.readAll()

	return x, nil
}

func (x *UDP_manager) readAll() {
	b := make([]byte, 1024)

	for {
		_, payload, _, err := x.conn.ReadFrom(b)
		if err != nil {
			continue
		}

		dest := (((uint16)(payload[2])) << 8) + ((uint16)(payload[3]))
		//		fmt.Println(dest)
		//		fmt.Println(payload)
		//
		//		fmt.Println(x.buff)
		c, ok := x.buff[dest]
		//fmt.Println(ok)
		payload = payload[8:]
		if ok {
			go func() {
				for _, elem := range payload {
					//fmt.Println("Writing")
					c <- elem
				}
			}()
		}
	}
}

func (x *UDP_manager) NewUDP(src, dest uint16) (*UDP, error) {
	x.buff[src] = make(chan byte, 1024)
	return &UDP{src: src, dest: dest, conn: x.conn, bytes: x.buff[src], manager: x}, nil
}

func (c *UDP) read(size int) ([]byte, error) {
	data := make([]byte, size)
	for i := 0; i < size; i++ {
		//fmt.Println("test")
		data[i] = <-c.bytes
		//fmt.Println(data[i])
	}
	return data, nil
}
func (c *UDP) write(x []byte) error {
	UDPHeader := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte slice
		(byte)(c.dest >> 8), (byte)(c.dest), // Dest port in byte slice
		(byte)((8 + len(x)) >> 8), (byte)(8 + len(x)), // Length in bytes of UDP header + data
		0, 0, // Checksum
	}

	x = append(UDPHeader, x...)

	h := &ipv4.Header{
		Version:  ipv4.Version,      // protocol version
		Len:      20,                // header length
		TOS:      0,                 // type-of-service (0 is everything normal)
		TotalLen: len(x) + 20,       // packet total length (octets)
		ID:       0,                 // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0,                 // fragment offset
		TTL:      8,                 // time-to-live (maximum lifespan in seconds)
		Protocol: 17,                // next protocol (17 is UDP)
		Checksum: 0,                 // checksum (apparently autocomputed)
		//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
		Dst: net.ParseIP(c.manager.ipAddress), // destination address
		//Options                         // options, extension headers
	}
	err := c.conn.WriteTo(h, x, nil)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}
func (c *UDP) close() error {
	return c.conn.Close()
}
<commit_msg>Minor error<commit_after>package main

import (
	"fmt"
	"golang.org/x/net/ipv4"
	"net"
)

func main() {
	manager, _ := NewUDP_Manager("127.0.0.1")
	c := []*UDP{}
	for i := 0; i < 20; i++ {
		udp, _ := manager.NewUDP(((uint16)(20000 + i)), ((uint16)(20000 + i)))
		c = append(c, udp)
		udp.write([]byte{((byte)(100 + i))})
		a, _ := udp.read(1)
		fmt.Println(a)
	}
	for _, element := range c {
		element.close()
	}
}

type UDP_manager struct {
    ipAddress string
	pl   net.PacketConn
	open bool
	conn *ipv4.RawConn
	buff map[uint16](chan byte)
}

type UDP struct {
    manager   *UDP_manager
	conn      *ipv4.RawConn
	bytes     chan byte
	src, dest uint16
}

func NewUDP_Manager(ip string) (*UDP_manager, error) {
	p, err := net.ListenPacket("ip4:17", ip)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	x := &UDP_manager{open: true, conn: r, pl: p, buff: make(map[uint16](chan byte)), ipAddress: ip}

	go x.readAll()

	return x, nil
}

func (x *UDP_manager) readAll() {
	b := make([]byte, 1024)

	for {
		_, payload, _, err := x.conn.ReadFrom(b)
		if err != nil {
			continue
		}

		dest := (((uint16)(payload[2])) << 8) + ((uint16)(payload[3]))
		//		fmt.Println(dest)
		//		fmt.Println(payload)
		//
		//		fmt.Println(x.buff)
		c, ok := x.buff[dest]
		//fmt.Println(ok)
		payload = payload[8:]
		if ok {
			go func() {
				for _, elem := range payload {
					//fmt.Println("Writing")
					c <- elem
				}
			}()
		}
	}
}

func (x *UDP_manager) NewUDP(src, dest uint16) (*UDP, error) {
	x.buff[src] = make(chan byte, 1024)
	return &UDP{src: src, dest: dest, conn: x.conn, bytes: x.buff[src], manager: x}, nil
}

func (c *UDP) read(size int) ([]byte, error) {
	data := make([]byte, size)
	for i := 0; i < size; i++ {
		//fmt.Println("test")
		data[i] = <-c.bytes
		//fmt.Println(data[i])
	}
	return data, nil
}
func (c *UDP) write(x []byte) error {
	UDPHeader := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte slice
		(byte)(c.dest >> 8), (byte)(c.dest), // Dest port in byte slice
		(byte)((8 + len(x)) >> 8), (byte)(8 + len(x)), // Length in bytes of UDP header + data
		0, 0, // Checksum
	}

	x = append(UDPHeader, x...)

	h := &ipv4.Header{
		Version:  ipv4.Version,      // protocol version
		Len:      20,                // header length
		TOS:      0,                 // type-of-service (0 is everything normal)
		TotalLen: len(x) + 20,       // packet total length (octets)
		ID:       0,                 // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0,                 // fragment offset
		TTL:      8,                 // time-to-live (maximum lifespan in seconds)
		Protocol: 17,                // next protocol (17 is UDP)
		Checksum: 0,                 // checksum (apparently autocomputed)
		//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
		Dst: net.ParseIP(c.manager.ipAddress), // destination address
		//Options                         // options, extension headers
	}
	err := c.conn.WriteTo(h, x, nil)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}
func (c *UDP) close() error {
	return c.conn.Close()
}
<|endoftext|>


<commit_before>package assets

import (
	"bytes"
	"net/http"
	"os"
	"path"
	"time"
)

// An in-memory asset file system. The file system implements the
// http.FileSystem interface.
type FileSystem struct {
	// A map of directory paths to the files in those directories.
	Dirs map[string][]string

	// A map of file/directory paths to assets.File types.
	Files map[string]*File
}

func (f *FileSystem) NewFile(path string, filemode os.FileMode, mtime time.Time, data []byte) *File {
	return &File{
		Path:     path,
		FileMode: filemode,
		Mtime:    mtime,
		Data:     data,

		fs: f,
	}
}

// Implementation of http.FileSystem
func (f *FileSystem) Open(p string) (http.File, error) {
	p = path.Clean(p)

	if fi, ok := f.Files[p]; ok {
		if !fi.IsDir() {
			// Make a copy for reading
			ret := fi
			ret.buf = bytes.NewReader(ret.Data)

			return ret, nil
		}

		return fi, nil
	}

	return nil, os.ErrNotExist
}

func (f *FileSystem) readDir(p string, index int, count int) ([]os.FileInfo, error) {
	if d, ok := f.Dirs[p]; ok {
		maxl := index + count

		if maxl > len(d) {
			maxl = len(d)
		}

		ret := make([]os.FileInfo, 0, maxl-index)

		for i := index; i < maxl; i++ {
			ret = append(ret, f.Files[path.Join(p, d[i])])
		}

		return ret, nil
	}

	return nil, os.ErrNotExist
}
<commit_msg>Add LocaLPath override<commit_after>package assets

import (
	"bytes"
	"net/http"
	"os"
	"path"
	"time"
)

// An in-memory asset file system. The file system implements the
// http.FileSystem interface.
type FileSystem struct {
	// A map of directory paths to the files in those directories.
	Dirs map[string][]string

	// A map of file/directory paths to assets.File types.
	Files map[string]*File

	// Override loading assets from local path. Useful for development.
	LocalPath string
}

func (f *FileSystem) NewFile(path string, filemode os.FileMode, mtime time.Time, data []byte) *File {
	return &File{
		Path:     path,
		FileMode: filemode,
		Mtime:    mtime,
		Data:     data,

		fs: f,
	}
}

// Implementation of http.FileSystem
func (f *FileSystem) Open(p string) (http.File, error) {
	p = path.Clean(p)

	if len(f.LocalPath) != 0 {
		return http.Dir(f.LocalPath).Open(p)
	}

	if fi, ok := f.Files[p]; ok {
		if !fi.IsDir() {
			// Make a copy for reading
			ret := fi
			ret.buf = bytes.NewReader(ret.Data)

			return ret, nil
		}

		return fi, nil
	}

	return nil, os.ErrNotExist
}

func (f *FileSystem) readDir(p string, index int, count int) ([]os.FileInfo, error) {
	if d, ok := f.Dirs[p]; ok {
		maxl := index + count

		if maxl > len(d) {
			maxl = len(d)
		}

		ret := make([]os.FileInfo, 0, maxl-index)

		for i := index; i < maxl; i++ {
			ret = append(ret, f.Files[path.Join(p, d[i])])
		}

		return ret, nil
	}

	return nil, os.ErrNotExist
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"testing"

	api "github.com/osrg/gobgp/api"
	"github.com/osrg/gobgp/config"
	"github.com/osrg/gobgp/server"
	"github.com/stretchr/testify/assert"
)

func TestGetNeighbor(test *testing.T) {
	assert := assert.New(test)
	s := server.NewBgpServer()
	go s.Serve()
	g := api.NewGrpcServer(s, ":50051")
	go g.Serve()
	cli, err := New("")
	err = cli.StartServer(&config.Global{
		Config: config.GlobalConfig{
			As:       1,
			RouterId: "1.1.1.1",
			Port:     1790,
		},
	})
	assert.Nil(err)
	err = cli.AddNeighbor(&config.Neighbor{
		Config: config.NeighborConfig{
			NeighborAddress: "10.0.0.1",
			PeerAs:          2,
		},
	})
	assert.Nil(err)
	_, err = cli.GetNeighbor("10.0.0.1")
	assert.Nil(err)
	_, err = cli.GetNeighbor("10.0.0.2")
	assert.Equal(err.Error(), "not found neighbor 10.0.0.2")
}
<commit_msg>client: fix client_test<commit_after>// Copyright (C) 2016 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"testing"
	"time"

	api "github.com/osrg/gobgp/api"
	"github.com/osrg/gobgp/config"
	"github.com/osrg/gobgp/server"
	"github.com/stretchr/testify/assert"
)

func TestGetNeighbor(test *testing.T) {
	assert := assert.New(test)
	s := server.NewBgpServer()
	go s.Serve()
	g := api.NewGrpcServer(s, ":50051")
	go g.Serve()
	time.Sleep(time.Second)
	cli, err := New("")
	assert.Nil(err)
	err = cli.StartServer(&config.Global{
		Config: config.GlobalConfig{
			As:       1,
			RouterId: "1.1.1.1",
			Port:     1790,
		},
	})
	assert.Nil(err)
	err = cli.AddNeighbor(&config.Neighbor{
		Config: config.NeighborConfig{
			NeighborAddress: "10.0.0.1",
			PeerAs:          2,
		},
	})
	assert.Nil(err)
	_, err = cli.GetNeighbor("10.0.0.1")
	assert.Nil(err)
	_, err = cli.GetNeighbor("10.0.0.2")
	assert.Equal(err.Error(), "not found neighbor 10.0.0.2")
}
<|endoftext|>


<commit_before>package client_test

import (
	"encoding/json"
	"errors"
	"net"
	"net/http"

	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/client"
	"github.com/cloudfoundry-incubator/ducati-daemon/fakes"
	"github.com/cloudfoundry-incubator/ducati-daemon/ipam"
	"github.com/cloudfoundry-incubator/ducati-daemon/models"
	"github.com/onsi/gomega/ghttp"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Client", func() {
	var (
		c           client.DaemonClient
		server      *ghttp.Server
		marshaler   *fakes.Marshaler
		unmarshaler *fakes.Unmarshaler
		container   models.Container
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		marshaler = &fakes.Marshaler{}
		unmarshaler = &fakes.Unmarshaler{}
		c = client.DaemonClient{
			BaseURL:     server.URL(),
			Marshaler:   marshaler,
			Unmarshaler: unmarshaler,
		}

		marshaler.MarshalReturns([]byte(`{"id":"some-container-id"}`), nil)

		container = models.Container{
			ID: "some-container-id",
		}
	})

	AfterEach(func() {
		server.Close()
	})

	Describe("SaveContainer", func() {
		It("should call the backend to save the container", func() {
			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers"),
				ghttp.VerifyJSON(`{"id":"some-container-id"}`),
				ghttp.VerifyHeaderKV("Content-type", "application/json"),
				ghttp.RespondWith(http.StatusCreated, nil),
			))

			Expect(c.SaveContainer(container)).To(Succeed())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
			Expect(marshaler.MarshalCallCount()).To(Equal(1))
			Expect(marshaler.MarshalArgsForCall(0)).To(Equal(container))
		})

		Context("when an error occurs", func() {
			Context("when the container fails to marshal", func() {
				It("should return an error", func() {
					marshaler.MarshalReturns(nil, errors.New("explosion with marshal"))

					err := c.SaveContainer(container)
					Expect(err).To(MatchError("failed to marshal container: explosion with marshal"))
				})
			})

			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c = client.DaemonClient{
						BaseURL:   "%%%%",
						Marshaler: marshaler,
					}

					err := c.SaveContainer(container)
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http request fails", func() {
				It("should return an error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/containers"),
						ghttp.RespondWith(http.StatusInternalServerError, nil),
					))

					err := c.SaveContainer(container)
					Expect(err).To(MatchError(`unexpected status code on SaveContainer: expected 201 but got 500`))
				})
			})
		})
	})

	Describe("RemoveContainer", func() {
		It("should call the backend to remove the container", func() {
			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("DELETE", "/containers/some-container"),
				ghttp.RespondWith(http.StatusNoContent, nil),
			))

			Expect(c.RemoveContainer("some-container")).To(Succeed())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
		})

		Context("when an error occurs", func() {
			Context("when the container does not exist", func() {
				It("it should return a RecordNotFound error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("DELETE", "/containers/non-existent-container-id"),
						ghttp.RespondWith(http.StatusNotFound, nil),
					))

					err := c.RemoveContainer("non-existent-container-id")
					Expect(err).To(Equal(client.RecordNotFoundError))
				})
			})

			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c = client.DaemonClient{
						BaseURL:   "%%%%",
						Marshaler: marshaler,
					}

					err := c.RemoveContainer("whatever")
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http request fails", func() {
				It("should return an error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("DELETE", "/containers/whatever"),
						ghttp.RespondWith(http.StatusInternalServerError, nil),
					))

					err := c.RemoveContainer("whatever")
					Expect(err).To(MatchError(`unexpected status code on RemoveContainer: expected 204 but got 500`))
				})
			})
		})
	})

	Describe("AllocateIP", func() {
		var returnedResult types.Result

		BeforeEach(func() {
			returnedResult = types.Result{
				IP4: &types.IPConfig{
					IP: net.IPNet{
						IP:   net.ParseIP("192.168.100.1"),
						Mask: net.ParseIP("192.168.100.1").DefaultMask(),
					},
					Gateway: net.ParseIP("192.168.100.1"),
					Routes: []types.Route{
						{
							Dst: net.IPNet{
								IP:   net.ParseIP("192.168.1.5"),
								Mask: net.ParseIP("192.168.1.5").DefaultMask(),
							},
							GW: net.ParseIP("192.168.1.1"),
						},
					},
				},
			}

			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/ipam/some-container-id/some-container-id"),
				ghttp.VerifyHeaderKV("Content-type", "application/json"),
				ghttp.RespondWithJSONEncoded(http.StatusCreated, returnedResult),
			))
		})

		It("should call the backend to allocate an IP", func() {
			unmarshaler.UnmarshalStub = json.Unmarshal

			receivedResult, err := c.AllocateIP("some-container-id", "some-container-id")
			Expect(err).NotTo(HaveOccurred())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))

			Expect(receivedResult).To(Equal(returnedResult))
		})

		Context("when an error occurs", func() {
			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c.BaseURL = "%%%%"

					_, err := c.AllocateIP("some-container-id", "some-container-id")
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http response code is a 409 Conflict", func() {
				It("should return an ipam.NoMoreAddressesError", func() {
					server.SetHandler(0, ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/ipam/some-container-id/some-container-id"),
						ghttp.RespondWith(http.StatusConflict, `{ "error": "boom" }`),
					))

					_, err := c.AllocateIP("some-container-id", "some-container-id")
					Expect(err).To(Equal(ipam.NoMoreAddressesError))
				})
			})

			Context("when the http response code is unexpected", func() {
				It("should return an error", func() {
					server.SetHandler(0, ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/ipam/some-container-id/some-container-id"),
						ghttp.RespondWith(http.StatusTeapot, `{{{`),
					))

					_, err := c.AllocateIP("some-container-id", "some-container-id")
					Expect(err).To(MatchError(`unexpected status code on AllocateIP: expected 201 but got 418`))
				})
			})

			Context("when the container fails to marshal", func() {
				It("should return an error", func() {
					unmarshaler.UnmarshalReturns(errors.New("explosion with marshal"))

					_, err := c.AllocateIP("some-container-id", "some-container-id")
					Expect(err).To(MatchError("failed to unmarshal IPAM result: explosion with marshal"))
				})
			})
		})
	})
})
<commit_msg>Fix up fake data in client test<commit_after>package client_test

import (
	"encoding/json"
	"errors"
	"net"
	"net/http"

	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/client"
	"github.com/cloudfoundry-incubator/ducati-daemon/fakes"
	"github.com/cloudfoundry-incubator/ducati-daemon/ipam"
	"github.com/cloudfoundry-incubator/ducati-daemon/models"
	"github.com/onsi/gomega/ghttp"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Client", func() {
	var (
		c           client.DaemonClient
		server      *ghttp.Server
		marshaler   *fakes.Marshaler
		unmarshaler *fakes.Unmarshaler
		container   models.Container
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		marshaler = &fakes.Marshaler{}
		unmarshaler = &fakes.Unmarshaler{}
		c = client.DaemonClient{
			BaseURL:     server.URL(),
			Marshaler:   marshaler,
			Unmarshaler: unmarshaler,
		}

		marshaler.MarshalReturns([]byte(`{"id":"some-container-id"}`), nil)

		container = models.Container{
			ID: "some-container-id",
		}
	})

	AfterEach(func() {
		server.Close()
	})

	Describe("SaveContainer", func() {
		It("should call the backend to save the container", func() {
			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers"),
				ghttp.VerifyJSON(`{"id":"some-container-id"}`),
				ghttp.VerifyHeaderKV("Content-type", "application/json"),
				ghttp.RespondWith(http.StatusCreated, nil),
			))

			Expect(c.SaveContainer(container)).To(Succeed())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
			Expect(marshaler.MarshalCallCount()).To(Equal(1))
			Expect(marshaler.MarshalArgsForCall(0)).To(Equal(container))
		})

		Context("when an error occurs", func() {
			Context("when the container fails to marshal", func() {
				It("should return an error", func() {
					marshaler.MarshalReturns(nil, errors.New("explosion with marshal"))

					err := c.SaveContainer(container)
					Expect(err).To(MatchError("failed to marshal container: explosion with marshal"))
				})
			})

			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c = client.DaemonClient{
						BaseURL:   "%%%%",
						Marshaler: marshaler,
					}

					err := c.SaveContainer(container)
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http request fails", func() {
				It("should return an error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/containers"),
						ghttp.RespondWith(http.StatusInternalServerError, nil),
					))

					err := c.SaveContainer(container)
					Expect(err).To(MatchError(`unexpected status code on SaveContainer: expected 201 but got 500`))
				})
			})
		})
	})

	Describe("RemoveContainer", func() {
		It("should call the backend to remove the container", func() {
			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("DELETE", "/containers/some-container"),
				ghttp.RespondWith(http.StatusNoContent, nil),
			))

			Expect(c.RemoveContainer("some-container")).To(Succeed())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
		})

		Context("when an error occurs", func() {
			Context("when the container does not exist", func() {
				It("it should return a RecordNotFound error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("DELETE", "/containers/non-existent-container-id"),
						ghttp.RespondWith(http.StatusNotFound, nil),
					))

					err := c.RemoveContainer("non-existent-container-id")
					Expect(err).To(Equal(client.RecordNotFoundError))
				})
			})

			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c = client.DaemonClient{
						BaseURL:   "%%%%",
						Marshaler: marshaler,
					}

					err := c.RemoveContainer("whatever")
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http request fails", func() {
				It("should return an error", func() {
					server.AppendHandlers(ghttp.CombineHandlers(
						ghttp.VerifyRequest("DELETE", "/containers/whatever"),
						ghttp.RespondWith(http.StatusInternalServerError, nil),
					))

					err := c.RemoveContainer("whatever")
					Expect(err).To(MatchError(`unexpected status code on RemoveContainer: expected 204 but got 500`))
				})
			})
		})
	})

	Describe("AllocateIP", func() {
		var returnedResult types.Result

		BeforeEach(func() {
			returnedResult = types.Result{
				IP4: &types.IPConfig{
					IP: net.IPNet{
						IP:   net.ParseIP("192.168.100.1"),
						Mask: net.ParseIP("192.168.100.1").DefaultMask(),
					},
					Gateway: net.ParseIP("192.168.100.1"),
					Routes: []types.Route{
						{
							Dst: net.IPNet{
								IP:   net.ParseIP("192.168.1.5"),
								Mask: net.ParseIP("192.168.1.5").DefaultMask(),
							},
							GW: net.ParseIP("192.168.1.1"),
						},
					},
				},
			}

			server.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/ipam/some-network-id/some-container-id"),
				ghttp.VerifyHeaderKV("Content-type", "application/json"),
				ghttp.RespondWithJSONEncoded(http.StatusCreated, returnedResult),
			))
		})

		It("should call the backend to allocate an IP", func() {
			unmarshaler.UnmarshalStub = json.Unmarshal

			receivedResult, err := c.AllocateIP("some-network-id", "some-container-id")
			Expect(err).NotTo(HaveOccurred())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))

			Expect(receivedResult).To(Equal(returnedResult))
		})

		Context("when an error occurs", func() {
			Context("when the request cannot be constructed", func() {
				It("should return an error", func() {
					c.BaseURL = "%%%%"

					_, err := c.AllocateIP("some-network-id", "some-container-id")
					Expect(err).To(MatchError(ContainSubstring("failed to construct request: parse")))
				})
			})

			Context("when the http response code is a 409 Conflict", func() {
				It("should return an ipam.NoMoreAddressesError", func() {
					server.SetHandler(0, ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/ipam/some-network-id/some-container-id"),
						ghttp.RespondWith(http.StatusConflict, `{ "error": "boom" }`),
					))

					_, err := c.AllocateIP("some-network-id", "some-container-id")
					Expect(err).To(Equal(ipam.NoMoreAddressesError))
				})
			})

			Context("when the http response code is unexpected", func() {
				It("should return an error", func() {
					server.SetHandler(0, ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", "/ipam/some-network-id/some-container-id"),
						ghttp.RespondWith(http.StatusTeapot, `{{{`),
					))

					_, err := c.AllocateIP("some-network-id", "some-container-id")
					Expect(err).To(MatchError(`unexpected status code on AllocateIP: expected 201 but got 418`))
				})
			})

			Context("when the container fails to marshal", func() {
				It("should return an error", func() {
					unmarshaler.UnmarshalReturns(errors.New("explosion with marshal"))

					_, err := c.AllocateIP("some-network-id", "some-container-id")
					Expect(err).To(MatchError("failed to unmarshal IPAM result: explosion with marshal"))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa test command
// Author: jonmv

package cmd

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"io/ioutil"
	"math"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

func init() {
	rootCmd.AddCommand(testCmd)
}

var testCmd = &cobra.Command{
	Use:   "test [tests directory or test file]",
	Short: "Run a test suite, or a single test",
	Long: `Run a test suite, or a single test

Runs all JSON test files in the specified directory (the working
directory by default), or the single JSON test file specified.

See https://cloud.vespa.ai/en/reference/testing.html for details.`,
	Example: `$ vespa test src/test/application/tests/system-test
$ vespa test src/test/application/tests/system-test/feed-and-query.json`,
	Args:              cobra.MaximumNArgs(1),
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		target := getTarget()
		testPath := "."
		if len(args) > 0 {
			testPath = args[0]
		}
		if count, failed := runTests(testPath, target, false); len(failed) != 0 {
			fmt.Fprintf(stdout, "\nFailed %d of %d tests:\n", len(failed), count)
			for _, test := range failed {
				fmt.Fprintln(stdout, test)
			}
			exitFunc(3)
		} else {
			plural := "s"
			if count == 1 {
				plural = ""
			}
			fmt.Fprintf(stdout, "\n%d test%s completed successfully\n", count, plural)
		}
	},
}

func runTests(rootPath string, target vespa.Target, dryRun bool) (int, []string) {
	count := 0
	failed := make([]string, 0)
	if stat, err := os.Stat(rootPath); err != nil {
		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
	} else if stat.IsDir() {
		tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required.
		if err != nil {
			fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
		}
		previousFailed := false
		for _, test := range tests {
			if !test.IsDir() && filepath.Ext(test.Name()) == ".json" {
				testPath := path.Join(rootPath, test.Name())
				if previousFailed {
					fmt.Fprintln(stdout, "")
					previousFailed = false
				}
				failure := runTest(testPath, target, dryRun)
				if failure != "" {
					failed = append(failed, failure)
					previousFailed = true
				}
				count++
			}
		}
	} else if strings.HasSuffix(stat.Name(), ".json") {
		failure := runTest(rootPath, target, dryRun)
		if failure != "" {
			failed = append(failed, failure)
		}
		count++
	}
	if count == 0 {
		fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing")
	}
	return count, failed
}

// Runs the test at the given path, and returns the specified test name if the test fails
func runTest(testPath string, target vespa.Target, dryRun bool) string {
	var test test
	testBytes, err := ioutil.ReadFile(testPath)
	if err != nil {
		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
	}
	if err = json.Unmarshal(testBytes, &test); err != nil {
		fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
	}

	testName := test.Name
	if test.Name == "" {
		testName = filepath.Base(testPath)
	}
	if !dryRun {
		fmt.Fprintf(stdout, "Running %s:", testName)
	}

	defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath))
	if err != nil {
		fmt.Fprintln(stderr)
		fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing")
	}

	if len(test.Steps) == 0 {
		fmt.Fprintln(stderr)
		fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
	}
	for i, step := range test.Steps {
		stepName := step.Name
		if stepName == "" {
			stepName = fmt.Sprintf("step %d", i+1)
		}
		failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target, dryRun)
		if err != nil {
			fmt.Fprintln(stderr)
			fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing")
		}
		if !dryRun {
			if failure != "" {
				fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure)
				return fmt.Sprintf("%s: %s: %s", testName, stepName, failure)
			}
			if i == 0 {
				fmt.Fprintf(stdout, " ")
			}
			fmt.Fprint(stdout, ".")
		}
	}
	if !dryRun {
		fmt.Fprintln(stdout, " OK")
	}
	return ""
}

// Asserts specified response is obtained for request, or returns a failure message, or an error if this fails
func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target, dryRun bool) (string, string, error) {
	requestBody, err := getBody(step.Request.BodyRaw, testsPath)
	if err != nil {
		return "", "", err
	}

	parameters, err := getParameters(step.Request.ParametersRaw, testsPath)
	if err != nil {
		return "", "", err
	}
	for name, value := range defaultParameters {
		if _, present := parameters[name]; !present {
			parameters[name] = value
		}
	}

	cluster := step.Request.Cluster
	if cluster == "" {
		cluster = defaultCluster
	}

	var service *vespa.Service
	if !dryRun {
		service, err = target.Service("query", 0, 0, cluster)
		if err != nil {
			return "", "", err
		}
	}

	method := step.Request.Method
	if method == "" {
		method = "GET"
	}

	requestUri := step.Request.URI
	if requestUri == "" {
		requestUri = "/search/"
	}
	requestUrl, err := url.ParseRequestURI(requestUri)
	if err != nil {
		return "", "", err
	}
	externalEndpoint := requestUrl.IsAbs()
	if !externalEndpoint {
		baseURL := "http://dummy/"
		if service != nil {
			baseURL = service.BaseURL
		}
		requestUrl, err = url.ParseRequestURI(baseURL + requestUri)
		if err != nil {
			return "", "", err
		}
	}
	query := requestUrl.Query()
	for name, value := range parameters {
		query.Add(name, value)
	}
	requestUrl.RawQuery = query.Encode()

	header := http.Header{}
	header.Add("Content-Type", "application/json") // TODO: Not guaranteed to be true ...

	request := &http.Request{
		URL:    requestUrl,
		Method: method,
		Header: header,
		Body:   ioutil.NopCloser(bytes.NewReader(requestBody)),
	}
	defer request.Body.Close()

	statusCode := step.Response.Code
	if statusCode == 0 {
		statusCode = 200
	}

	responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath)
	if err != nil {
		return "", "", err
	}
	var responseBodySpec interface{}
	if responseBodySpecBytes != nil {
		err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec)
		if err != nil {
			return "", "", fmt.Errorf("invalid response body spec: %w", err)
		}
	}

	if dryRun {
		return "", "", nil
	}

	var response *http.Response
	if externalEndpoint {
		util.ActiveHttpClient.UseCertificate([]tls.Certificate{})
		response, err = util.ActiveHttpClient.Do(request, 60*time.Second)
	} else {
		response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout
	}
	if err != nil {
		return "", "", err
	}
	defer response.Body.Close()

	if statusCode != response.StatusCode {
		failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode)
		return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil
	}

	if responseBodySpec == nil {
		return "", "", nil
	}

	responseBodyBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", "", err
	}
	var responseBody interface{}
	err = json.Unmarshal(responseBodyBytes, &responseBody)
	if err != nil {
		return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes))
	}

	failure, expected, err := compare(responseBodySpec, responseBody, "")
	if failure != "" {
		responsePretty, _ := json.MarshalIndent(responseBody, "", "  ")
		longFailure := failure
		if expected != "" {
			longFailure += "\n" + expected
		}
		longFailure += "\nActual response:\n" + string(responsePretty)
		return failure, longFailure, err
	}
	return "", "", err
}

func compare(expected interface{}, actual interface{}, path string) (string, string, error) {
	typeMatch := false
	valueMatch := false
	switch u := expected.(type) {
	case nil:
		typeMatch = actual == nil
		valueMatch = actual == nil
	case bool:
		v, ok := actual.(bool)
		typeMatch = ok
		valueMatch = ok && u == v
	case float64:
		v, ok := actual.(float64)
		typeMatch = ok
		valueMatch = ok && math.Abs(u-v) < 1e-9
	case string:
		v, ok := actual.(string)
		typeMatch = ok
		valueMatch = ok && (u == v)
	case []interface{}:
		v, ok := actual.([]interface{})
		typeMatch = ok
		if ok {
			if len(u) == len(v) {
				for i, e := range u {
					failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i))
					if failure != "" || err != nil {
						return failure, expected, err
					}
				}
				valueMatch = true
			} else {
				return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil
			}
		}
	case map[string]interface{}:
		v, ok := actual.(map[string]interface{})
		typeMatch = ok
		if ok {
			for n, e := range u {
				childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1"))
				f, ok := v[n]
				if !ok {
					return fmt.Sprintf("Missing expected field at %s", childPath), "", nil
				}
				failure, expected, err := compare(e, f, childPath)
				if failure != "" || err != nil {
					return failure, expected, err
				}
			}
			valueMatch = true
		}
	default:
		return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected)
	}

	if !valueMatch {
		if path == "" {
			path = "root"
		}
		mismatched := "type"
		if typeMatch {
			mismatched = "value"
		}
		expectedJson, _ := json.Marshal(expected)
		actualJson, _ := json.Marshal(actual)
		return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil
	}
	return "", "", nil
}

func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) {
	if parametersRaw != nil {
		var parametersPath string
		if err := json.Unmarshal(parametersRaw, &parametersPath); err == nil {
			resolvedParametersPath := path.Join(testsPath, parametersPath)
			parametersRaw, err = ioutil.ReadFile(resolvedParametersPath)
			if err != nil {
				return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err)
			}
		}
		var parameters map[string]string
		if err := json.Unmarshal(parametersRaw, &parameters); err != nil {
			return nil, fmt.Errorf("request parameters must be JSON with only string values: %w", err)
		}
		return parameters, nil
	}
	return make(map[string]string), nil
}

func getBody(bodyRaw []byte, testsPath string) ([]byte, error) {
	var bodyPath string
	if err := json.Unmarshal(bodyRaw, &bodyPath); err == nil {
		resolvedBodyPath := path.Join(testsPath, bodyPath)
		bodyRaw, err = ioutil.ReadFile(resolvedBodyPath)
		if err != nil {
			return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err)
		}
	}
	return bodyRaw, nil
}

type test struct {
	Name     string   `json:"name"`
	Defaults defaults `json:"defaults"`
	Steps    []step   `json:"steps"`
}

type defaults struct {
	Cluster       string          `json:"cluster"`
	ParametersRaw json.RawMessage `json:"parameters"`
}

type step struct {
	Name     string   `json:"name"`
	Request  request  `json:"request"`
	Response response `json:"response"`
}

type request struct {
	Cluster       string          `json:"cluster"`
	Method        string          `json:"method"`
	URI           string          `json:"uri"`
	ParametersRaw json.RawMessage `json:"parameters"`
	BodyRaw       json.RawMessage `json:"body"`
}

type response struct {
	Code    int             `json:"code"`
	BodyRaw json.RawMessage `json:"body"`
}
<commit_msg>Add zone flag to test scommand<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa test command
// Author: jonmv

package cmd

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/spf13/cobra"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"io/ioutil"
	"math"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

func init() {
	rootCmd.AddCommand(testCmd)
	testCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "dev.aws-us-east-1c", "The zone to use for deployment")
}

var testCmd = &cobra.Command{
	Use:   "test [tests directory or test file]",
	Short: "Run a test suite, or a single test",
	Long: `Run a test suite, or a single test

Runs all JSON test files in the specified directory (the working
directory by default), or the single JSON test file specified.

See https://cloud.vespa.ai/en/reference/testing.html for details.`,
	Example: `$ vespa test src/test/application/tests/system-test
$ vespa test src/test/application/tests/system-test/feed-and-query.json`,
	Args:              cobra.MaximumNArgs(1),
	DisableAutoGenTag: true,
	Run: func(cmd *cobra.Command, args []string) {
		target := getTarget()
		testPath := "."
		if len(args) > 0 {
			testPath = args[0]
		}
		if count, failed := runTests(testPath, target, false); len(failed) != 0 {
			fmt.Fprintf(stdout, "\nFailed %d of %d tests:\n", len(failed), count)
			for _, test := range failed {
				fmt.Fprintln(stdout, test)
			}
			exitFunc(3)
		} else {
			plural := "s"
			if count == 1 {
				plural = ""
			}
			fmt.Fprintf(stdout, "\n%d test%s completed successfully\n", count, plural)
		}
	},
}

func runTests(rootPath string, target vespa.Target, dryRun bool) (int, []string) {
	count := 0
	failed := make([]string, 0)
	if stat, err := os.Stat(rootPath); err != nil {
		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
	} else if stat.IsDir() {
		tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required.
		if err != nil {
			fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
		}
		previousFailed := false
		for _, test := range tests {
			if !test.IsDir() && filepath.Ext(test.Name()) == ".json" {
				testPath := path.Join(rootPath, test.Name())
				if previousFailed {
					fmt.Fprintln(stdout, "")
					previousFailed = false
				}
				failure := runTest(testPath, target, dryRun)
				if failure != "" {
					failed = append(failed, failure)
					previousFailed = true
				}
				count++
			}
		}
	} else if strings.HasSuffix(stat.Name(), ".json") {
		failure := runTest(rootPath, target, dryRun)
		if failure != "" {
			failed = append(failed, failure)
		}
		count++
	}
	if count == 0 {
		fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing")
	}
	return count, failed
}

// Runs the test at the given path, and returns the specified test name if the test fails
func runTest(testPath string, target vespa.Target, dryRun bool) string {
	var test test
	testBytes, err := ioutil.ReadFile(testPath)
	if err != nil {
		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
	}
	if err = json.Unmarshal(testBytes, &test); err != nil {
		fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
	}

	testName := test.Name
	if test.Name == "" {
		testName = filepath.Base(testPath)
	}
	if !dryRun {
		fmt.Fprintf(stdout, "Running %s:", testName)
	}

	defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath))
	if err != nil {
		fmt.Fprintln(stderr)
		fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing")
	}

	if len(test.Steps) == 0 {
		fmt.Fprintln(stderr)
		fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
	}
	for i, step := range test.Steps {
		stepName := step.Name
		if stepName == "" {
			stepName = fmt.Sprintf("step %d", i+1)
		}
		failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target, dryRun)
		if err != nil {
			fmt.Fprintln(stderr)
			fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing")
		}
		if !dryRun {
			if failure != "" {
				fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure)
				return fmt.Sprintf("%s: %s: %s", testName, stepName, failure)
			}
			if i == 0 {
				fmt.Fprintf(stdout, " ")
			}
			fmt.Fprint(stdout, ".")
		}
	}
	if !dryRun {
		fmt.Fprintln(stdout, " OK")
	}
	return ""
}

// Asserts specified response is obtained for request, or returns a failure message, or an error if this fails
func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target, dryRun bool) (string, string, error) {
	requestBody, err := getBody(step.Request.BodyRaw, testsPath)
	if err != nil {
		return "", "", err
	}

	parameters, err := getParameters(step.Request.ParametersRaw, testsPath)
	if err != nil {
		return "", "", err
	}
	for name, value := range defaultParameters {
		if _, present := parameters[name]; !present {
			parameters[name] = value
		}
	}

	cluster := step.Request.Cluster
	if cluster == "" {
		cluster = defaultCluster
	}

	var service *vespa.Service
	if !dryRun {
		service, err = target.Service("query", 0, 0, cluster)
		if err != nil {
			return "", "", err
		}
	}

	method := step.Request.Method
	if method == "" {
		method = "GET"
	}

	requestUri := step.Request.URI
	if requestUri == "" {
		requestUri = "/search/"
	}
	requestUrl, err := url.ParseRequestURI(requestUri)
	if err != nil {
		return "", "", err
	}
	externalEndpoint := requestUrl.IsAbs()
	if !externalEndpoint {
		baseURL := "http://dummy/"
		if service != nil {
			baseURL = service.BaseURL
		}
		requestUrl, err = url.ParseRequestURI(baseURL + requestUri)
		if err != nil {
			return "", "", err
		}
	}
	query := requestUrl.Query()
	for name, value := range parameters {
		query.Add(name, value)
	}
	requestUrl.RawQuery = query.Encode()

	header := http.Header{}
	header.Add("Content-Type", "application/json") // TODO: Not guaranteed to be true ...

	request := &http.Request{
		URL:    requestUrl,
		Method: method,
		Header: header,
		Body:   ioutil.NopCloser(bytes.NewReader(requestBody)),
	}
	defer request.Body.Close()

	statusCode := step.Response.Code
	if statusCode == 0 {
		statusCode = 200
	}

	responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath)
	if err != nil {
		return "", "", err
	}
	var responseBodySpec interface{}
	if responseBodySpecBytes != nil {
		err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec)
		if err != nil {
			return "", "", fmt.Errorf("invalid response body spec: %w", err)
		}
	}

	if dryRun {
		return "", "", nil
	}

	var response *http.Response
	if externalEndpoint {
		util.ActiveHttpClient.UseCertificate([]tls.Certificate{})
		response, err = util.ActiveHttpClient.Do(request, 60*time.Second)
	} else {
		response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout
	}
	if err != nil {
		return "", "", err
	}
	defer response.Body.Close()

	if statusCode != response.StatusCode {
		failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode)
		return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil
	}

	if responseBodySpec == nil {
		return "", "", nil
	}

	responseBodyBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", "", err
	}
	var responseBody interface{}
	err = json.Unmarshal(responseBodyBytes, &responseBody)
	if err != nil {
		return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes))
	}

	failure, expected, err := compare(responseBodySpec, responseBody, "")
	if failure != "" {
		responsePretty, _ := json.MarshalIndent(responseBody, "", "  ")
		longFailure := failure
		if expected != "" {
			longFailure += "\n" + expected
		}
		longFailure += "\nActual response:\n" + string(responsePretty)
		return failure, longFailure, err
	}
	return "", "", err
}

func compare(expected interface{}, actual interface{}, path string) (string, string, error) {
	typeMatch := false
	valueMatch := false
	switch u := expected.(type) {
	case nil:
		typeMatch = actual == nil
		valueMatch = actual == nil
	case bool:
		v, ok := actual.(bool)
		typeMatch = ok
		valueMatch = ok && u == v
	case float64:
		v, ok := actual.(float64)
		typeMatch = ok
		valueMatch = ok && math.Abs(u-v) < 1e-9
	case string:
		v, ok := actual.(string)
		typeMatch = ok
		valueMatch = ok && (u == v)
	case []interface{}:
		v, ok := actual.([]interface{})
		typeMatch = ok
		if ok {
			if len(u) == len(v) {
				for i, e := range u {
					failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i))
					if failure != "" || err != nil {
						return failure, expected, err
					}
				}
				valueMatch = true
			} else {
				return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil
			}
		}
	case map[string]interface{}:
		v, ok := actual.(map[string]interface{})
		typeMatch = ok
		if ok {
			for n, e := range u {
				childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1"))
				f, ok := v[n]
				if !ok {
					return fmt.Sprintf("Missing expected field at %s", childPath), "", nil
				}
				failure, expected, err := compare(e, f, childPath)
				if failure != "" || err != nil {
					return failure, expected, err
				}
			}
			valueMatch = true
		}
	default:
		return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected)
	}

	if !valueMatch {
		if path == "" {
			path = "root"
		}
		mismatched := "type"
		if typeMatch {
			mismatched = "value"
		}
		expectedJson, _ := json.Marshal(expected)
		actualJson, _ := json.Marshal(actual)
		return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil
	}
	return "", "", nil
}

func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) {
	if parametersRaw != nil {
		var parametersPath string
		if err := json.Unmarshal(parametersRaw, &parametersPath); err == nil {
			resolvedParametersPath := path.Join(testsPath, parametersPath)
			parametersRaw, err = ioutil.ReadFile(resolvedParametersPath)
			if err != nil {
				return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err)
			}
		}
		var parameters map[string]string
		if err := json.Unmarshal(parametersRaw, &parameters); err != nil {
			return nil, fmt.Errorf("request parameters must be JSON with only string values: %w", err)
		}
		return parameters, nil
	}
	return make(map[string]string), nil
}

func getBody(bodyRaw []byte, testsPath string) ([]byte, error) {
	var bodyPath string
	if err := json.Unmarshal(bodyRaw, &bodyPath); err == nil {
		resolvedBodyPath := path.Join(testsPath, bodyPath)
		bodyRaw, err = ioutil.ReadFile(resolvedBodyPath)
		if err != nil {
			return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err)
		}
	}
	return bodyRaw, nil
}

type test struct {
	Name     string   `json:"name"`
	Defaults defaults `json:"defaults"`
	Steps    []step   `json:"steps"`
}

type defaults struct {
	Cluster       string          `json:"cluster"`
	ParametersRaw json.RawMessage `json:"parameters"`
}

type step struct {
	Name     string   `json:"name"`
	Request  request  `json:"request"`
	Response response `json:"response"`
}

type request struct {
	Cluster       string          `json:"cluster"`
	Method        string          `json:"method"`
	URI           string          `json:"uri"`
	ParametersRaw json.RawMessage `json:"parameters"`
	BodyRaw       json.RawMessage `json:"body"`
}

type response struct {
	Code    int             `json:"code"`
	BodyRaw json.RawMessage `json:"body"`
}
<|endoftext|>


<commit_before>// Copyright 2019 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin

package metal

import (
	"sync"

	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal/ca"
	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal/mtl"
)

type view struct {
	window uintptr
	uiview uintptr

	windowChanged bool

	device mtl.Device
	ml     ca.MetalLayer

	once sync.Once
}

func (v *view) setDrawableSize(width, height int) {
	v.ml.SetDrawableSize(width, height)
}

func (v *view) getMTLDevice() mtl.Device {
	return v.device
}

func (v *view) setDisplaySyncEnabled(enabled bool) {
	// TODO: Now SetVsyncEnabled is called only from the main thread, and d.t.Run is not available since
	// recursive function call via Run is forbidden.
	// Fix this to use d.t.Run to avoid confusion.
	v.ml.SetDisplaySyncEnabled(enabled)
}

func (v *view) colorPixelFormat() mtl.PixelFormat {
	return v.ml.PixelFormat()
}

func (v *view) reset() error {
	var err error
	v.device, err = mtl.CreateSystemDefaultDevice()
	if err != nil {
		return err
	}

	v.ml = ca.MakeMetalLayer()
	v.ml.SetDevice(v.device)
	// https://developer.apple.com/documentation/quartzcore/cametallayer/1478155-pixelformat
	//
	// The pixel format for a Metal layer must be MTLPixelFormatBGRA8Unorm,
	// MTLPixelFormatBGRA8Unorm_sRGB, MTLPixelFormatRGBA16Float, MTLPixelFormatBGRA10_XR, or
	// MTLPixelFormatBGRA10_XR_sRGB.
	v.ml.SetPixelFormat(mtl.PixelFormatBGRA8UNorm)
	v.ml.SetMaximumDrawableCount(3)
	return nil
}

func (v *view) drawable() ca.MetalDrawable {
	d, err := v.ml.NextDrawable()
	if err != nil {
		// Drawable is nil. This can happen at the initial state. Let's wait and see.
		return ca.MetalDrawable{}
	}
	return d
}
<commit_msg>graphicsdriver/metal: Remove an old comment<commit_after>// Copyright 2019 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin

package metal

import (
	"sync"

	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal/ca"
	"github.com/hajimehoshi/ebiten/v2/internal/graphicsdriver/metal/mtl"
)

type view struct {
	window uintptr
	uiview uintptr

	windowChanged bool

	device mtl.Device
	ml     ca.MetalLayer

	once sync.Once
}

func (v *view) setDrawableSize(width, height int) {
	v.ml.SetDrawableSize(width, height)
}

func (v *view) getMTLDevice() mtl.Device {
	return v.device
}

func (v *view) setDisplaySyncEnabled(enabled bool) {
	v.ml.SetDisplaySyncEnabled(enabled)
}

func (v *view) colorPixelFormat() mtl.PixelFormat {
	return v.ml.PixelFormat()
}

func (v *view) reset() error {
	var err error
	v.device, err = mtl.CreateSystemDefaultDevice()
	if err != nil {
		return err
	}

	v.ml = ca.MakeMetalLayer()
	v.ml.SetDevice(v.device)
	// https://developer.apple.com/documentation/quartzcore/cametallayer/1478155-pixelformat
	//
	// The pixel format for a Metal layer must be MTLPixelFormatBGRA8Unorm,
	// MTLPixelFormatBGRA8Unorm_sRGB, MTLPixelFormatRGBA16Float, MTLPixelFormatBGRA10_XR, or
	// MTLPixelFormatBGRA10_XR_sRGB.
	v.ml.SetPixelFormat(mtl.PixelFormatBGRA8UNorm)
	v.ml.SetMaximumDrawableCount(3)
	return nil
}

func (v *view) drawable() ca.MetalDrawable {
	d, err := v.ml.NextDrawable()
	if err != nil {
		// Drawable is nil. This can happen at the initial state. Let's wait and see.
		return ca.MetalDrawable{}
	}
	return d
}
<|endoftext|>


<commit_before>package stats

import "fmt"
import "math"

type Stats struct {
	Avg float64
	Min float64
	Max float64
	P50 float64
	P75 float64
	P90 float64
	P95 float64
	P99 float64
}

const msFactor = 1000000

// Accepts a sorted slice of durations in nanoseconds
// Returns a Stats struct of millisecond statistics
func Get(data []int) Stats {
	min, max := minmax(data)

	return Stats{
		Avg: avg(data) / msFactor,
		Min: min / msFactor,
		Max: max / msFactor,
		P50: p(data, 0.5) / msFactor,
		P75: p(data, 0.75) / msFactor,
		P90: p(data, 0.9) / msFactor,
		P95: p(data, 0.95) / msFactor,
		P99: p(data, 0.99) / msFactor,
	}
}

func minmax(data []int) (float64, float64) {
	min := int(math.MaxInt64)
	max := int(math.MinInt64)
	for _, d := range data {
		if d < min {
			min = d
		}
		if d > max {
			max = d
		}
	}
	return float64(min), float64(max)
}

func avg(data []int) float64 {
	sum := float64(0)
	for _, d := range data {
		sum += float64(d)
	}
	return sum / float64(len(data))
}

func p(data []int, p float64) float64 {
	idx := pIdx(data, p)
	return float64(data[idx])
}

func pIdx(data []int, p float64) int {
	return int(math.Ceil(float64(len(data)) * p))
}

const numBuckets = 100
const maxHeight = 50

// Accepts a sorted slice of durations in nanoseconds
// Prints a histogram to stdout
func PrintHist(data []int) {
	// Cut the data at the 99th percentile
	p99Idx := pIdx(data, 0.99)
	data = data[:p99Idx]

	buckets := make([]int, numBuckets)
	min := data[0]
	max := data[len(data)-1]
	step := float64(max-min) / numBuckets
	prevCutIdx := 0
	maxBucket := 0

	for i := 0; i < numBuckets; i++ {
		cut := float64(min) + step*float64(i+1)
		count := 0
		j := prevCutIdx

		for ; j < len(data) && float64(data[j]) < cut; j++ {
			count++
		}

		prevCutIdx = j
		buckets[i] = count

		if count > maxBucket {
			maxBucket = count
		}
	}

	// Scale the graph to a reasonable maximum height
	heightRatio := float64(maxHeight) / float64(maxBucket)
	for i := 0; i < len(buckets); i++ {
		buckets[i] = int(math.Ceil(float64(buckets[i]) * heightRatio))
	}

	// Scan downward over the histogram printing line by line
	hist := make([]rune, 0)
	for i := maxHeight; i >= 0; i-- {
		for j := 0; j < numBuckets; j++ {
			if i == 0 {
				hist = append(hist, '=')
			} else if buckets[j] == i {
				hist = append(hist, '|')
				buckets[j]--
			} else {
				hist = append(hist, ' ')
			}
		}

		hist = append(hist, '\n')
	}

	gmin := float64(min) / msFactor
	gmax := float64(max) / msFactor
	gmid := gmin + (gmax-gmin)/2

	pointerRow := make([]rune, numBuckets)

	for i := 1; i < numBuckets-1; i++ {
		pointerRow[i] = ' '
	}

	pointerRow[0] = '^'
	pointerRow[len(pointerRow)/2] = '^'
	pointerRow[len(pointerRow)-1] = '^'

	fmt.Print(string(hist))
	fmt.Println(string(pointerRow))
	fmt.Printf("%.4fms                                       %.4fms                                     %.4fms\n", gmin, gmid, gmax)
}
<commit_msg>client stats: Adding pointer to maximum bucket, fixing a small display bug<commit_after>package stats

import "fmt"
import "math"

type Stats struct {
	Avg float64
	Min float64
	Max float64
	P50 float64
	P75 float64
	P90 float64
	P95 float64
	P99 float64
}

const msFactor = 1000000

// Accepts a sorted slice of durations in nanoseconds
// Returns a Stats struct of millisecond statistics
func Get(data []int) Stats {
	min, max := minmax(data)

	return Stats{
		Avg: avg(data) / msFactor,
		Min: min / msFactor,
		Max: max / msFactor,
		P50: p(data, 0.5) / msFactor,
		P75: p(data, 0.75) / msFactor,
		P90: p(data, 0.9) / msFactor,
		P95: p(data, 0.95) / msFactor,
		P99: p(data, 0.99) / msFactor,
	}
}

func minmax(data []int) (float64, float64) {
	min := int(math.MaxInt64)
	max := int(math.MinInt64)
	for _, d := range data {
		if d < min {
			min = d
		}
		if d > max {
			max = d
		}
	}
	return float64(min), float64(max)
}

func avg(data []int) float64 {
	sum := float64(0)
	for _, d := range data {
		sum += float64(d)
	}
	return sum / float64(len(data))
}

func p(data []int, p float64) float64 {
	idx := pIdx(data, p)
	return float64(data[idx])
}

func pIdx(data []int, p float64) int {
	return int(math.Ceil(float64(len(data)) * p))
}

const numBuckets = 100
const maxHeight = 50

// Accepts a sorted slice of durations in nanoseconds
// Prints a histogram to stdout
func PrintHist(data []int) {
	// Cut the data at the 99th percentile
	p99Idx := pIdx(data, 0.99)
	data = data[:p99Idx]

	buckets := make([]int, numBuckets)
	min := data[0]
	max := data[len(data)-1]
	step := float64(max-min) / numBuckets
	prevCutIdx := 0
	maxBucket := 0
	maxBucketIdx := 0

	for i := 0; i < numBuckets; i++ {
		cut := float64(min) + step*float64(i+1)
		count := 0
		j := prevCutIdx

		for ; j < len(data) && float64(data[j]) < cut; j++ {
			count++
		}

		prevCutIdx = j
		buckets[i] = count

		if count > maxBucket {
			maxBucket = count
			maxBucketIdx = i
		}
	}

	topBucketFmt := fmt.Sprintf("%%%vd\n", maxBucketIdx + 3)
	topPointerRow := make([]rune, numBuckets)

	for i := 1; i < numBuckets-1; i++ {
		topPointerRow[i] = ' '
	}

	topPointerRow[maxBucketIdx+1] = 'v'

	// Scale the graph to a reasonable maximum height
	heightRatio := float64(maxHeight) / float64(maxBucket)
	for i := 0; i < len(buckets); i++ {
		buckets[i] = int(math.Min(float64(maxHeight), math.Ceil(float64(buckets[i]) * heightRatio)))
	}

	// Scan downward over the histogram printing line by line
	hist := make([]rune, 0)
	for i := maxHeight; i >= 0; i-- {
		for j := 0; j < numBuckets; j++ {
			if i == 0 {
				hist = append(hist, '=')
			} else if buckets[j] == i {
				hist = append(hist, '|')
				buckets[j]--
			} else {
				hist = append(hist, ' ')
			}
		}

		hist = append(hist, '\n')
	}

	gmin := float64(min) / msFactor
	gmax := float64(max) / msFactor
	gmid := gmin + (gmax-gmin)/2

	pointerRow := make([]rune, numBuckets)

	for i := 1; i < numBuckets-1; i++ {
		pointerRow[i] = ' '
	}

	pointerRow[0] = '^'
	pointerRow[len(pointerRow)/2] = '^'
	pointerRow[len(pointerRow)-1] = '^'

	fmt.Printf(topBucketFmt, maxBucket)
	fmt.Println(string(topPointerRow))
	fmt.Print(string(hist))
	fmt.Println(string(pointerRow))
	fmt.Printf("%.4fms                                       %.4fms                                     %.4fms\n", gmin, gmid, gmax)
}
<|endoftext|>


<commit_before>package postgres

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"time"

	"go.uber.org/zap"
)

var Timeout = errors.New("postgres connection timeout")

type Opts struct {
	Logger         *zap.Logger
	Retry, Timeout time.Duration
}

// Init ...
func Init(address string, opts Opts) (*sql.DB, error) {
	timeout, retry := opts.Timeout, opts.Retry
	if timeout == time.Duration(0) {
		timeout = 10 * time.Second
	}
	if retry == time.Duration(0) {
		retry = 1 * time.Second
	}

	u, err := url.Parse(address)
	if err != nil {
		return nil, err
	}
	username := ""
	if u.User != nil {
		username = u.User.Username()
	}

	opts.Logger.Debug("postgres connection attempt", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

	db, err := sql.Open("postgres", address)
	if err != nil {
		return nil, fmt.Errorf("postgres connection failure: %s", err.Error())
	}

	// Otherwise 1 second cooldown is going to be multiplied by number of tests.
	ctx, cancel := context.WithTimeout(context.Background(), retry)
	defer cancel()

	if err := db.PingContext(ctx); err != nil {
		cancel := time.NewTimer(timeout)

	PingLoop:
		for {
			select {
			case <-time.After(retry):
				ctx, cancel := context.WithTimeout(context.Background(), retry)
				if err := db.PingContext(ctx); err != nil {
					opts.Logger.Debug("postgres connection ping failure", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

					cancel()
					continue PingLoop
				}
				opts.Logger.Info("postgres connection has been established", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

				cancel()
				break PingLoop
			case <-cancel.C:
				return nil, Timeout
			}
		}
	}

	opts.Logger.Info("postgres connection has been established", zap.String("address", address))

	return db, nil
}
<commit_msg>postgres credentials are no more visible in logs<commit_after>package postgres

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"time"

	"go.uber.org/zap"
)

var Timeout = errors.New("postgres connection timeout")

type Opts struct {
	Logger         *zap.Logger
	Retry, Timeout time.Duration
}

// Init ...
func Init(address string, opts Opts) (*sql.DB, error) {
	timeout, retry := opts.Timeout, opts.Retry
	if timeout == time.Duration(0) {
		timeout = 10 * time.Second
	}
	if retry == time.Duration(0) {
		retry = 1 * time.Second
	}

	u, err := url.Parse(address)
	if err != nil {
		return nil, err
	}
	username := ""
	if u.User != nil {
		username = u.User.Username()
	}

	opts.Logger.Debug("postgres connection attempt", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

	db, err := sql.Open("postgres", address)
	if err != nil {
		return nil, fmt.Errorf("postgres connection failure: %s", err.Error())
	}

	// Otherwise 1 second cooldown is going to be multiplied by number of tests.
	ctx, cancel := context.WithTimeout(context.Background(), retry)
	defer cancel()

	if err := db.PingContext(ctx); err != nil {
		cancel := time.NewTimer(timeout)

	PingLoop:
		for {
			select {
			case <-time.After(retry):
				ctx, cancel := context.WithTimeout(context.Background(), retry)
				if err := db.PingContext(ctx); err != nil {
					opts.Logger.Debug("postgres connection ping failure", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

					cancel()
					continue PingLoop
				}
				opts.Logger.Info("postgres connection has been established", zap.String("postgres_host", u.Host), zap.String("postgres_user", username))

				cancel()
				break PingLoop
			case <-cancel.C:
				return nil, Timeout
			}
		}
	}

	return db, nil
}
<|endoftext|>


<commit_before>// +build !local_tests

package sessiontest

import (
	"testing"

	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/test"
	"github.com/privacybydesign/irmago/irmaclient"
	"github.com/stretchr/testify/require"
)

func TestManualKeyshareSession(t *testing.T) {
	request := irma.NewSignatureRequest("I owe you everything", irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"))
	ms := createManualSessionHandler(t, nil)

	_, status := manualSessionHelper(t, nil, ms, request, request, false)
	require.Equal(t, irma.ProofStatusValid, status)
	_, status = manualSessionHelper(t, nil, ms, request, nil, false)
	require.Equal(t, irma.ProofStatusValid, status)
}

func TestRequestorIssuanceKeyshareSession(t *testing.T) {
	testRequestorIssuance(t, true, nil)
}

func TestKeyshareRegister(t *testing.T) {
	client, handler := parseStorage(t)
	defer test.ClearTestStorage(t, handler.storage)

	require.NoError(t, client.KeyshareRemoveAll())
	require.NoError(t, client.RemoveStorage())

	client.SetPreferences(irmaclient.Preferences{DeveloperMode: true})
	client.KeyshareEnroll(irma.NewSchemeManagerIdentifier("test"), nil, "12345", "en")
	require.NoError(t, <-handler.c)

	require.Len(t, client.CredentialInfoList(), 1)

	sessionHelper(t, getIssuanceRequest(true), "issue", client)
	keyshareSessions(t, client)
}

// Use the existing keyshare enrollment and credentials
// in a keyshare session of each session type.
// Use keyshareuser.sql to enroll the user at the keyshare server.
func TestKeyshareSessions(t *testing.T) {
	client, handler := parseStorage(t)
	defer test.ClearTestStorage(t, handler.storage)
	keyshareSessions(t, client)
}

func keyshareSessions(t *testing.T, client *irmaclient.Client) {
	id := irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")
	expiry := irma.Timestamp(irma.NewMetadataAttribute(0).Expiry())
	issuanceRequest := getCombinedIssuanceRequest(id)
	issuanceRequest.Credentials = append(issuanceRequest.Credentials,
		&irma.CredentialRequest{
			Validity:         &expiry,
			CredentialTypeID: irma.NewCredentialTypeIdentifier("test.test.mijnirma"),
			Attributes:       map[string]string{"email": "testusername"},
		},
	)
	sessionHelper(t, issuanceRequest, "issue", client)

	disclosureRequest := getDisclosureRequest(id)
	disclosureRequest.AddSingle(irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"), nil, nil)
	sessionHelper(t, disclosureRequest, "verification", client)

	sigRequest := getSigningRequest(id)
	sigRequest.AddSingle(irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"), nil, nil)
	sessionHelper(t, sigRequest, "signature", client)
}

func TestIssuanceCombinedMultiSchemeSession(t *testing.T) {
	id := irma.NewAttributeTypeIdentifier("test.test.mijnirma.email")
	request := getCombinedIssuanceRequest(id)
	sessionHelper(t, request, "issue", nil)

	sessionHelper(t, irma.NewIssuanceRequest([]*irma.CredentialRequest{
		{
			CredentialTypeID: irma.NewCredentialTypeIdentifier("test.test.mijnirma"),
			Attributes: map[string]string{
				"email": "example@example.com",
			},
		},
	}, irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")), "issue", nil)
}

func TestKeyshareRevocation(t *testing.T) {
	t.Run("Keyshare", func(t *testing.T) {
		startRevocationServer(t, true)
		defer stopRevocationServer()
		client, handler := parseStorage(t)
		defer test.ClearTestStorage(t, handler.storage)

		testRevocation(t, revKeyshareTestAttr, client, handler)
	})

	t.Run("Both", func(t *testing.T) {
		startRevocationServer(t, true)
		defer stopRevocationServer()
		client, handler := parseStorage(t)
		defer test.ClearTestStorage(t, handler.storage)

		testRevocation(t, revKeyshareTestAttr, client, handler)
		testRevocation(t, revocationTestAttr, client, handler)
	})
}
<commit_msg>test: reuse IRMA server in keyshare session tests for efficiency<commit_after>// +build !local_tests

package sessiontest

import (
	"testing"

	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/test"
	"github.com/privacybydesign/irmago/irmaclient"
	"github.com/stretchr/testify/require"
)

func TestManualKeyshareSession(t *testing.T) {
	request := irma.NewSignatureRequest("I owe you everything", irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"))
	ms := createManualSessionHandler(t, nil)

	_, status := manualSessionHelper(t, nil, ms, request, request, false)
	require.Equal(t, irma.ProofStatusValid, status)
	_, status = manualSessionHelper(t, nil, ms, request, nil, false)
	require.Equal(t, irma.ProofStatusValid, status)
}

func TestRequestorIssuanceKeyshareSession(t *testing.T) {
	testRequestorIssuance(t, true, nil)
}

func TestKeyshareRegister(t *testing.T) {
	client, handler := parseStorage(t)
	defer test.ClearTestStorage(t, handler.storage)

	require.NoError(t, client.KeyshareRemoveAll())
	require.NoError(t, client.RemoveStorage())

	client.SetPreferences(irmaclient.Preferences{DeveloperMode: true})
	client.KeyshareEnroll(irma.NewSchemeManagerIdentifier("test"), nil, "12345", "en")
	require.NoError(t, <-handler.c)

	require.Len(t, client.CredentialInfoList(), 1)

	StartIrmaServer(t, false, "")
	defer StopIrmaServer()

	requestorSessionHelper(t, getIssuanceRequest(true), client, sessionOptionReuseServer)
	keyshareSessions(t, client)
}

// Use the existing keyshare enrollment and credentials
// in a keyshare session of each session type.
// Use keyshareuser.sql to enroll the user at the keyshare server.
func TestKeyshareSessions(t *testing.T) {
	client, handler := parseStorage(t)
	defer test.ClearTestStorage(t, handler.storage)
	StartIrmaServer(t, false, "")
	defer StopIrmaServer()
	keyshareSessions(t, client)
}

func keyshareSessions(t *testing.T, client *irmaclient.Client) {
	id := irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")
	expiry := irma.Timestamp(irma.NewMetadataAttribute(0).Expiry())
	issuanceRequest := getCombinedIssuanceRequest(id)
	issuanceRequest.Credentials = append(issuanceRequest.Credentials,
		&irma.CredentialRequest{
			Validity:         &expiry,
			CredentialTypeID: irma.NewCredentialTypeIdentifier("test.test.mijnirma"),
			Attributes:       map[string]string{"email": "testusername"},
		},
	)
	requestorSessionHelper(t, issuanceRequest, client, sessionOptionReuseServer)

	disclosureRequest := getDisclosureRequest(id)
	disclosureRequest.AddSingle(irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"), nil, nil)
	requestorSessionHelper(t, disclosureRequest, client, sessionOptionReuseServer)

	sigRequest := getSigningRequest(id)
	sigRequest.AddSingle(irma.NewAttributeTypeIdentifier("test.test.mijnirma.email"), nil, nil)
	requestorSessionHelper(t, sigRequest, client, sessionOptionReuseServer)
}

func TestIssuanceCombinedMultiSchemeSession(t *testing.T) {
	StartIrmaServer(t, false, "")
	defer StopIrmaServer()

	id := irma.NewAttributeTypeIdentifier("test.test.mijnirma.email")
	request := getCombinedIssuanceRequest(id)
	requestorSessionHelper(t, request, nil, sessionOptionReuseServer)

	id = irma.NewAttributeTypeIdentifier("irma-demo.RU.studentCard.studentID")
	request = irma.NewIssuanceRequest([]*irma.CredentialRequest{
		{
			CredentialTypeID: irma.NewCredentialTypeIdentifier("test.test.mijnirma"),
			Attributes: map[string]string{
				"email": "example@example.com",
			},
		},
	}, id)
	requestorSessionHelper(t, request, nil, sessionOptionReuseServer)
}

func TestKeyshareRevocation(t *testing.T) {
	t.Run("Keyshare", func(t *testing.T) {
		startRevocationServer(t, true)
		defer stopRevocationServer()
		client, handler := parseStorage(t)
		defer test.ClearTestStorage(t, handler.storage)

		testRevocation(t, revKeyshareTestAttr, client, handler)
	})

	t.Run("Both", func(t *testing.T) {
		startRevocationServer(t, true)
		defer stopRevocationServer()
		client, handler := parseStorage(t)
		defer test.ClearTestStorage(t, handler.storage)

		testRevocation(t, revKeyshareTestAttr, client, handler)
		testRevocation(t, revocationTestAttr, client, handler)
	})
}
<|endoftext|>