<commit_before>package main

import (
	"fmt"
	"strings"
	"testing"

	mkr "github.com/mackerelio/mackerel-client-go"
)

func TestIsSameMonitor(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}
	b := &mkr.Monitor{Name: "foo", Type: "connectivity"}

	_, ret := isSameMonitor(a, b, true)
	if ret != true {
		t.Error("should recognize same monitors")
	}

	_, ret = isSameMonitor(a, b, false)
	if ret == true {
		t.Error("should not recognize same monitors")
	}
}

func TestValidateRoles(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}

	ret, err := validateRules([](*mkr.Monitor){a}, "test monitor")
	if ret != true {
		t.Errorf("should validate the rule: %s", err.Error())
	}

}

func TestDiffMonitors(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "external", URL: "http://example.com", Service: "bar", ResponseTimeCritical: 1000}
	b := &mkr.Monitor{ID: "12345", Name: "foo", Type: "external", URL: "http://example.com", Service: "bar"}

	ret := diffMonitor(a, b)

	correct := strings.Join([]string{
		"  {",
		"   \"name\": \"foo\",",
		"   \"type\": \"external\",",
		"   \"url\": \"http://example.com\",",
		"   \"service\": \"bar\",",
		"-  \"responseTimeCritical\": 1000.000000,",
		"+  \"responseTimeCritical\": 0.000000,",
		"  },",
	}, "\n")

	if ret != correct {
		t.Errorf("should validate the rule: %s\nbut result: %s", correct, ret)
	}

}

func TestStringifyMonitor(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}

	if false {
		t.Errorf("somthing went wrong")
	}

	fmt.Println(stringifyMonitor(a, "+"))

}
<commit_msg>add test<commit_after>package main

import (
	"strings"
	"testing"

	mkr "github.com/mackerelio/mackerel-client-go"
)

func TestIsSameMonitor(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}
	b := &mkr.Monitor{Name: "foo", Type: "connectivity"}

	_, ret := isSameMonitor(a, b, true)
	if ret != true {
		t.Error("should recognize same monitors")
	}

	_, ret = isSameMonitor(a, b, false)
	if ret == true {
		t.Error("should not recognize same monitors")
	}
}

func TestValidateRoles(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}

	ret, err := validateRules([](*mkr.Monitor){a}, "test monitor")
	if ret != true {
		t.Errorf("should validate the rule: %s", err.Error())
	}

}

func TestDiffMonitors(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "external", URL: "http://example.com", Service: "bar", ResponseTimeCritical: 1000}
	b := &mkr.Monitor{ID: "12345", Name: "foo", Type: "external", URL: "http://example.com", Service: "bar"}

	ret := diffMonitor(a, b)

	correct := strings.Join([]string{
		"  {",
		"   \"name\": \"foo\",",
		"   \"type\": \"external\",",
		"   \"url\": \"http://example.com\",",
		"   \"service\": \"bar\",",
		"-  \"responseTimeCritical\": 1000.000000,",
		"+  \"responseTimeCritical\": 0.000000,",
		"  },",
	}, "\n")

	if ret != correct {
		t.Errorf("should validate the rule: %s\nbut result: %s", correct, ret)
	}

}

func TestStringifyMonitor(t *testing.T) {
	a := &mkr.Monitor{ID: "12345", Name: "foo", Type: "connectivity"}
	expected := `+ {
+   "id": "12345",
+   "name": "foo",
+   "type": "connectivity",
+ },`

	r := stringifyMonitor(a, "+")
	if r != expected {
		t.Errorf("stringifyMonitor should be:\n%s, but:\n%s", expected, r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

//+build ignore

package main

import (
	"bufio"
	"flag"
	"os"
	"strings"
	"text/template"
)

// Taken from
//  http://id3.org/id3v2.4.0-frames
//   id3v2.4.0-frames.txt,v 1.1 2003/07/27 18:28:34 id3 Exp
const spec = `
  4.19  AENC Audio encryption
  4.14  APIC Attached picture
  4.30  ASPI Audio seek point index

  4.10  COMM Comments
  4.24  COMR Commercial frame

  4.25  ENCR Encryption method registration
  4.12  EQU2 Equalisation (2)
  4.5   ETCO Event timing codes

  4.15  GEOB General encapsulated object
  4.26  GRID Group identification registration

  4.20  LINK Linked information

  4.4   MCDI Music CD identifier
  4.6   MLLT MPEG location lookup table

  4.23  OWNE Ownership frame

  4.27  PRIV Private frame
  4.16  PCNT Play counter
  4.17  POPM Popularimeter
  4.21  POSS Position synchronisation frame

  4.18  RBUF Recommended buffer size
  4.11  RVA2 Relative volume adjustment (2)
  4.13  RVRB Reverb

  4.29  SEEK Seek frame
  4.28  SIGN Signature frame
  4.9   SYLT Synchronised lyric/text
  4.7   SYTC Synchronised tempo codes

  4.2.1 TALB Album/Movie/Show title
  4.2.3 TBPM BPM (beats per minute)
  4.2.2 TCOM Composer
  4.2.3 TCON Content type
  4.2.4 TCOP Copyright message
  4.2.5 TDEN Encoding time
  4.2.5 TDLY Playlist delay
  4.2.5 TDOR Original release time
  4.2.5 TDRC Recording time
  4.2.5 TDRL Release time
  4.2.5 TDTG Tagging time
  4.2.2 TENC Encoded by
  4.2.2 TEXT Lyricist/Text writer
  4.2.3 TFLT File type
  4.2.2 TIPL Involved people list
  4.2.1 TIT1 Content group description
  4.2.1 TIT2 Title/songname/content description
  4.2.1 TIT3 Subtitle/Description refinement
  4.2.3 TKEY Initial key
  4.2.3 TLAN Language(s)
  4.2.3 TLEN Length
  4.2.2 TMCL Musician credits list
  4.2.3 TMED Media type
  4.2.3 TMOO Mood
  4.2.1 TOAL Original album/movie/show title
  4.2.5 TOFN Original filename
  4.2.2 TOLY Original lyricist(s)/text writer(s)
  4.2.2 TOPE Original artist(s)/performer(s)
  4.2.4 TOWN File owner/licensee
  4.2.2 TPE1 Lead performer(s)/Soloist(s)
  4.2.2 TPE2 Band/orchestra/accompaniment
  4.2.2 TPE3 Conductor/performer refinement
  4.2.2 TPE4 Interpreted, remixed, or otherwise modified by
  4.2.1 TPOS Part of a set
  4.2.4 TPRO Produced notice
  4.2.4 TPUB Publisher
  4.2.1 TRCK Track number/Position in set
  4.2.4 TRSN Internet radio station name
  4.2.4 TRSO Internet radio station owner
  4.2.5 TSOA Album sort order
  4.2.5 TSOP Performer sort order
  4.2.5 TSOT Title sort order
  4.2.1 TSRC ISRC (international standard recording code)
  4.2.5 TSSE Software/Hardware and settings used for encoding
  4.2.1 TSST Set subtitle
  4.2.2 TXXX User defined text information frame

  4.1   UFID Unique file identifier
  4.22  USER Terms of use
  4.8   USLT Unsynchronised lyric/text transcription

  4.3.1 WCOM Commercial information
  4.3.1 WCOP Copyright/Legal information
  4.3.1 WOAF Official audio file webpage
  4.3.1 WOAR Official artist/performer webpage
  4.3.1 WOAS Official audio source webpage
  4.3.1 WORS Official Internet radio station homepage
  4.3.1 WPAY Payment
  4.3.1 WPUB Publishers official webpage
  4.3.2 WXXX User defined URL link frame
`

var tmpl = template.Must(template.New("").Parse(
	"// Code generated by `go run generate_ids.go`. DO NOT EDIT." + `.

// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

package id3v2

const (
{{- range .}}
	Frame{{.ID}} FrameID = '{{index .ID 0 | printf "%c"}}'<<24 | '{{index .ID 1 | printf "%c"}}'<<16 | '{{index .ID 2 | printf "%c"}}'<<8 | '{{index .ID 3 | printf "%c"}}' // {{.Description}}
{{- end}}
)

func (id FrameID) String() string {
	switch id {
{{- range .}}
	case '{{index .ID 0 | printf "%c"}}'<<24 | '{{index .ID 1 | printf "%c"}}'<<16 | '{{index .ID 2 | printf "%c"}}'<<8 | '{{index .ID 3 | printf "%c"}}':
		return "{{.ID}}: {{.Description}}"
{{- end}}
	default:
		buf := [4]byte{
			byte(id >> 24),
			byte(id >> 16),
			byte(id >> 8),
			byte(id),
		}
		return "FrameID(\"" + string(buf[:]) + "\")"
	}
}
`))

type frameID struct {
	ID, Description string
}

func main() {
	out := flag.String("out", "frame_ids.go", "the file to write the ids to")

	s := bufio.NewScanner(strings.NewReader(spec))
	var ids []frameID

	for s.Scan() {
		parts := strings.Fields(s.Text())
		if len(parts) < 2 || parts[0][:2] != "4." {
			continue
		}

		ids = append(ids, frameID{parts[1], strings.Join(parts[2:], " ")})
	}

	if s.Err() != nil {
		panic(s.Err())
	}

	w, err := os.Create(*out)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	if err := tmpl.Execute(w, ids); err != nil {
		panic(err)
	}
}
<commit_msg>Add missing flag.Parse in generate_ids.go<commit_after>// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

//+build ignore

package main

import (
	"bufio"
	"flag"
	"os"
	"strings"
	"text/template"
)

// Taken from
//  http://id3.org/id3v2.4.0-frames
//   id3v2.4.0-frames.txt,v 1.1 2003/07/27 18:28:34 id3 Exp
const spec = `
  4.19  AENC Audio encryption
  4.14  APIC Attached picture
  4.30  ASPI Audio seek point index

  4.10  COMM Comments
  4.24  COMR Commercial frame

  4.25  ENCR Encryption method registration
  4.12  EQU2 Equalisation (2)
  4.5   ETCO Event timing codes

  4.15  GEOB General encapsulated object
  4.26  GRID Group identification registration

  4.20  LINK Linked information

  4.4   MCDI Music CD identifier
  4.6   MLLT MPEG location lookup table

  4.23  OWNE Ownership frame

  4.27  PRIV Private frame
  4.16  PCNT Play counter
  4.17  POPM Popularimeter
  4.21  POSS Position synchronisation frame

  4.18  RBUF Recommended buffer size
  4.11  RVA2 Relative volume adjustment (2)
  4.13  RVRB Reverb

  4.29  SEEK Seek frame
  4.28  SIGN Signature frame
  4.9   SYLT Synchronised lyric/text
  4.7   SYTC Synchronised tempo codes

  4.2.1 TALB Album/Movie/Show title
  4.2.3 TBPM BPM (beats per minute)
  4.2.2 TCOM Composer
  4.2.3 TCON Content type
  4.2.4 TCOP Copyright message
  4.2.5 TDEN Encoding time
  4.2.5 TDLY Playlist delay
  4.2.5 TDOR Original release time
  4.2.5 TDRC Recording time
  4.2.5 TDRL Release time
  4.2.5 TDTG Tagging time
  4.2.2 TENC Encoded by
  4.2.2 TEXT Lyricist/Text writer
  4.2.3 TFLT File type
  4.2.2 TIPL Involved people list
  4.2.1 TIT1 Content group description
  4.2.1 TIT2 Title/songname/content description
  4.2.1 TIT3 Subtitle/Description refinement
  4.2.3 TKEY Initial key
  4.2.3 TLAN Language(s)
  4.2.3 TLEN Length
  4.2.2 TMCL Musician credits list
  4.2.3 TMED Media type
  4.2.3 TMOO Mood
  4.2.1 TOAL Original album/movie/show title
  4.2.5 TOFN Original filename
  4.2.2 TOLY Original lyricist(s)/text writer(s)
  4.2.2 TOPE Original artist(s)/performer(s)
  4.2.4 TOWN File owner/licensee
  4.2.2 TPE1 Lead performer(s)/Soloist(s)
  4.2.2 TPE2 Band/orchestra/accompaniment
  4.2.2 TPE3 Conductor/performer refinement
  4.2.2 TPE4 Interpreted, remixed, or otherwise modified by
  4.2.1 TPOS Part of a set
  4.2.4 TPRO Produced notice
  4.2.4 TPUB Publisher
  4.2.1 TRCK Track number/Position in set
  4.2.4 TRSN Internet radio station name
  4.2.4 TRSO Internet radio station owner
  4.2.5 TSOA Album sort order
  4.2.5 TSOP Performer sort order
  4.2.5 TSOT Title sort order
  4.2.1 TSRC ISRC (international standard recording code)
  4.2.5 TSSE Software/Hardware and settings used for encoding
  4.2.1 TSST Set subtitle
  4.2.2 TXXX User defined text information frame

  4.1   UFID Unique file identifier
  4.22  USER Terms of use
  4.8   USLT Unsynchronised lyric/text transcription

  4.3.1 WCOM Commercial information
  4.3.1 WCOP Copyright/Legal information
  4.3.1 WOAF Official audio file webpage
  4.3.1 WOAR Official artist/performer webpage
  4.3.1 WOAS Official audio source webpage
  4.3.1 WORS Official Internet radio station homepage
  4.3.1 WPAY Payment
  4.3.1 WPUB Publishers official webpage
  4.3.2 WXXX User defined URL link frame
`

var tmpl = template.Must(template.New("").Parse(
	"// Code generated by `go run generate_ids.go`. DO NOT EDIT." + `.

// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

package id3v2

const (
{{- range .}}
	Frame{{.ID}} FrameID = '{{index .ID 0 | printf "%c"}}'<<24 | '{{index .ID 1 | printf "%c"}}'<<16 | '{{index .ID 2 | printf "%c"}}'<<8 | '{{index .ID 3 | printf "%c"}}' // {{.Description}}
{{- end}}
)

func (id FrameID) String() string {
	switch id {
{{- range .}}
	case '{{index .ID 0 | printf "%c"}}'<<24 | '{{index .ID 1 | printf "%c"}}'<<16 | '{{index .ID 2 | printf "%c"}}'<<8 | '{{index .ID 3 | printf "%c"}}':
		return "{{.ID}}: {{.Description}}"
{{- end}}
	default:
		buf := [4]byte{
			byte(id >> 24),
			byte(id >> 16),
			byte(id >> 8),
			byte(id),
		}
		return "FrameID(\"" + string(buf[:]) + "\")"
	}
}
`))

type frameID struct {
	ID, Description string
}

func main() {
	out := flag.String("out", "frame_ids.go", "the file to write the ids to")

	flag.Parse()

	s := bufio.NewScanner(strings.NewReader(spec))
	var ids []frameID

	for s.Scan() {
		parts := strings.Fields(s.Text())
		if len(parts) < 2 || parts[0][:2] != "4." {
			continue
		}

		ids = append(ids, frameID{parts[1], strings.Join(parts[2:], " ")})
	}

	if s.Err() != nil {
		panic(s.Err())
	}

	w, err := os.Create(*out)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	if err := tmpl.Execute(w, ids); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>デフォルト値を調整<commit_after><|endoftext|>


<commit_before>package response

import (
	"code.google.com/p/go-uuid/uuid"
	"net/http"
)

type Responder struct {
	apiVersion string
	error      *ErrorResponse
	data       *DataResponse
}

func New(apiVersion string) *Responder {
	return &Responder{
		apiVersion: apiVersion,
		error: &ErrorResponse{
			Error: Error{
				Errors: make([]ErrorItem, 0, 1),
			},
		},
		data: &DataResponse{},
	}
}

func (r *Responder) Success(data interface{}) Response {
	r.data = &DataResponse{
		ApiVersion: r.apiVersion,
		Id:         uuid.NewRandom(),
		Data:       data,
	}
	return *r.data
}

func (r *Responder) AddError(e ErrorItem) {
	r.error.Error.Errors = append(r.error.Error.Errors, e)
}

func (r *Responder) Error(code int, message string) Response {
	r.error = &ErrorResponse{
		ApiVersion: r.apiVersion,
		Id:         uuid.NewRandom(),
		Error: Error{
			Code:    code,
			Message: message,
		},
	}
	return r.error
}

func (r *Responder) Write(w http.ResponseWriter, s Response) error {
	m, err := s.Marshal()
	if err != nil {
		return err
	}
	w.Header().Set("Content-Type", "application/json")
	switch s := s.(type) {
	case *ErrorResponse:
		if s.Error.Code == 0 {
			s.Error.Code = http.StatusInternalServerError
		}
		code := s.Error.Code
		w.WriteHeader(code)
	}
	w.Write(m)
	return nil
}
<commit_msg>response: let write handle marshalling error<commit_after>package response

import (
	"code.google.com/p/go-uuid/uuid"
	"net/http"
)

type Responder struct {
	apiVersion string
	errorResponse      *ErrorResponse
	dataResponse       *DataResponse
}

func New(apiVersion string) *Responder {
	return &Responder{
		apiVersion: apiVersion,
        errorResponse: &ErrorResponse{
			Error: Error{
				Errors: make([]ErrorItem, 0, 1),
			},
		},
        dataResponse: &DataResponse{},
	}
}

func (r *Responder) Success(data interface{}) Response {
	r.dataResponse = &DataResponse{
		ApiVersion: r.apiVersion,
		Id:         uuid.NewRandom(),
		Data:       data,
	}
	return *r.dataResponse
}

func (r *Responder) AddError(e ErrorItem) {
	r.errorResponse.Error.Errors = append(r.errorResponse.Error.Errors, e)
}

func (r *Responder) Error(code int, message string) Response {
	r.errorResponse = &ErrorResponse{
		ApiVersion: r.apiVersion,
		Id:         uuid.NewRandom(),
		Error: Error{
			Code:    code,
			Message: message,
		},
	}
	return r.errorResponse
}

func (r *Responder) Write(w http.ResponseWriter, s Response) error {
	m, err := s.Marshal()
    if err != nil {
        responseError := r.Error(http.StatusInternalServerError, err.Error())
        return r.Write(w, responseError)
    }
	w.Header().Set("Content-Type", "application/json")
	switch s := s.(type) {
	case *ErrorResponse:
		if s.Error.Code == 0 {
			s.Error.Code = http.StatusInternalServerError
		}
		code := s.Error.Code
		w.WriteHeader(code)
	}
	w.Write(m)
	return nil
}
<|endoftext|>


<commit_before>package app

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/braintree/manners"
	"github.com/gorilla/mux"
)

type ErrorResponse struct {
	Id      string `json:"id"`
	Message string `json:"message"`
}

type orderBody struct {
	Pizza int `json:"id"`
}

func home(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func pizzasList(w http.ResponseWriter, r *http.Request) {
	list, _ := PizzaList()

	if err := json.NewEncoder(w).Encode(list); err != nil {
		panic(err)
	}
}

func findOrder(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, _ := strconv.Atoi(vars["id"])

	order, err := FindOrder(id)
	if err != nil {
		w.WriteHeader(404)
		response := &ErrorResponse{Id: "not_found", Message: "Order not found"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	if err := json.NewEncoder(w).Encode(order); err != nil {
		panic(err)
	}
}

func createOrder(w http.ResponseWriter, r *http.Request) {
	var orderContent orderBody
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}
	if err := json.Unmarshal(body, &orderContent); err != nil {
		w.WriteHeader(422)
		response := &ErrorResponse{Id: "invalid", Message: "Invalid data"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	pizza, err := FindPizza(orderContent.Pizza)
	if err != nil {
		w.WriteHeader(422)
		response := &ErrorResponse{Id: "not_found", Message: "Pizza not found"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	order := CreateOrder(pizza)
	w.WriteHeader(http.StatusCreated)

	if err := json.NewEncoder(w).Encode(order); err != nil {
		panic(err)
	}
}

type upgradeBody struct {
	Name string `json:"name"`
	Ip   string `json:"ip"`
	Key  string `json:"key"`
}

func upgrade(w http.ResponseWriter, r *http.Request) {
	var upgradeContent upgradeBody
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}
	if err := json.Unmarshal(body, &upgradeContent); err != nil {
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}
		response := &ErrorResponse{Id: "invalid", Message: "Invalid data"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	if upgradeContent.Key != os.Getenv("UPGRADE_KEY") {
		response := &ErrorResponse{Id: "unauthorized", Message: "Not authorized"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	var constraint *Constraint
	if upgradeContent.Ip == "all" {
		constraint = globalConstraint
	} else {
		constraint, _ = findConstraint(upgradeContent.Ip, false)
	}
	constraint.Constraint = upgradeContent.Name

	request_id := requestId(r)
	log.Printf("count#upgraded name=%s ip=%s request_id=", upgradeContent.Name, upgradeContent.Ip, request_id)
}

func app() http.Handler {
	r := mux.NewRouter()
	r.HandleFunc("/", home).Methods("GET")
	r.HandleFunc("/pizzas", pizzasList).Methods("GET")
	r.HandleFunc("/orders/{id}", findOrder).Methods("GET")

	r.HandleFunc("/orders", createOrder).Methods("POST")
	r.HandleFunc("/upgrade", upgrade).Methods("POST")

	return applyConstraints(logRequest(r.ServeHTTP))
}

func StartServer(port string, shutdown <-chan struct{}) {
	log.Printf("http.start.port=%s\n", port)
	http.Handle("/", app())
	go listenForShutdown(shutdown)

	if err := manners.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("server.server error=%v", err)
	}
}

func listenForShutdown(shutdown <-chan struct{}) {
	log.Println("http.graceful.await")
	<-shutdown
	log.Println("http.graceful.shutdown")
	manners.Close()
}
<commit_msg>better logging when rejecting an order creation<commit_after>package app

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/braintree/manners"
	"github.com/gorilla/mux"
)

type ErrorResponse struct {
	Id      string `json:"id"`
	Message string `json:"message"`
}

type orderBody struct {
	Pizza int `json:"id"`
}

func home(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func pizzasList(w http.ResponseWriter, r *http.Request) {
	list, _ := PizzaList()

	if err := json.NewEncoder(w).Encode(list); err != nil {
		panic(err)
	}
}

func findOrder(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, _ := strconv.Atoi(vars["id"])

	order, err := FindOrder(id)
	if err != nil {
		w.WriteHeader(404)
		response := &ErrorResponse{Id: "not_found", Message: "Order not found"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	if err := json.NewEncoder(w).Encode(order); err != nil {
		panic(err)
	}
}

func createOrder(w http.ResponseWriter, r *http.Request) {
	var orderContent orderBody
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}
	if err := json.Unmarshal(body, &orderContent); err != nil {
		w.WriteHeader(422)
		response := &ErrorResponse{Id: "invalid", Message: fmt.Sprintf("Invalid data. Expected '{\"id\":1}'. Got %s", body)}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	pizza, err := FindPizza(orderContent.Pizza)
	if err != nil {
		w.WriteHeader(422)
		response := &ErrorResponse{Id: "not_found", Message: "Pizza not found"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	order := CreateOrder(pizza)
	w.WriteHeader(http.StatusCreated)

	if err := json.NewEncoder(w).Encode(order); err != nil {
		panic(err)
	}
}

type upgradeBody struct {
	Name string `json:"name"`
	Ip   string `json:"ip"`
	Key  string `json:"key"`
}

func upgrade(w http.ResponseWriter, r *http.Request) {
	var upgradeContent upgradeBody
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}
	if err := json.Unmarshal(body, &upgradeContent); err != nil {
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}
		response := &ErrorResponse{Id: "invalid", Message: "Invalid data"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	if upgradeContent.Key != os.Getenv("UPGRADE_KEY") {
		response := &ErrorResponse{Id: "unauthorized", Message: "Not authorized"}
		if err := json.NewEncoder(w).Encode(response); err != nil {
			panic(err)
		}
		return
	}

	var constraint *Constraint
	if upgradeContent.Ip == "all" {
		constraint = globalConstraint
	} else {
		constraint, _ = findConstraint(upgradeContent.Ip, false)
	}
	constraint.Constraint = upgradeContent.Name

	request_id := requestId(r)
	log.Printf("count#upgraded name=%s ip=%s request_id=", upgradeContent.Name, upgradeContent.Ip, request_id)
}

func app() http.Handler {
	r := mux.NewRouter()
	r.HandleFunc("/", home).Methods("GET")
	r.HandleFunc("/pizzas", pizzasList).Methods("GET")
	r.HandleFunc("/orders/{id}", findOrder).Methods("GET")

	r.HandleFunc("/orders", createOrder).Methods("POST")
	r.HandleFunc("/upgrade", upgrade).Methods("POST")

	return applyConstraints(logRequest(r.ServeHTTP))
}

func StartServer(port string, shutdown <-chan struct{}) {
	log.Printf("http.start.port=%s\n", port)
	http.Handle("/", app())
	go listenForShutdown(shutdown)

	if err := manners.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("server.server error=%v", err)
	}
}

func listenForShutdown(shutdown <-chan struct{}) {
	log.Println("http.graceful.await")
	<-shutdown
	log.Println("http.graceful.shutdown")
	manners.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "fmt"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
// "flag"
  "sort"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

func main(){
  if( len(os.Args) < 3 ){
    fmt.Println( "myarbitratord usage: myarbitratord <seed_host> <seed_port> [true] (enable debug logging)" )
    os.Exit(1);
  }

  seed_host := os.Args[1]
  seed_port := os.Args[2]
  if( len(os.Args) > 3 ){
    debug_mode := os.Args[3]

    if( debug_mode == "true" ){
      instances.Debug = true
      debug = true
    }
  }

  // Currently these credentials should work on any participating MySQL instance
  mysql_user := "root"
  mysql_pass := "!root19M"

  // check out flag.String( "foo" ) for option handling ...

  fmt.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  fmt.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err := seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      for _, member := range last_view {
        member.Connect()
        if( err == nil && member.Member_state == "ONLINE" ){
          seed_node = &member
          fmt.Println( "Updated seed node! New seed node is: ", seed_node.Mysql_host, ":", seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      log.Fatal( err )
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      log.Fatal( err )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          fmt.Println( "Shutting down non-healthy node: ", member.Mysql_host, ":", member.Mysql_port )
          err = member.Shutdown()

          if( err != nil ){
            fmt.Println( "Could not shutdown instance: ", member.Mysql_host, ":", member.Mysql_port )
          }
        }
      }
    } else {
      // handling network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one we'll then need to contact 1 node in the primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"' and finally we'll need
      // to try and connect to the nodes on the losing side of the partition and attempt to shutdown mysqld

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false
      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If noone in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members; the participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      // ToDo: should we consider GTID_EXECUTED sets when choosing a partition???
      if( primary_partition == false ){
        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        if( debug ){
          fmt.Printf( "Member view sorted by number of online nodes: %v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()
        force_member_string := ""
        for i, member := range *members {
          if( member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        fmt.Println( "Forcing group membership! using: '", force_member_string, "'" )
        err = seed_node.ForceMembers( force_member_string )
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<commit_msg>Improved handling of complete loss of quorum<commit_after>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "fmt"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
// "flag"
  "sort"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

func main(){
  if( len(os.Args) < 3 ){
    fmt.Println( "myarbitratord usage: myarbitratord <seed_host> <seed_port> [true] (enable debug logging)" )
    os.Exit(1);
  }

  seed_host := os.Args[1]
  seed_port := os.Args[2]
  if( len(os.Args) > 3 ){
    debug_mode := os.Args[3]

    if( debug_mode == "true" ){
      instances.Debug = true
      debug = true
    }
  }

  // Currently these credentials should work on any participating MySQL instance
  mysql_user := "root"
  mysql_pass := "!root19M"

  // check out flag.String( "foo" ) for option handling ...

  fmt.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  fmt.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err := seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      for _, member := range last_view {
        member.Connect()
        if( err == nil && member.Member_state == "ONLINE" ){
          seed_node = &member
          fmt.Println( "Updated seed node! New seed node is: ", seed_node.Mysql_host, ":", seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      log.Fatal( err )
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      log.Fatal( err )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          fmt.Println( "Shutting down non-healthy node: ", member.Mysql_host, ":", member.Mysql_port )
          err = member.Shutdown()

          if( err != nil ){
            fmt.Println( "Could not shutdown instance: ", member.Mysql_host, ":", member.Mysql_port )
          }
        }
      }
    } else {
      // handling network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one we'll then need to contact 1 node in the primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"' and finally we'll need
      // to try and connect to the nodes on the losing side of the partition and attempt to shutdown mysqld

      fmt.Println( "Network partition detected! Attempting to handle... " )

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false
      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If noone in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members; the participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      // ToDo: should we consider GTID_EXECUTED sets when choosing a partition???
      if( primary_partition == false ){
        fmt.Println( "No primary partition found! Attempting to choose and force a new one ... " )

        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        if( debug ){
          fmt.Printf( "Member view sorted by number of online nodes: %v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()
        force_member_string := ""
        for i, member := range *members {
          if( member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        fmt.Println( "Forcing group membership to form new primary partition! Using: '", force_member_string, "'" )
        err = seed_node.ForceMembers( force_member_string )
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<|endoftext|>


<commit_before>package main

import (
        "fmt"
//        "io"
	"path"
        "io/ioutil"
//        "strings"
        "strconv"
        "encoding/csv"
        "os"
)
func checkErr(e error){
	if (e != nil) {
		fmt.Println("POOP!")
		os.Exit(1)
	}
}

// Removes non-directory elements of any []os.FileInfo (a helper function, for convenience)
// Makes use of idiomatic Golang return: out_ls is declared and returned implicitly
func onlyDirectories(potential_files []os.FileInfo) (out_ls []os.FileInfo) {
	for _, fd := range potential_files {
		if fd.IsDir() {
			out_ls = append(out_ls, fd)
		}
	}
	return
}

// Extend "records" matrix to have rows until time "desired_time"
// Return: Extended version of record
func extendRecordsToTime(records [][]string, desired_time int, recordCols int) ([][]string)  {
	lenr := len(records)
	for j := lenr; j < desired_time + 1; j++ {
		records = append(records, make([]string, recordCols))
		records[j][0] = strconv.Itoa(j)
		for k := 1; k < recordCols; k++ {
			records[j][k] = ""
		}
	}
	return records
}

// Enters all report subdirectories, from benchmark to fengine to trial;
// composes individual CSVs (only two columns) into larger CSVs
func composeAllNamed(desired_report_fname string) {
	master_path := "./reports"
	bmarks, err := ioutil.ReadDir(master_path)
	checkErr(err)
	for _, bmark := range bmarks {
		// all_fe_file, _ := os.Create(path.Join(master_path, bmark.Name(), desired_report_fname))
		// defer all_fe_file.Close()
		// all_fe_writer := csv.NewWriter(all_fe_file)
		potential_fengines, err := ioutil.ReadDir(path.Join(master_path, bmark.Name()))
		checkErr(err)
		// narrow potential_fengines to fengines so the indices of `range fengines` are useful
		fengines := onlyDirectories(potential_fengines)

		for _, fengine := range fengines {
			// Create fds
			this_fe_file, err := os.Create(path.Join(master_path, bmark.Name(), fengine.Name(), desired_report_fname))
			checkErr(err)
			defer this_fe_file.Close()
			this_fe_writer := csv.NewWriter(this_fe_file)

			// Create matrix, to eventually become a CSV
			records := [][]string{{"time"}}

			// Enter sub-directories
			potential_trials, _ := ioutil.ReadDir(path.Join(master_path, bmark.Name(), fengine.Name()))
			trials := onlyDirectories(potential_trials)

			num_record_columns := len(trials) + 1
			for j, trial := range trials {
				// Create fds
				this_file, err := os.Open(path.Join(master_path, bmark.Name(), fengine.Name(), trial.Name(), desired_report_fname))
				checkErr(err)
				defer this_file.Close()
				this_reader := csv.NewReader(this_file)

				experiment_records, err := this_reader.ReadAll()
				checkErr(err)
				records[0] = append(records[0], fengine.Name() + trial.Name())

				for _, row := range experiment_records {
					time_now, err := strconv.Atoi(row[0])
					checkErr(err)
					//If this test went longer than all of the others, so far
					if(len(records) < time_now + 1) {
						records = extendRecordsToTime(records, time_now, num_record_columns)
					}
					records[time_now][j+1] = row[1]
				}
				this_fe_writer.WriteAll(records)
			}
			// TODO: create comparison between fengines, having already composed trials
			// Do this by identifying the max (or potentially median) performing trial
			// For each fengine, and putting them all into a CSV which can be graphed
		}
	}
}

func main(){
	composeAllNamed("coverage-graph.csv")
	composeAllNamed("corpus-size-graph.csv")
	composeAllNamed("corpus-elems-graph.csv")
	// createIFramesFor("setOfFrames.html")
	//
}

<commit_msg>Document report-gen<commit_after>package main

import (
        "fmt"
//        "io"
	"path"
        "io/ioutil"
//        "strings"
        "strconv"
        "encoding/csv"
        "os"
)
func checkErr(e error){
	if (e != nil) {
		fmt.Println("POOP!")
		os.Exit(1)
	}
}

// Removes non-directory elements of any []os.FileInfo (a helper function, for convenience)
// Makes use of idiomatic Golang return: out_ls is declared and returned implicitly
func onlyDirectories(potential_files []os.FileInfo) (out_ls []os.FileInfo) {
	for _, fd := range potential_files {
		if fd.IsDir() {
			out_ls = append(out_ls, fd)
		}
	}
	return
}

// Extend "records" matrix to have rows until time "desired_time"
// Return: Extended version of record
func extendRecordsToTime(records [][]string, desired_time int, recordCols int) ([][]string)  {
	lenr := len(records)
	// records[1] stores cycle [1], as records[0] is column names
	for j := lenr; j < desired_time + 1; j++ {
		records = append(records, make([]string, recordCols))
		records[j][0] = strconv.Itoa(j)
		for k := 1; k < recordCols; k++ {
			records[j][k] = ""
		}
	}
	return records
}

// Enters all report subdirectories, from benchmark to fengine to trial;
// composes individual CSVs (only two columns) into larger CSVs
func composeAllNamed(desired_report_fname string) {
	master_path := "./reports"
	bmarks, err := ioutil.ReadDir(master_path)
	checkErr(err)
	for _, bmark := range bmarks {
		// all_fe_file, _ := os.Create(path.Join(master_path, bmark.Name(), desired_report_fname))
		// defer all_fe_file.Close()
		// all_fe_writer := csv.NewWriter(all_fe_file)
		potential_fengines, err := ioutil.ReadDir(path.Join(master_path, bmark.Name()))
		checkErr(err)
		// narrow potential_fengines to fengines so the indices of `range fengines` are useful
		fengines := onlyDirectories(potential_fengines)

		for _, fengine := range fengines {
			// Create fds
			this_fe_file, err := os.Create(path.Join(master_path, bmark.Name(), fengine.Name(), desired_report_fname))
			checkErr(err)
			defer this_fe_file.Close()
			this_fe_writer := csv.NewWriter(this_fe_file)

			// Create matrix, to eventually become a CSV
			records := [][]string{{"time"}}

			// Enter sub-directories
			potential_trials, _ := ioutil.ReadDir(path.Join(master_path, bmark.Name(), fengine.Name()))
			trials := onlyDirectories(potential_trials)

			num_record_columns := len(trials) + 1
			for j, trial := range trials {
				// Create fds
				this_file, err := os.Open(path.Join(master_path, bmark.Name(), fengine.Name(), trial.Name(), desired_report_fname))
				checkErr(err)
				defer this_file.Close()
				this_reader := csv.NewReader(this_file)

				// Read whole CSV to an array
				experiment_records, err := this_reader.ReadAll()
				checkErr(err)
				// Add the name of this new column to records[0]
				records[0] = append(records[0], fengine.Name() + trial.Name())

				for _, row := range experiment_records {
					// row[0] is time, on the x-axis; row[1] is value, on the y-axis
					time_now, err := strconv.Atoi(row[0])
					checkErr(err)
					//If this test went longer than all of the others, so far
					if(len(records) < time_now + 1) {
						records = extendRecordsToTime(records, time_now, num_record_columns)
					}
					records[time_now][j+1] = row[1]
				}
			}
			this_fe_writer.WriteAll(records)
			// Potentially put this fengine into a broader comparison CSV
		}
		// TODO: create comparison between fengines, having already composed trials
		// Do this by identifying the max (or potentially median) performing trial
		// For each fengine, and putting them all into a CSV which can be graphed
	}
}

func main(){
	composeAllNamed("coverage-graph.csv")
	composeAllNamed("corpus-size-graph.csv")
	composeAllNamed("corpus-elems-graph.csv")
	// createIFramesFor("setOfFrames.html")
	//
}

<|endoftext|>


<commit_before>package security

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
	"google.golang.org/grpc/security/advancedtls"
	"io/ioutil"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

const credRefreshingInterval = time.Duration(5) * time.Hour

type Authenticator struct {
	AllowedWildcardDomain string
	AllowedCommonNames    map[string]bool
}

func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption, grpc.ServerOption) {
	if config == nil {
		return nil, nil
	}

	serverOptions := pemfile.Options{
		CertFile:        config.GetString(component + ".cert"),
		KeyFile:         config.GetString(component + ".key"),
		RefreshDuration: credRefreshingInterval,
	}

	serverIdentityProvider, err := pemfile.NewProvider(serverOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverOptions, err)
		return nil, nil
	}
	//defer serverIdentityProvider.Close()

	serverRootOptions := pemfile.Options{
		RootFile:        config.GetString("grpc.ca"),
		RefreshDuration: credRefreshingInterval,
	}
	serverRootProvider, err := pemfile.NewProvider(serverRootOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverRootOptions, err)
		return nil, nil
	}
	// defer serverIdentityProvider.Close()
	// Start a server and create a client using advancedtls API with Provider.
	options := &advancedtls.ServerOptions{
		IdentityOptions: advancedtls.IdentityCertificateOptions{
			IdentityProvider: serverIdentityProvider,
		},
		RootOptions: advancedtls.RootCertificateOptions{
			RootProvider: serverRootProvider,
		},
		RequireClientCert: true,
		VType:             advancedtls.CertVerification,
	}
	allowedCommonNames := config.GetString(component + ".allowed_commonNames")
	allowedWildcardDomain := config.GetString("grpc.allowed_wildcard_domain")
	if allowedCommonNames != "" || allowedWildcardDomain != "" {
		allowedCommonNamesMap := make(map[string]bool)
		for _, s := range strings.Split(allowedCommonNames, ",") {
			allowedCommonNamesMap[s] = true
		}
		auther := Authenticator{
			AllowedCommonNames:    allowedCommonNamesMap,
			AllowedWildcardDomain: allowedWildcardDomain,
		}
		options.VerifyPeer = auther.Authenticate
	} else {
		options.VerifyPeer = func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
			return &advancedtls.VerificationResults{}, nil
		}
	}
	ta, err := advancedtls.NewServerCreds(options)
	if err != nil {
		glog.Warningf("advancedtls.NewServerCreds(%v) failed: %v", options, err)
		return nil, nil
	}
	return grpc.Creds(ta), nil
}

func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
	if config == nil {
		return grpc.WithInsecure()
	}

	certFileName, keyFileName, caFileName := config.GetString(component+".cert"), config.GetString(component+".key"), config.GetString("grpc.ca")
	if certFileName == "" || keyFileName == "" || caFileName == "" {
		return grpc.WithInsecure()
	}

	clientOptions := pemfile.Options{
		CertFile:        certFileName,
		KeyFile:         keyFileName,
		RefreshDuration: credRefreshingInterval,
	}
	clientProvider, err := pemfile.NewProvider(clientOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed %v", clientOptions, err)
		return grpc.WithInsecure()
	}
	clientRootOptions := pemfile.Options{
		RootFile:        config.GetString("grpc.ca"),
		RefreshDuration: credRefreshingInterval,
	}
	clientRootProvider, err := pemfile.NewProvider(clientRootOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", clientRootOptions, err)
		return grpc.WithInsecure()
	}
	options := &advancedtls.ClientOptions{
		IdentityOptions: advancedtls.IdentityCertificateOptions{
			IdentityProvider: clientProvider,
		},
		VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
			return &advancedtls.VerificationResults{}, nil
		},
		RootOptions: advancedtls.RootCertificateOptions{
			RootProvider: clientRootProvider,
		},
		VType: advancedtls.CertVerification,
	}
	ta, err := advancedtls.NewClientCreds(options)
	if err != nil {
		glog.Warningf("advancedtls.NewClientCreds(%v) failed: %v", options, err)
		return grpc.WithInsecure()
	}
	return grpc.WithTransportCredentials(ta)
}

func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
	clientCerts, err := ioutil.ReadFile(clientCertFile)
	if err != nil {
		glog.Fatal(err)
	}
	certPool := x509.NewCertPool()
	ok := certPool.AppendCertsFromPEM(clientCerts)
	if !ok {
		glog.Fatalf("Error processing client certificate in %s\n", clientCertFile)
	}

	return &tls.Config{
		ClientCAs:  certPool,
		ClientAuth: tls.RequireAndVerifyClientCert,
	}
}

func (a Authenticator) Authenticate(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
	if a.AllowedWildcardDomain != "" && strings.HasSuffix(params.Leaf.Subject.CommonName, a.AllowedWildcardDomain) {
		return &advancedtls.VerificationResults{}, nil
	}
	if _, ok := a.AllowedCommonNames[params.Leaf.Subject.CommonName]; ok {
		return &advancedtls.VerificationResults{}, nil
	}
	err := fmt.Errorf("Authenticate: invalid subject client common name: %s", params.Leaf.Subject.CommonName)
	glog.Error(err)
	return nil, err
}
<commit_msg>rm defer comments<commit_after>package security

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
	"google.golang.org/grpc/security/advancedtls"
	"io/ioutil"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

const credRefreshingInterval = time.Duration(5) * time.Hour

type Authenticator struct {
	AllowedWildcardDomain string
	AllowedCommonNames    map[string]bool
}

func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption, grpc.ServerOption) {
	if config == nil {
		return nil, nil
	}

	serverOptions := pemfile.Options{
		CertFile:        config.GetString(component + ".cert"),
		KeyFile:         config.GetString(component + ".key"),
		RefreshDuration: credRefreshingInterval,
	}

	serverIdentityProvider, err := pemfile.NewProvider(serverOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverOptions, err)
		return nil, nil
	}

	serverRootOptions := pemfile.Options{
		RootFile:        config.GetString("grpc.ca"),
		RefreshDuration: credRefreshingInterval,
	}
	serverRootProvider, err := pemfile.NewProvider(serverRootOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverRootOptions, err)
		return nil, nil
	}

	// Start a server and create a client using advancedtls API with Provider.
	options := &advancedtls.ServerOptions{
		IdentityOptions: advancedtls.IdentityCertificateOptions{
			IdentityProvider: serverIdentityProvider,
		},
		RootOptions: advancedtls.RootCertificateOptions{
			RootProvider: serverRootProvider,
		},
		RequireClientCert: true,
		VType:             advancedtls.CertVerification,
	}
	allowedCommonNames := config.GetString(component + ".allowed_commonNames")
	allowedWildcardDomain := config.GetString("grpc.allowed_wildcard_domain")
	if allowedCommonNames != "" || allowedWildcardDomain != "" {
		allowedCommonNamesMap := make(map[string]bool)
		for _, s := range strings.Split(allowedCommonNames, ",") {
			allowedCommonNamesMap[s] = true
		}
		auther := Authenticator{
			AllowedCommonNames:    allowedCommonNamesMap,
			AllowedWildcardDomain: allowedWildcardDomain,
		}
		options.VerifyPeer = auther.Authenticate
	} else {
		options.VerifyPeer = func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
			return &advancedtls.VerificationResults{}, nil
		}
	}
	ta, err := advancedtls.NewServerCreds(options)
	if err != nil {
		glog.Warningf("advancedtls.NewServerCreds(%v) failed: %v", options, err)
		return nil, nil
	}
	return grpc.Creds(ta), nil
}

func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
	if config == nil {
		return grpc.WithInsecure()
	}

	certFileName, keyFileName, caFileName := config.GetString(component+".cert"), config.GetString(component+".key"), config.GetString("grpc.ca")
	if certFileName == "" || keyFileName == "" || caFileName == "" {
		return grpc.WithInsecure()
	}

	clientOptions := pemfile.Options{
		CertFile:        certFileName,
		KeyFile:         keyFileName,
		RefreshDuration: credRefreshingInterval,
	}
	clientProvider, err := pemfile.NewProvider(clientOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed %v", clientOptions, err)
		return grpc.WithInsecure()
	}
	clientRootOptions := pemfile.Options{
		RootFile:        config.GetString("grpc.ca"),
		RefreshDuration: credRefreshingInterval,
	}
	clientRootProvider, err := pemfile.NewProvider(clientRootOptions)
	if err != nil {
		glog.Warningf("pemfile.NewProvider(%v) failed: %v", clientRootOptions, err)
		return grpc.WithInsecure()
	}
	options := &advancedtls.ClientOptions{
		IdentityOptions: advancedtls.IdentityCertificateOptions{
			IdentityProvider: clientProvider,
		},
		VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
			return &advancedtls.VerificationResults{}, nil
		},
		RootOptions: advancedtls.RootCertificateOptions{
			RootProvider: clientRootProvider,
		},
		VType: advancedtls.CertVerification,
	}
	ta, err := advancedtls.NewClientCreds(options)
	if err != nil {
		glog.Warningf("advancedtls.NewClientCreds(%v) failed: %v", options, err)
		return grpc.WithInsecure()
	}
	return grpc.WithTransportCredentials(ta)
}

func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
	clientCerts, err := ioutil.ReadFile(clientCertFile)
	if err != nil {
		glog.Fatal(err)
	}
	certPool := x509.NewCertPool()
	ok := certPool.AppendCertsFromPEM(clientCerts)
	if !ok {
		glog.Fatalf("Error processing client certificate in %s\n", clientCertFile)
	}

	return &tls.Config{
		ClientCAs:  certPool,
		ClientAuth: tls.RequireAndVerifyClientCert,
	}
}

func (a Authenticator) Authenticate(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) {
	if a.AllowedWildcardDomain != "" && strings.HasSuffix(params.Leaf.Subject.CommonName, a.AllowedWildcardDomain) {
		return &advancedtls.VerificationResults{}, nil
	}
	if _, ok := a.AllowedCommonNames[params.Leaf.Subject.CommonName]; ok {
		return &advancedtls.VerificationResults{}, nil
	}
	err := fmt.Errorf("Authenticate: invalid subject client common name: %s", params.Leaf.Subject.CommonName)
	glog.Error(err)
	return nil, err
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import "testing"

func TestLocalPortString(t *testing.T) {
	testCases := []struct {
		description string
		ip          string
		family      IPFamily
		port        int
		protocol    string
		expectedStr string
	}{
		{"IPv4 UDP", "1.2.3.4", "", 9999, "udp", "\"IPv4 UDP\" (1.2.3.4:9999/udp)"},
		{"IPv4 TCP", "5.6.7.8", "", 1053, "tcp", "\"IPv4 TCP\" (5.6.7.8:1053/tcp)"},
		{"IPv4 TCP", "", IPv4, 1053, "tcp", "\"IPv4 TCP\" (:1053/tcp4)"},
		{"IPv6 TCP", "2001:db8::1", "", 80, "tcp", "\"IPv6 TCP\" ([2001:db8::1]:80/tcp)"},
		{"IPv4 SCTP", "9.10.11.12", "", 7777, "sctp", "\"IPv4 SCTP\" (9.10.11.12:7777/sctp)"},
		{"IPv6 SCTP", "2001:db8::2", "", 80, "sctp", "\"IPv6 SCTP\" ([2001:db8::2]:80/sctp)"},
	}

	for _, tc := range testCases {
		lp := &LocalPort{
			Description: tc.description,
			IP:          tc.ip,
			IPFamily:    tc.family,
			Port:        tc.port,
			Protocol:    tc.protocol,
		}
		str := lp.String()
		if str != tc.expectedStr {
			t.Errorf("Unexpected output for %s, expected: %s, got: %s", tc.description, tc.expectedStr, str)
		}
	}
}
<commit_msg>net: Add additional LocalPort tests<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package net

import "testing"

func TestLocalPortString(t *testing.T) {
	testCases := []struct {
		description string
		ip          string
		family      IPFamily
		port        int
		protocol    string
		expectedStr string
	}{
		{"IPv4 UDP", "1.2.3.4", "", 9999, "udp", "\"IPv4 UDP\" (1.2.3.4:9999/udp)"},
		{"IPv4 TCP", "5.6.7.8", "", 1053, "tcp", "\"IPv4 TCP\" (5.6.7.8:1053/tcp)"},
		{"IPv6 TCP", "2001:db8::1", "", 80, "tcp", "\"IPv6 TCP\" ([2001:db8::1]:80/tcp)"},
		{"IPv4 SCTP", "9.10.11.12", "", 7777, "sctp", "\"IPv4 SCTP\" (9.10.11.12:7777/sctp)"},
		{"IPv6 SCTP", "2001:db8::2", "", 80, "sctp", "\"IPv6 SCTP\" ([2001:db8::2]:80/sctp)"},
		{"IPv4 TCP, all addresses", "", IPv4, 1053, "tcp", "\"IPv4 TCP, all addresses\" (:1053/tcp4)"},
		{"IPv6 TCP, all addresses", "", IPv6, 80, "tcp", "\"IPv6 TCP, all addresses\" (:80/tcp6)"},
		{"No ip family TCP, all addresses", "", "", 80, "tcp", "\"No ip family TCP, all addresses\" (:80/tcp)"},
	}

	for _, tc := range testCases {
		lp := &LocalPort{
			Description: tc.description,
			IP:          tc.ip,
			IPFamily:    tc.family,
			Port:        tc.port,
			Protocol:    tc.protocol,
		}
		str := lp.String()
		if str != tc.expectedStr {
			t.Errorf("Unexpected output for %s, expected: %s, got: %s", tc.description, tc.expectedStr, str)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Make fatal logging level exit the application<commit_after><|endoftext|>


<commit_before><commit_msg>[Add]spec: Clct: 定数<commit_after><|endoftext|>


<commit_before>package tnt

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSelect(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
		Space: 15,
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 15 does not exist", err.Error())
}

func TestInsert(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	value1 := uint32(rand.Int31())
	value2 := uint32(rand.Int31())
	value3 := uint32(rand.Int31())
	value4 := uint32(rand.Int31())

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value1),
			PackInt(value3),
		},
	})

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value1),
			PackInt(value4),
		},
	})

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value2),
			PackInt(value4),
		},
	})

	// select 1

	data, err := conn.Execute(&Select{
		Value: PackInt(value1),
	})

	assert.NoError(err)
	assert.Equal(
		[]Tuple{
			Tuple{
				PackInt(value1),
				PackInt(value4),
			},
		},
		data,
	)

	// select 2
	data, err = conn.Execute(&Select{
		Value: PackInt(value4),
		Index: 1,
	})

	assert.NoError(err)
	assert.Equal(2, len(data))
	assert.Equal(Bytes(PackInt(value4)), data[0][1])
	assert.Equal(Bytes(PackInt(value4)), data[1][1])
}

func TestDefaultSpace(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 24 does not exist", err.Error())
}

func TestDefaultSpace2(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 48 does not exist", err.Error())
}
<commit_msg>fix default space tests<commit_after>package tnt

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSelect(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
		Space: 15,
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 15 does not exist", err.Error())
}

func TestInsert(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	value1 := uint32(rand.Int31())
	value2 := uint32(rand.Int31())
	value3 := uint32(rand.Int31())
	value4 := uint32(rand.Int31())

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value1),
			PackInt(value3),
		},
	})

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value1),
			PackInt(value4),
		},
	})

	conn.Execute(&Insert{
		Tuple: Tuple{
			PackInt(value2),
			PackInt(value4),
		},
	})

	// select 1

	data, err := conn.Execute(&Select{
		Value: PackInt(value1),
	})

	assert.NoError(err)
	assert.Equal(
		[]Tuple{
			Tuple{
				PackInt(value1),
				PackInt(value4),
			},
		},
		data,
	)

	// select 2
	data, err = conn.Execute(&Select{
		Value: PackInt(value4),
		Index: 1,
	})

	assert.NoError(err)
	assert.Equal(2, len(data))
	assert.Equal(Bytes(PackInt(value4)), data[0][1])
	assert.Equal(Bytes(PackInt(value4)), data[1][1])
}

func TestDefaultSpace(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d/24", primaryPort), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 24 does not exist", err.Error())
}

func TestDefaultSpace2(t *testing.T) {
	assert := assert.New(t)

	primaryPort, tearDown, err := setUp()
	assert.NoError(err)
	defer tearDown()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d/24", primaryPort), &Options{
		DefaultSpace: 48,
	})
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	data, err := conn.Execute(&Select{
		Value: PackInt(0),
	})
	assert.Nil(data)
	assert.Error(err)
	assert.Equal("Space 48 does not exist", err.Error())
}
<|endoftext|>


<commit_before>package ams

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"
)

func TestClient_GetAsset(t *testing.T) {
	expected := testAsset("sample-id", "Sample")
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')", expected.ID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, expected),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAsset(context.TODO(), expected.ID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_GetAssets(t *testing.T) {
	expected := []Asset{
		testAsset("asset1", "Sample 1"),
		testAsset("asset2", "Sample 2"),
		testAsset("asset4", "Example 1"),
	}
	m := http.NewServeMux()
	m.HandleFunc("/Assets",
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, testWrapValue(expected)),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAssets(context.TODO())
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected assets. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_CreateAsset(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/Assets", func(w http.ResponseWriter, r *http.Request) {
		testRequestMethod(t, r, http.MethodPost)
		testAMSHeader(t, r.Header, false)

		var asset Asset
		if err := json.NewDecoder(r.Body).Decode(&asset); err != nil {
			t.Fatal(err)
		}
		if len(asset.Name) == 0 {
			t.Fatal("Name is required")
		}
		asset.ID = "created-id"
		asset.Created = formatTime(time.Now())
		asset.LastModified = formatTime(time.Now())
		asset.State = StateInitialized
		asset.FormatOption = FormatOptionNoFormat
		asset.StorageAccountName = "sampleStorage"
		asset.URI = "http://your.asset.uri"

		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(asset); err != nil {
			t.Fatal(err)
		}
	})

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	name := "sample"
	asset, err := client.CreateAsset(context.TODO(), name)
	if err != nil {
		t.Error(err)
	}
	if asset.Name != name {
		t.Errorf("unexpected Name. expected: %#v, actual: %#v", name, asset.Name)
	}
}

func TestClient_GetAssetFiles(t *testing.T) {
	assetID := "test-asset-id"
	expected := []AssetFile{
		{
			ID:              "asset-file-1",
			Name:            "sample1",
			ContentFileSize: "0",
			ParentAssetID:   assetID,
			IsPrimary:       false,
			LastModified:    formatTime(time.Now()),
			Created:         formatTime(time.Now()),
			MIMEType:        "text/plain",
			ContentChecksum: "",
		},
		{
			ID:              "asset-file-2",
			Name:            "sample2",
			ContentFileSize: "100000000000000000",
			ParentAssetID:   assetID,
			IsPrimary:       true,
			LastModified:    formatTime(time.Now()),
			Created:         formatTime(time.Now()),
			MIMEType:        "vide/mp4",
			ContentChecksum: "",
		},
	}
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')/Files", assetID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, testWrapValue(expected)),
	)
	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAssetFiles(context.TODO(), assetID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset files. expected: %#v, actual: %#v", expected, actual)
	}
}
<commit_msg>test: add DeleteAsset test<commit_after>package ams

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"
)

func TestClient_GetAsset(t *testing.T) {
	expected := testAsset("sample-id", "Sample")
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')", expected.ID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, expected),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAsset(context.TODO(), expected.ID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_GetAssets(t *testing.T) {
	expected := []Asset{
		testAsset("asset1", "Sample 1"),
		testAsset("asset2", "Sample 2"),
		testAsset("asset4", "Example 1"),
	}
	m := http.NewServeMux()
	m.HandleFunc("/Assets",
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, testWrapValue(expected)),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAssets(context.TODO())
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected assets. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_CreateAsset(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/Assets", func(w http.ResponseWriter, r *http.Request) {
		testRequestMethod(t, r, http.MethodPost)
		testAMSHeader(t, r.Header, false)

		var asset Asset
		if err := json.NewDecoder(r.Body).Decode(&asset); err != nil {
			t.Fatal(err)
		}
		if len(asset.Name) == 0 {
			t.Fatal("Name is required")
		}
		asset.ID = "created-id"
		asset.Created = formatTime(time.Now())
		asset.LastModified = formatTime(time.Now())
		asset.State = StateInitialized
		asset.FormatOption = FormatOptionNoFormat
		asset.StorageAccountName = "sampleStorage"
		asset.URI = "http://your.asset.uri"

		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(asset); err != nil {
			t.Fatal(err)
		}
	})

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	name := "sample"
	asset, err := client.CreateAsset(context.TODO(), name)
	if err != nil {
		t.Error(err)
	}
	if asset.Name != name {
		t.Errorf("unexpected Name. expected: %#v, actual: %#v", name, asset.Name)
	}
}

func TestClient_GetAssetFiles(t *testing.T) {
	assetID := "test-asset-id"
	expected := []AssetFile{
		{
			ID:              "asset-file-1",
			Name:            "sample1",
			ContentFileSize: "0",
			ParentAssetID:   assetID,
			IsPrimary:       false,
			LastModified:    formatTime(time.Now()),
			Created:         formatTime(time.Now()),
			MIMEType:        "text/plain",
			ContentChecksum: "",
		},
		{
			ID:              "asset-file-2",
			Name:            "sample2",
			ContentFileSize: "100000000000000000",
			ParentAssetID:   assetID,
			IsPrimary:       true,
			LastModified:    formatTime(time.Now()),
			Created:         formatTime(time.Now()),
			MIMEType:        "vide/mp4",
			ContentChecksum: "",
		},
	}
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')/Files", assetID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, testWrapValue(expected)),
	)
	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAssetFiles(context.TODO(), assetID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset files. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_DeleteAsset(t *testing.T) {
	assetID := "delete-asset-id"
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')", assetID),
		testJSONHandler(t, http.MethodDelete, false, http.StatusNoContent, nil),
	)
	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	if err := client.DeleteAsset(context.TODO(), assetID); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package qtypes

import (
	"github.com/docker/docker/api/types"
	"fmt"
	"strings"
)
// AssembleServiceSlot create {{.Service.Name}}.{{.Task.Slot}}
func AssembleServiceSlot(cnt *types.Container) string {
	if tn, tnok := cnt.Labels["com.docker.swarm.task.name"]; tnok {
		arr := strings.Split(tn, ".")
		if len(arr) != 3 {
			return "<nil>"
		}
		return fmt.Sprintf("%s.%s", arr[0], arr[1])
	}
	return "<nil>"
}

func AssembleDefaultDimensions(cnt *types.Container) map[string]string {
	dims := map[string]string{
		"container_id":   cnt.ID,
		"container_name": strings.Trim(cnt.Names[0], "/"),
		"image_name":     cnt.Image,
		"service_slot":   AssembleServiceSlot(cnt),
		"command":        strings.Replace(cnt.Command, " ", "#", -1),
		"created":        fmt.Sprintf("%d", cnt.Created),
	}
	for k, v := range cnt.Labels {
		dv := strings.Replace(v, " ", "#", -1)
		dv = strings.Replace(v, ".", "_", -1)
		dims[k] = dv
	}
	return dims
}<commit_msg>AssembleTaskSlot<commit_after>package qtypes

import (
	"github.com/docker/docker/api/types"
	"fmt"
	"strings"
)

// AssembleServiceSlot create {{.Service.Name}}.{{.Task.Slot}}
func AssembleServiceSlot(cnt *types.Container) string {
	if tn, tnok := cnt.Labels["com.docker.swarm.task.name"]; tnok {
		arr := strings.Split(tn, ".")
		if len(arr) != 3 {
			return "<nil>"
		}
		return fmt.Sprintf("%s.%s", arr[0], arr[1])
	}
	return "<nil>"
}

// AssembleServiceSlot create {{.Service.Name}}.{{.Task.Slot}}
func AssembleTaskSlot(cnt *types.Container) string {
	if tn, tnok := cnt.Labels["com.docker.swarm.task.name"]; tnok {
		arr := strings.Split(tn, ".")
		if len(arr) != 3 {
			return "<nil>"
		}
		return arr[1]
	}
	return "<nil>"
}

func AssembleDefaultDimensions(cnt *types.Container) map[string]string {
	dims := map[string]string{
		"container_id":   cnt.ID,
		"container_name": strings.Trim(cnt.Names[0], "/"),
		"image_name":     cnt.Image,
		"service_slot":   AssembleServiceSlot(cnt),
		"task_slot":   	  AssembleTaskSlot(cnt),
		"command":        strings.Replace(cnt.Command, " ", "#", -1),
		"created":        fmt.Sprintf("%d", cnt.Created),
	}
	for k, v := range cnt.Labels {
		dv := strings.Replace(v, " ", "#", -1)
		dv = strings.Replace(v, ".", "_", -1)
		dims[k] = dv
	}
	return dims
}<|endoftext|>


<commit_before>// Copyright 2014 Fredrik Ehnbom
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package container

import (
	"fmt"
	"github.com/quarnster/util"
	"sort"
)

var (
	ErrNotInt           = fmt.Errorf("Attempting to insert a non-int type")
	ErrIndexOOB         = fmt.Errorf("Index is out of bounds")
	ErrNotManipulatable = fmt.Errorf("Filtered arrays are not directly manipulatable")
)

type (
	RemovedData struct {
		Index int
		Data  interface{}
	}

	InsertedData struct {
		Index int
	}

	Array interface {
		Insert(index int, data interface{}) error
		Remove(i int) (olddata interface{}, err error)
		Get(index int) interface{}
		Len() int
	}
	IntArray struct {
		model []int
	}
	BasicArray struct {
		model []interface{}
	}
	BoundsCheckingArray struct {
		Array
	}
	ObservableArray struct {
		util.BasicObservable
		Array
	}
	Acceptable    func(data interface{}) bool
	filteredArray struct {
		indices IntArray
		accept  Acceptable
		Array
	}
)

func NewFilteredArray(inner Array, accept Acceptable) Array {
	if _, ok := inner.(util.Observable); !ok {
		inner = &ObservableArray{Array: inner}
	}
	fa := filteredArray{accept: accept, Array: inner}
	inner.(util.Observable).AddObserver(&fa)
	for i := 0; i < inner.Len(); i++ {
		if !accept(inner.Get(i)) {
			continue
		}
		fa.indices.Insert(fa.indices.Len(), i)
	}
	return &fa
}

func (b *BoundsCheckingArray) Insert(index int, data interface{}) error {
	if index < 0 || index > b.Len() {
		return ErrIndexOOB
	}
	return b.Array.Insert(index, data)
}

func (b *BoundsCheckingArray) Remove(index int) (interface{}, error) {
	if index < 0 || index >= b.Len() {
		return nil, ErrIndexOOB
	}
	return b.Array.Remove(index)
}

func (b *BoundsCheckingArray) Get(index int) interface{} {
	if index < 0 || index >= b.Len() {
		return nil
	}
	return b.Array.Get(index)
}

func (i *IntArray) Insert(index int, data interface{}) error {
	ii, ok := data.(int)
	if !ok {
		return ErrNotInt
	}
	nmodel := make([]int, len(i.model)+1)
	copy(nmodel, i.model[:index])
	nmodel[index] = ii
	copy(nmodel[index+1:], i.model[index:])
	return nil
}

func (i *IntArray) Remove(index int) (olddata interface{}, err error) {
	olddata = i.model[index]
	copy(i.model[index:], i.model[index+1:])
	return olddata, nil
}

func (i *IntArray) Get(index int) interface{} {
	return i.model[index]
}

func (i *IntArray) Len() int {
	return len(i.model)
}

func (a *BasicArray) Insert(index int, data interface{}) error {
	nmodel := make([]interface{}, len(a.model)+1)
	copy(nmodel, a.model[:index])
	nmodel[index] = data
	copy(nmodel[index+1:], a.model[index:])
	return nil
}

func (a *BasicArray) Remove(i int) (olddata interface{}, err error) {
	olddata = a.model[i]
	copy(a.model[i:], a.model[i+1:])
	return olddata, nil
}

func (a *BasicArray) Get(index int) interface{} {
	return a.model[index]
}

func (a *BasicArray) Len() int {
	return len(a.model)
}

func (a *ObservableArray) Insert(index int, data interface{}) error {
	if err := a.Array.Insert(index, data); err != nil {
		return err
	}
	a.NotifyObservers(InsertedData{index})
	return nil
}

func (a *ObservableArray) Remove(i int) (olddata interface{}, err error) {
	if olddata, err = a.Array.Remove(i); err != nil {
		return
	}
	a.NotifyObservers(RemovedData{i, olddata})
	return
}

func (fa *filteredArray) Changed(data interface{}) {
	switch d := data.(type) {
	case RemovedData:
		for i, k := range fa.indices.model {
			if k == d.Index {
				fa.indices.Remove(i)
				break
			}
		}
	case InsertedData:
		data := fa.Get(d.Index)
		if !fa.accept(data) {
			return
		}
		idx := sort.Search(fa.indices.Len(), func(i int) bool {
			return fa.Get(i).(int) < d.Index
		})
		fa.indices.Insert(idx+1, data)
	}
}

func (fa *filteredArray) Insert(index int, data interface{}) error {
	return ErrNotManipulatable
}

func (fa *filteredArray) Remove(index int) (interface{}, error) {
	return nil, ErrNotManipulatable
}

func (fa *filteredArray) Get(index int) interface{} {
	return fa.Array.Get(fa.indices.Get(index).(int))
}
func (fa *filteredArray) Len() int {
	return fa.indices.Len()
}
<commit_msg>container/array: Insert the index rather than the data in fa.Changed<commit_after>// Copyright 2014 Fredrik Ehnbom
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package container

import (
	"fmt"
	"github.com/quarnster/util"
	"sort"
)

var (
	ErrNotInt           = fmt.Errorf("Attempting to insert a non-int type")
	ErrIndexOOB         = fmt.Errorf("Index is out of bounds")
	ErrNotManipulatable = fmt.Errorf("Filtered arrays are not directly manipulatable")
)

type (
	RemovedData struct {
		Index int
		Data  interface{}
	}

	InsertedData struct {
		Index int
	}

	Array interface {
		Insert(index int, data interface{}) error
		Remove(i int) (olddata interface{}, err error)
		Get(index int) interface{}
		Len() int
	}
	IntArray struct {
		model []int
	}
	BasicArray struct {
		model []interface{}
	}
	BoundsCheckingArray struct {
		Array
	}
	ObservableArray struct {
		util.BasicObservable
		Array
	}
	Acceptable    func(data interface{}) bool
	filteredArray struct {
		indices IntArray
		accept  Acceptable
		Array
	}
)

func NewFilteredArray(inner Array, accept Acceptable) Array {
	if _, ok := inner.(util.Observable); !ok {
		inner = &ObservableArray{Array: inner}
	}
	fa := filteredArray{accept: accept, Array: inner}
	inner.(util.Observable).AddObserver(&fa)
	for i := 0; i < inner.Len(); i++ {
		if !accept(inner.Get(i)) {
			continue
		}
		fa.indices.Insert(fa.indices.Len(), i)
	}
	return &fa
}

func (b *BoundsCheckingArray) Insert(index int, data interface{}) error {
	if index < 0 || index > b.Len() {
		return ErrIndexOOB
	}
	return b.Array.Insert(index, data)
}

func (b *BoundsCheckingArray) Remove(index int) (interface{}, error) {
	if index < 0 || index >= b.Len() {
		return nil, ErrIndexOOB
	}
	return b.Array.Remove(index)
}

func (b *BoundsCheckingArray) Get(index int) interface{} {
	if index < 0 || index >= b.Len() {
		return nil
	}
	return b.Array.Get(index)
}

func (i *IntArray) Insert(index int, data interface{}) error {
	ii, ok := data.(int)
	if !ok {
		return ErrNotInt
	}
	nmodel := make([]int, len(i.model)+1)
	copy(nmodel, i.model[:index])
	nmodel[index] = ii
	copy(nmodel[index+1:], i.model[index:])
	return nil
}

func (i *IntArray) Remove(index int) (olddata interface{}, err error) {
	olddata = i.model[index]
	copy(i.model[index:], i.model[index+1:])
	return olddata, nil
}

func (i *IntArray) Get(index int) interface{} {
	return i.model[index]
}

func (i *IntArray) Len() int {
	return len(i.model)
}

func (a *BasicArray) Insert(index int, data interface{}) error {
	nmodel := make([]interface{}, len(a.model)+1)
	copy(nmodel, a.model[:index])
	nmodel[index] = data
	copy(nmodel[index+1:], a.model[index:])
	return nil
}

func (a *BasicArray) Remove(i int) (olddata interface{}, err error) {
	olddata = a.model[i]
	copy(a.model[i:], a.model[i+1:])
	return olddata, nil
}

func (a *BasicArray) Get(index int) interface{} {
	return a.model[index]
}

func (a *BasicArray) Len() int {
	return len(a.model)
}

func (a *ObservableArray) Insert(index int, data interface{}) error {
	if err := a.Array.Insert(index, data); err != nil {
		return err
	}
	a.NotifyObservers(InsertedData{index})
	return nil
}

func (a *ObservableArray) Remove(i int) (olddata interface{}, err error) {
	if olddata, err = a.Array.Remove(i); err != nil {
		return
	}
	a.NotifyObservers(RemovedData{i, olddata})
	return
}

func (fa *filteredArray) Changed(data interface{}) {
	switch d := data.(type) {
	case RemovedData:
		for i, k := range fa.indices.model {
			if k == d.Index {
				fa.indices.Remove(i)
				break
			}
		}
	case InsertedData:
		data := fa.Get(d.Index)
		if !fa.accept(data) {
			return
		}
		idx := sort.Search(fa.indices.Len(), func(i int) bool {
			return fa.Get(i).(int) < d.Index
		})
		fa.indices.Insert(idx+1, d.Index)
	}
}

func (fa *filteredArray) Insert(index int, data interface{}) error {
	return ErrNotManipulatable
}

func (fa *filteredArray) Remove(index int) (interface{}, error) {
	return nil, ErrNotManipulatable
}

func (fa *filteredArray) Get(index int) interface{} {
	return fa.Array.Get(fa.indices.Get(index).(int))
}
func (fa *filteredArray) Len() int {
	return fa.indices.Len()
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package content

import (
	"context"
	"io"
	"io/ioutil"
	"math/rand"
	"sync"
	"time"

	"github.com/containerd/containerd/errdefs"
	"github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

var bufPool = sync.Pool{
	New: func() interface{} {
		buffer := make([]byte, 1<<20)
		return &buffer
	},
}

// NewReader returns a io.Reader from a ReaderAt
func NewReader(ra ReaderAt) io.Reader {
	rd := io.NewSectionReader(ra, 0, ra.Size())
	return rd
}

// ReadBlob retrieves the entire contents of the blob from the provider.
//
// Avoid using this for large blobs, such as layers.
func ReadBlob(ctx context.Context, provider Provider, desc ocispec.Descriptor) ([]byte, error) {
	ra, err := provider.ReaderAt(ctx, desc)
	if err != nil {
		return nil, err
	}
	defer ra.Close()

	p := make([]byte, ra.Size())

	_, err = ra.ReadAt(p, 0)
	return p, err
}

// WriteBlob writes data with the expected digest into the content store. If
// expected already exists, the method returns immediately and the reader will
// not be consumed.
//
// This is useful when the digest and size are known beforehand.
//
// Copy is buffered, so no need to wrap reader in buffered io.
func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc ocispec.Descriptor, opts ...Opt) error {
	cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc))
	if err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return errors.Wrap(err, "failed to open writer")
		}

		return nil // all ready present
	}
	defer cw.Close()

	return Copy(ctx, cw, r, desc.Size, desc.Digest, opts...)
}

// OpenWriter opens a new writer for the given reference, retrying if the writer
// is locked until the reference is available or returns an error.
func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, error) {
	var (
		cw    Writer
		err   error
		retry = 16
	)
	for {
		cw, err = cs.Writer(ctx, opts...)
		if err != nil {
			if !errdefs.IsUnavailable(err) {
				return nil, err
			}

			// TODO: Check status to determine if the writer is active,
			// continue waiting while active, otherwise return lock
			// error or abort. Requires asserting for an ingest manager

			select {
			case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
				if retry < 2048 {
					retry = retry << 1
				}
				continue
			case <-ctx.Done():
				// Propagate lock error
				return nil, err
			}

		}
		break
	}

	return cw, err
}

// Copy copies data with the expected digest from the reader into the
// provided content store writer. This copy commits the writer.
//
// This is useful when the digest and size are known beforehand. When
// the size or digest is unknown, these values may be empty.
//
// Copy is buffered, so no need to wrap reader in buffered io.
func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error {
	ws, err := cw.Status()
	if err != nil {
		return errors.Wrap(err, "failed to get status")
	}

	if ws.Offset > 0 {
		r, err = seekReader(r, ws.Offset, size)
		if err != nil {
			return errors.Wrapf(err, "unable to resume write to %v", ws.Ref)
		}
	}

	if _, err := copyWithBuffer(cw, r); err != nil {
		return errors.Wrap(err, "failed to copy")
	}

	if err := cw.Commit(ctx, size, expected, opts...); err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return errors.Wrapf(err, "failed commit on ref %q", ws.Ref)
		}
	}

	return nil
}

// CopyReaderAt copies to a writer from a given reader at for the given
// number of bytes. This copy does not commit the writer.
func CopyReaderAt(cw Writer, ra ReaderAt, n int64) error {
	ws, err := cw.Status()
	if err != nil {
		return err
	}

	_, err = copyWithBuffer(cw, io.NewSectionReader(ra, ws.Offset, n))
	return err
}

// seekReader attempts to seek the reader to the given offset, either by
// resolving `io.Seeker`, by detecting `io.ReaderAt`, or discarding
// up to the given offset.
func seekReader(r io.Reader, offset, size int64) (io.Reader, error) {
	// attempt to resolve r as a seeker and setup the offset.
	seeker, ok := r.(io.Seeker)
	if ok {
		nn, err := seeker.Seek(offset, io.SeekStart)
		if nn != offset {
			return nil, errors.Wrapf(err, "failed to seek to offset %v", offset)
		}

		if err != nil {
			return nil, err
		}

		return r, nil
	}

	// ok, let's try io.ReaderAt!
	readerAt, ok := r.(io.ReaderAt)
	if ok && size > offset {
		sr := io.NewSectionReader(readerAt, offset, size)
		return sr, nil
	}

	// well then, let's just discard up to the offset
	n, err := copyWithBuffer(ioutil.Discard, io.LimitReader(r, offset))
	if err != nil {
		return nil, errors.Wrap(err, "failed to discard to offset")
	}
	if n != offset {
		return nil, errors.Errorf("unable to discard to offset")
	}

	return r, nil
}

func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
	buf := bufPool.Get().(*[]byte)
	written, err = io.CopyBuffer(dst, src, *buf)
	bufPool.Put(buf)
	return
}
<commit_msg>Handle EOF from ReadAt in content.ReadBlob<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package content

import (
	"context"
	"io"
	"io/ioutil"
	"math/rand"
	"sync"
	"time"

	"github.com/containerd/containerd/errdefs"
	"github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

var bufPool = sync.Pool{
	New: func() interface{} {
		buffer := make([]byte, 1<<20)
		return &buffer
	},
}

// NewReader returns a io.Reader from a ReaderAt
func NewReader(ra ReaderAt) io.Reader {
	rd := io.NewSectionReader(ra, 0, ra.Size())
	return rd
}

// ReadBlob retrieves the entire contents of the blob from the provider.
//
// Avoid using this for large blobs, such as layers.
func ReadBlob(ctx context.Context, provider Provider, desc ocispec.Descriptor) ([]byte, error) {
	ra, err := provider.ReaderAt(ctx, desc)
	if err != nil {
		return nil, err
	}
	defer ra.Close()

	p := make([]byte, ra.Size())

	n, err := ra.ReadAt(p, 0)
	if err == io.EOF {
		if int64(n) != ra.Size() {
			err = io.ErrUnexpectedEOF
		} else {
			err = nil
		}
	}
	return p, err
}

// WriteBlob writes data with the expected digest into the content store. If
// expected already exists, the method returns immediately and the reader will
// not be consumed.
//
// This is useful when the digest and size are known beforehand.
//
// Copy is buffered, so no need to wrap reader in buffered io.
func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc ocispec.Descriptor, opts ...Opt) error {
	cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc))
	if err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return errors.Wrap(err, "failed to open writer")
		}

		return nil // all ready present
	}
	defer cw.Close()

	return Copy(ctx, cw, r, desc.Size, desc.Digest, opts...)
}

// OpenWriter opens a new writer for the given reference, retrying if the writer
// is locked until the reference is available or returns an error.
func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, error) {
	var (
		cw    Writer
		err   error
		retry = 16
	)
	for {
		cw, err = cs.Writer(ctx, opts...)
		if err != nil {
			if !errdefs.IsUnavailable(err) {
				return nil, err
			}

			// TODO: Check status to determine if the writer is active,
			// continue waiting while active, otherwise return lock
			// error or abort. Requires asserting for an ingest manager

			select {
			case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))):
				if retry < 2048 {
					retry = retry << 1
				}
				continue
			case <-ctx.Done():
				// Propagate lock error
				return nil, err
			}

		}
		break
	}

	return cw, err
}

// Copy copies data with the expected digest from the reader into the
// provided content store writer. This copy commits the writer.
//
// This is useful when the digest and size are known beforehand. When
// the size or digest is unknown, these values may be empty.
//
// Copy is buffered, so no need to wrap reader in buffered io.
func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error {
	ws, err := cw.Status()
	if err != nil {
		return errors.Wrap(err, "failed to get status")
	}

	if ws.Offset > 0 {
		r, err = seekReader(r, ws.Offset, size)
		if err != nil {
			return errors.Wrapf(err, "unable to resume write to %v", ws.Ref)
		}
	}

	if _, err := copyWithBuffer(cw, r); err != nil {
		return errors.Wrap(err, "failed to copy")
	}

	if err := cw.Commit(ctx, size, expected, opts...); err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return errors.Wrapf(err, "failed commit on ref %q", ws.Ref)
		}
	}

	return nil
}

// CopyReaderAt copies to a writer from a given reader at for the given
// number of bytes. This copy does not commit the writer.
func CopyReaderAt(cw Writer, ra ReaderAt, n int64) error {
	ws, err := cw.Status()
	if err != nil {
		return err
	}

	_, err = copyWithBuffer(cw, io.NewSectionReader(ra, ws.Offset, n))
	return err
}

// seekReader attempts to seek the reader to the given offset, either by
// resolving `io.Seeker`, by detecting `io.ReaderAt`, or discarding
// up to the given offset.
func seekReader(r io.Reader, offset, size int64) (io.Reader, error) {
	// attempt to resolve r as a seeker and setup the offset.
	seeker, ok := r.(io.Seeker)
	if ok {
		nn, err := seeker.Seek(offset, io.SeekStart)
		if nn != offset {
			return nil, errors.Wrapf(err, "failed to seek to offset %v", offset)
		}

		if err != nil {
			return nil, err
		}

		return r, nil
	}

	// ok, let's try io.ReaderAt!
	readerAt, ok := r.(io.ReaderAt)
	if ok && size > offset {
		sr := io.NewSectionReader(readerAt, offset, size)
		return sr, nil
	}

	// well then, let's just discard up to the offset
	n, err := copyWithBuffer(ioutil.Discard, io.LimitReader(r, offset))
	if err != nil {
		return nil, errors.Wrap(err, "failed to discard to offset")
	}
	if n != offset {
		return nil, errors.Errorf("unable to discard to offset")
	}

	return r, nil
}

func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
	buf := bufPool.Get().(*[]byte)
	written, err = io.CopyBuffer(dst, src, *buf)
	bufPool.Put(buf)
	return
}
<|endoftext|>


<commit_before>package atime

import (
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestStat(t *testing.T) {
	f, err := ioutil.TempFile("", "")
	et := time.Now().Add(-time.Second)
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(f.Name())
	defer f.Close()

	at, err := Stat(f.Name())
	if err != nil {
		t.Error(err.Error())
	}
	if at.Before(et) {
		t.Errorf("expected atime to be recent: got %v instead of ~%v", at, et)
	}
}

func TestStatErr(t *testing.T) {
	_, err := Stat("badfile?")
	if err == nil {
		t.Error("expected an error")
	}
}
<commit_msg>adding a test for Get<commit_after>package atime

import (
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestStat(t *testing.T) {
	f, err := ioutil.TempFile("", "")
	et := time.Now().Add(-time.Second)
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(f.Name())
	defer f.Close()

	at, err := Stat(f.Name())
	if err != nil {
		t.Error(err.Error())
	}
	if at.Before(et) {
		t.Errorf("expected atime to be recent: got %v instead of ~%v", at, et)
	}
}

func TestGet(t *testing.T) {
	f, err := ioutil.TempFile("", "")
	et := time.Now().Add(-time.Second)
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(f.Name())
	defer f.Close()

	fi, err := os.Stat(f.Name())
	if err != nil {
		t.Error(err.Error())
	}
	at := Get(fi)
	if at.Before(et) {
		t.Errorf("expected atime to be recent: got %v instead of ~%v", at, et)
	}
}

func TestStatErr(t *testing.T) {
	_, err := Stat("badfile?")
	if err == nil {
		t.Error("expected an error")
	}
}
<|endoftext|>