<commit_before>package elastic

import ()

const (
	// Aggs abreviateed constant name for the Aggregation query.
	Aggs = "aggs"
	// Aggregations constant name for the Aggregation query.
	Aggregations = "aggregations"
	// Terms constant name of terms Bucket
	Terms = "terms"
	// Histogram constant name of the Histogram bucket
	Histogram = "histogram"
	// DateHistogram constant name of the Date Histogram bucket.
	DateHistogram = "date_histogram"
	// Global constant name of the global bucket which is used to by pass aggregation scope.
	Global = "global"
	// FilterBucket constant name of filter bucket which is used to filter aggregation results.
	FilterBucket = "filter"
)

// Constant name of Elasticsearch metrics
const (
	// Count constant name of 'count' metric.
	Count = "count"
	// Sum constant name of 'sum' metric.
	Sum = "sum"
	// Avg constant name of 'avg' metric.
	Avg = "avg"
	// Min constant name of 'min' metric.
	Min = "min"
	// Max constant name of 'max' metric.
	Max = "max"
	// ExtendedStats constant name of a metric that will return a variety of statistics (e.g. stats.avg, stats.count, stats.std_deviation).
	ExtendedStats = "extended_stats"
)

const (
	Field = "field"
	// Interval name of parameter that define a histogram interval, i.e. the value that Elasticsearch will use to create new buckets.
	Interval = "interval"
	// Size name of parameter that defines how many terms we want to generate. Example of values, for histograms: 10, for date histograms: "month", "quarter".
	Size = "size"
	// Format name of parameter in date histogram, used to define the  dates format for bucket keys.
	Format = "format"
	// MinDocCount name of parameter in date histogram, used to force empty buckets to be returned.
	MinDocCount = "min_doc_count"
	// ExtendedBound name of parameter in date histogram. It is used to extend the boudaries of bucket from the boudaries of actual data. This, it forces all bucket betwen the min and max bound to be returned.
	ExtendedBound = "extended_bound"
)

type Aggregation struct {
	client *Elasticsearch
	parser Parser
	url    string
	params map[string]string
	query  Dict
}

func (client *Elasticsearch) Aggs(index, doc string) *Aggregation {
	url := client.request(index, doc, -1, SEARCH)
	return &Aggregation{
		client: client,
		url:    url,
		params: make(map[string]string),
		query:  make(Dict),
	}
}

// urlString constructs the url of this Search API call
func (agg *Aggregation) urlString() string {
	return urlString(agg.url, agg.params)
}

// String returns a string representation of this Search API call
func (agg *Aggregation) String() string {
	body := ""
	if len(agg.query) > 0 {
		dict := make(Dict)
		dict[Aggs] = agg.query
		body = String(dict)
	}
	return body
}

// Get submits request mappings between the json fields and how Elasticsearch store them
// GET /:index/:type/_search
func (agg *Aggregation) Get() {
	// construct the url
	url := agg.urlString()
	// construct the body
	query := agg.String()

	agg.client.Execute("GET", url, query, agg.parser)
}

func (agg *Aggregation) SetMetric(name string) *Aggregation {
	agg.params[SearchType] = name
	return agg
}

type Bucket struct {
	name  string
	query Dict
}

func NewBucket(name string) *Bucket {
	return &Bucket{
		name:  name,
		query: make(Dict),
	}
}

func (bucket *Bucket) AddTerm(name string, value interface{}) *Bucket {
	bucket.AddMetric(Terms, name, value)
	return bucket
}

func (bucket *Bucket) AddMetric(metric, name string, value interface{}) *Bucket {
	if bucket.query[metric] == nil {
		bucket.query[metric] = make(Dict)
	}
	bucket.query[metric].(Dict)[name] = value
	return bucket
}

func (bucket *Bucket) AddDict(name string, value Dict) *Bucket {
	bucket.query[name] = value
	return bucket
}

func (bucket *Bucket) AddBucket(b *Bucket) *Bucket {
	if bucket.query[Aggs] == nil {
		bucket.query[Aggs] = make(Dict)
	}
	bucket.query[Aggs].(Dict)[b.name] = b.query
	return bucket
}

func (agg *Aggregation) Add(bucket *Bucket) *Aggregation {
	agg.query[bucket.name] = bucket.query
	return agg
}

func (agg *Aggregation) AddQuery(q Query) *Aggregation {
	if agg.query["query"] == nil {
		agg.query["query"] = make(Dict)
	}
	agg.query["query"].(Dict)[q.Name()] = q.KV()
	return agg
}

func (agg *Aggregation) AddPostFilter(q Query) *Aggregation {
	if agg.query[PostFilter] == nil {
		agg.query[PostFilter] = make(Dict)
	}
	agg.query[PostFilter].(Dict)[q.Name()] = q.KV()
	return agg
}
<commit_msg>fixed 'query' was not in top level in request body<commit_after>package elastic

import ()

const (
	// Aggs abreviateed constant name for the Aggregation query.
	Aggs = "aggs"
	// Aggregations constant name for the Aggregation query.
	Aggregations = "aggregations"
	// Terms constant name of terms Bucket
	Terms = "terms"
	// Histogram constant name of the Histogram bucket
	Histogram = "histogram"
	// DateHistogram constant name of the Date Histogram bucket.
	DateHistogram = "date_histogram"
	// Global constant name of the global bucket which is used to by pass aggregation scope.
	Global = "global"
	// FilterBucket constant name of filter bucket which is used to filter aggregation results.
	FilterBucket = "filter"
)

// Constant name of Elasticsearch metrics
const (
	// Count constant name of 'count' metric.
	Count = "count"
	// Sum constant name of 'sum' metric.
	Sum = "sum"
	// Avg constant name of 'avg' metric.
	Avg = "avg"
	// Min constant name of 'min' metric.
	Min = "min"
	// Max constant name of 'max' metric.
	Max = "max"
	// ExtendedStats constant name of a metric that will return a variety of statistics (e.g. stats.avg, stats.count, stats.std_deviation).
	ExtendedStats = "extended_stats"
)

const (
	Field = "field"
	// Interval name of parameter that define a histogram interval, i.e. the value that Elasticsearch will use to create new buckets.
	Interval = "interval"
	// Size name of parameter that defines how many terms we want to generate. Example of values, for histograms: 10, for date histograms: "month", "quarter".
	Size = "size"
	// Format name of parameter in date histogram, used to define the  dates format for bucket keys.
	Format = "format"
	// MinDocCount name of parameter in date histogram, used to force empty buckets to be returned.
	MinDocCount = "min_doc_count"
	// ExtendedBound name of parameter in date histogram. It is used to extend the boudaries of bucket from the boudaries of actual data. This, it forces all bucket betwen the min and max bound to be returned.
	ExtendedBound = "extended_bound"
)

type Aggregation struct {
	client *Elasticsearch
	parser Parser
	url    string
	params map[string]string
	query  Dict
}

func (client *Elasticsearch) Aggs(index, doc string) *Aggregation {
	url := client.request(index, doc, -1, SEARCH)
	return &Aggregation{
		client: client,
		url:    url,
		params: make(map[string]string),
		query:  make(Dict),
	}
}

// urlString constructs the url of this Search API call
func (agg *Aggregation) urlString() string {
	return urlString(agg.url, agg.params)
}

// String returns a string representation of this Search API call
func (agg *Aggregation) String() string {
	body := ""
	if len(agg.query) > 0 {
		body = String(agg.query)
	}
	return body
}

// Get submits request mappings between the json fields and how Elasticsearch store them
// GET /:index/:type/_search
func (agg *Aggregation) Get() {
	// construct the url
	url := agg.urlString()
	// construct the body
	query := agg.String()

	agg.client.Execute("GET", url, query, agg.parser)
}

func (agg *Aggregation) SetMetric(name string) *Aggregation {
	agg.params[SearchType] = name
	return agg
}

type Bucket struct {
	name  string
	query Dict
}

func NewBucket(name string) *Bucket {
	return &Bucket{
		name:  name,
		query: make(Dict),
	}
}

func (bucket *Bucket) AddTerm(name string, value interface{}) *Bucket {
	bucket.AddMetric(Terms, name, value)
	return bucket
}

func (bucket *Bucket) AddMetric(metric, name string, value interface{}) *Bucket {
	if bucket.query[metric] == nil {
		bucket.query[metric] = make(Dict)
	}
	bucket.query[metric].(Dict)[name] = value
	return bucket
}

func (bucket *Bucket) AddDict(name string, value Dict) *Bucket {
	bucket.query[name] = value
	return bucket
}

func (bucket *Bucket) AddBucket(b *Bucket) *Bucket {
	if bucket.query[Aggs] == nil {
		bucket.query[Aggs] = make(Dict)
	}
	bucket.query[Aggs].(Dict)[b.name] = b.query
	return bucket
}

func (agg *Aggregation) Add(bucket *Bucket) *Aggregation {
	if agg.query[Aggs] == nil {
		agg.query[Aggs] = make(Dict)
	}
	aggs := agg.query[Aggs].(Dict)
	aggs[bucket.name] = bucket.query
	return agg
}

func (agg *Aggregation) AddQuery(q Query) *Aggregation {
	if agg.query["query"] == nil {
		agg.query["query"] = make(Dict)
	}
	agg.query["query"].(Dict)[q.Name()] = q.KV()
	return agg
}

func (agg *Aggregation) AddPostFilter(q Query) *Aggregation {
	if agg.query[PostFilter] == nil {
		agg.query[PostFilter] = make(Dict)
	}
	agg.query[PostFilter].(Dict)[q.Name()] = q.KV()
	return agg
}
<|endoftext|>


<commit_before>package stash

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// MarshalJSON implements custom serialization for this type
func (v *PropertyValue) MarshalJSON() ([]byte, error) {
	raw := make([]interface{}, 2)
	raw[0] = v.Value
	raw[1] = v.PrintKey

	return json.Marshal(raw)
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Regexes we use for ItemMod parsing
var numberRegex = regexp.MustCompile(`\d{1,}`) // Grabbing digits
var hashRegex = regexp.MustCompile("#")        // Filling templates

// ItemMod is a modifier an item can have
type ItemMod struct {
	Template []byte
	Values   []int
}

// MarshalJSON implements custom serialization for this type
func (m *ItemMod) MarshalJSON() ([]byte, error) {
	result := []byte(m.Template)

	for _, value := range m.Values {

		loc := hashRegex.FindIndex(result)
		if loc == nil {
			return nil, fmt.Errorf("failed to match template string slot with expected value")
		}

		insert := []byte(strconv.Itoa(value))

		portion := hashRegex.ReplaceAll(result[:loc[1]], insert)
		result = append(portion, result[loc[1]:]...)
	}

	return result, nil
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return string which we run through
// regexp for ease
func (m *ItemMod) UnmarshalJSON(b []byte) error {
	// Find magnitudes in the mod
	matches := numberRegex.FindAll(b, -1)

	// Grab the template string representing the mod
	m.Template = numberRegex.ReplaceAll(b, []byte("#"))

	// Convert the matches to numbers
	m.Values = make([]int, len(matches))
	var err error
	for i, match := range matches {
		m.Values[i], err = strconv.Atoi(string(match))
		if err != nil {
			return fmt.Errorf("failed to convert match to int, err=%s", err)
		}
	}

	return nil
}

// Item represents a single item found from the stash api
type Item struct {
	Verified     bool      `json:"verified"`
	Ilvl         int       `json:"ilvl"`
	Icon         string    `json:"icon"`
	League       string    `json:"league"`
	ID           string    `json:"id"`
	Name         string    `json:"name"`
	TypeLine     string    `json:"typeLine"`
	Identified   bool      `json:"identified"`
	Corrupted    bool      `json:"corrupted"`
	ImplicitMods []ItemMod `json:"implicitMods,omitempty"`
	ExplicitMods []ItemMod `json:"explicitMods,omitempty"`
	FlavourText  []string  `json:"flavourText,omitempty"`
	Note         string    `json:"note,omitempty"`
	Properties   []struct {
		Name        string          `json:"name"`
		Values      []PropertyValue `json:"values"`
		DisplayMode int             `json:"displayMode"`
	} `json:"properties,omitempty"`
	UtilityMods []string `json:"utilityMods,omitempty"`
	DescrText   string   `json:"descrText,omitempty"`
}

// Response represents expected structure of a stash api call
type Response struct {
	NextChangeID string `json:"next_change_id"`
	Stashes      []struct {
		AccountName       string `json:"accountName"`
		LastCharacterName string `json:"lastCharacterName"`
		ID                string `json:"id"`
		Stash             string `json:"stash"`
		StashType         string `json:"stashType"`
		Items             []Item `json:"items"`
		Public            bool   `json:"public"`
	} `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	decoder := json.NewDecoder(f)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := json.Marshal(response)
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<commit_msg>stash hoist nested anonymous structs<commit_after>package stash

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// MarshalJSON implements custom serialization for this type
func (v *PropertyValue) MarshalJSON() ([]byte, error) {
	raw := make([]interface{}, 2)
	raw[0] = v.Value
	raw[1] = v.PrintKey

	return json.Marshal(raw)
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Regexes we use for ItemMod parsing
var numberRegex = regexp.MustCompile(`\d{1,}`) // Grabbing digits
var hashRegex = regexp.MustCompile("#")        // Filling templates

// ItemMod is a modifier an item can have
type ItemMod struct {
	Template []byte
	Values   []int
}

// MarshalJSON implements custom serialization for this type
func (m *ItemMod) MarshalJSON() ([]byte, error) {
	result := []byte(m.Template)

	for _, value := range m.Values {

		loc := hashRegex.FindIndex(result)
		if loc == nil {
			return nil, fmt.Errorf("failed to match template string slot with expected value")
		}

		insert := []byte(strconv.Itoa(value))

		portion := hashRegex.ReplaceAll(result[:loc[1]], insert)
		result = append(portion, result[loc[1]:]...)
	}

	return result, nil
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return string which we run through
// regexp for ease
func (m *ItemMod) UnmarshalJSON(b []byte) error {
	// Find magnitudes in the mod
	matches := numberRegex.FindAll(b, -1)

	// Grab the template string representing the mod
	m.Template = numberRegex.ReplaceAll(b, []byte("#"))

	// Convert the matches to numbers
	m.Values = make([]int, len(matches))
	var err error
	for i, match := range matches {
		m.Values[i], err = strconv.Atoi(string(match))
		if err != nil {
			return fmt.Errorf("failed to convert match to int, err=%s", err)
		}
	}

	return nil
}

// Property represents a single item property
type Property struct {
	Name        string          `json:"name"`
	Values      []PropertyValue `json:"values"`
	DisplayMode int             `json:"displayMode"`
}

// Item represents a single item found from the stash api
type Item struct {
	Verified     bool       `json:"verified"`
	Ilvl         int        `json:"ilvl"`
	Icon         string     `json:"icon"`
	League       string     `json:"league"`
	ID           string     `json:"id"`
	Name         string     `json:"name"`
	TypeLine     string     `json:"typeLine"`
	Identified   bool       `json:"identified"`
	Corrupted    bool       `json:"corrupted"`
	ImplicitMods []ItemMod  `json:"implicitMods,omitempty"`
	ExplicitMods []ItemMod  `json:"explicitMods,omitempty"`
	FlavourText  []string   `json:"flavourText,omitempty"`
	Note         string     `json:"note,omitempty"`
	Properties   []Property `json:"properties,omitempty"`
	UtilityMods  []string   `json:"utilityMods,omitempty"`
	DescrText    string     `json:"descrText,omitempty"`
}

// Stash represents a stash tab with items and associated metadata
type Stash struct {
	AccountName       string `json:"accountName"`
	LastCharacterName string `json:"lastCharacterName"`
	ID                string `json:"id"`
	Stash             string `json:"stash"`
	StashType         string `json:"stashType"`
	Items             []Item `json:"items"`
	Public            bool   `json:"public"`
}

// Response represents expected structure of a stash api call
type Response struct {
	NextChangeID string  `json:"next_change_id"`
	Stashes      []Stash `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	decoder := json.NewDecoder(f)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := json.Marshal(response)
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package api_test

import (
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/baggageclaim/api"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"net/http"
	"net/http/httptest"
	"regexp"
)

var _ = Describe("P2P Server", func() {
	var (
		handler http.Handler
		infc string
	)

	JustBeforeEach(func() {
		var err error
		logger := lagertest.NewTestLogger("p2p-server")
		re := regexp.MustCompile(infc)
		handler, err = api.NewHandler(logger, nil, nil, re, 4, 7766)
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("get p2p url", func() {
		var (
			request *http.Request
			recorder *httptest.ResponseRecorder
		)
		JustBeforeEach(func(){
			var err error
			request, err = http.NewRequest("GET", "/p2p-url", nil)
			Expect(err).NotTo(HaveOccurred())

			recorder = httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)
		})

		Context("when a valid interface name is given", func(){
			BeforeEach(func(){
				infc = "lo"
			})

			It("returns a url successfully", func() {
				Expect(recorder.Code).To(Equal(200))
				Expect(recorder.Body.String()).To(Equal("http://127.0.0.1:7766"))
			})
		})

		Context("when an invalid interface name is given", func(){
			BeforeEach(func(){
				infc = "dummy_interface"
			})

			It("returns a url successfully", func() {
				Expect(recorder.Code).To(Equal(500))
			})
		})
	})
})
<commit_msg>Change device regex if testing on windows<commit_after>package api_test

import (
	"net/http"
	"net/http/httptest"
	"regexp"
	"runtime"

	"code.cloudfoundry.org/lager/lagertest"
	"github.com/concourse/baggageclaim/api"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("P2P Server", func() {
	var (
		handler http.Handler
		infc    string
	)

	JustBeforeEach(func() {
		var err error
		logger := lagertest.NewTestLogger("p2p-server")
		re := regexp.MustCompile(infc)
		handler, err = api.NewHandler(logger, nil, nil, re, 4, 7766)
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("get p2p url", func() {
		var (
			request  *http.Request
			recorder *httptest.ResponseRecorder
		)
		JustBeforeEach(func() {
			var err error
			request, err = http.NewRequest("GET", "/p2p-url", nil)
			Expect(err).NotTo(HaveOccurred())

			recorder = httptest.NewRecorder()
			handler.ServeHTTP(recorder, request)
		})

		Context("when a valid interface name is given", func() {
			BeforeEach(func() {
				infc = "lo"
				if runtime.GOOS == "windows" {
					infc = "Loopback"
				}
			})

			It("returns a url successfully", func() {
				Expect(recorder.Code).To(Equal(200))
				Expect(recorder.Body.String()).To(Equal("http://127.0.0.1:7766"))
			})
		})

		Context("when an invalid interface name is given", func() {
			BeforeEach(func() {
				infc = "dummy_interface"
			})

			It("returns a url successfully", func() {
				Expect(recorder.Code).To(Equal(500))
			})
		})
	})
})
<|endoftext|>


<commit_before>package state

import (
	"errors"
	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/agent"
	"github.com/mesos/mesos-go/master"
)

var (
	ErrNotFound       = errors.New("Not found")
	ErrTooManyResults = errors.New("Too many results")
)

// Filter is used to query a State and match
// a single protobuf message.
type Filter func(proto.Message) bool

// State represents the state of a Mesos cluster.
// Mesos protobufs do not immplement an explicit
// message for this because it is a combination
// of different Mesos types.
// NOTE: This could be improved with fancier indexing
// but I am unsure if that is necessary.
type State struct {
	messages []proto.Message
}

// StateFromAgent loads a State with proto.messages
// from a GET_STATE call.
// TODO: Consider using reflect to reduce typing.
func StateFromAgent(resp *agent.Response_GetState) *State {
	state := &State{messages: []proto.Message{}}
	tasks := resp.GetTasks
	for _, task := range tasks.QueuedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.PendingTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.LaunchedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.CompletedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.TerminatedTasks {
		state.messages = append(state.messages, task)
	}
	frameworks := resp.GetFrameworks
	for _, framework := range frameworks.Frameworks {
		state.messages = append(state.messages, framework.FrameworkInfo)
	}
	for _, framework := range frameworks.CompletedFrameworks {
		state.messages = append(state.messages, framework.FrameworkInfo)
	}
	executors := resp.GetExecutors
	for _, executor := range executors.Executors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	for _, executor := range executors.CompletedExecutors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	return state
}

// StateFromMaster loads a State with proto.messages
// from a GET_STATE call.
// TODO: Consider using reflect to reduce typing.
func StateFromMaster(resp *master.Response_GetState) *State {
	state := &State{messages: []proto.Message{}}
	tasks := resp.GetTasks
	for _, task := range tasks.Tasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.OrphanTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.PendingTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.CompletedTasks {
		state.messages = append(state.messages, task)
	}
	frameworks := resp.GetFrameworks
	for _, framework := range frameworks.Frameworks {
		state.messages = append(state.messages, framework)
	}
	executors := resp.GetExecutors
	for _, executor := range executors.Executors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	agents := resp.GetAgents
	for _, agent := range agents.Agents {
		state.messages = append(state.messages, agent.AgentInfo)
	}
	return state
}

// FindAny will return the first message
// where all filters return true. If no
// messages match we will return ErrNotFound.
func (s State) FindAny(filters ...Filter) (proto.Message, error) {
	var match bool
loop:
	for _, message := range s.messages {
		for _, filter := range filters {
			match = filter(message)
			if !match {
				continue loop
			}
		}
		if match {
			return message, nil
		}
	}
	return nil, ErrNotFound
}

// FindOne will a single message where all
// filters return true. If more than one
// message matches return ErrTooManyResults.
// If no messages match all filters we will
// return ErrNotFound.
func (s State) FindOne(filters ...Filter) (proto.Message, error) {
	var (
		position int
		match    bool
	)
loop:
	for i, message := range s.messages {
		for _, filter := range filters {
			if !filter(message) {
				continue loop
			}
		}
		// Already matched a message
		if match {
			return nil, ErrTooManyResults
		}
		// Mark this message as matched
		match = true
		position = i
	}
	if !match {
		return nil, ErrNotFound
	}
	return s.messages[position], nil
}

// FindMany will return as many messages
// in which all of the filters match true.
// If no messages match all the filters
// we will return an empty []proto.Message.
func (s State) FindMany(filters ...Filter) []proto.Message {
	matches := []proto.Message{}
loop:
	for _, message := range s.messages {
		for _, filter := range filters {
			if !filter(message) {
				continue loop
			}
		}
		matches = append(matches, message)
	}
	return matches
}
<commit_msg>add state.Add method<commit_after>package state

import (
	"errors"
	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/agent"
	"github.com/mesos/mesos-go/master"
)

var (
	ErrNotFound       = errors.New("Not found")
	ErrTooManyResults = errors.New("Too many results")
)

// Filter is used to query a State and match
// a single protobuf message.
type Filter func(proto.Message) bool

// State represents the state of a Mesos cluster.
// Mesos protobufs do not immplement an explicit
// message for this because it is a combination
// of different Mesos types.
// NOTE: This could be improved with fancier indexing
// but I am unsure if that is necessary.
type State struct {
	messages []proto.Message
}

// StateFromAgent loads a State with proto.messages
// from a GET_STATE call.
// TODO: Consider using reflect to reduce typing.
func StateFromAgent(resp *agent.Response_GetState) *State {
	state := &State{messages: []proto.Message{}}
	tasks := resp.GetTasks
	for _, task := range tasks.QueuedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.PendingTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.LaunchedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.CompletedTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.TerminatedTasks {
		state.messages = append(state.messages, task)
	}
	frameworks := resp.GetFrameworks
	for _, framework := range frameworks.Frameworks {
		state.messages = append(state.messages, framework.FrameworkInfo)
	}
	for _, framework := range frameworks.CompletedFrameworks {
		state.messages = append(state.messages, framework.FrameworkInfo)
	}
	executors := resp.GetExecutors
	for _, executor := range executors.Executors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	for _, executor := range executors.CompletedExecutors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	return state
}

// StateFromMaster loads a State with proto.messages
// from a GET_STATE call.
// TODO: Consider using reflect to reduce typing.
func StateFromMaster(resp *master.Response_GetState) *State {
	state := &State{messages: []proto.Message{}}
	tasks := resp.GetTasks
	for _, task := range tasks.Tasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.OrphanTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.PendingTasks {
		state.messages = append(state.messages, task)
	}
	for _, task := range tasks.CompletedTasks {
		state.messages = append(state.messages, task)
	}
	frameworks := resp.GetFrameworks
	for _, framework := range frameworks.Frameworks {
		state.messages = append(state.messages, framework)
	}
	executors := resp.GetExecutors
	for _, executor := range executors.Executors {
		state.messages = append(state.messages, executor.ExecutorInfo)
	}
	agents := resp.GetAgents
	for _, agent := range agents.Agents {
		state.messages = append(state.messages, agent.AgentInfo)
	}
	return state
}

func (s *State) Add(msg proto.Message) {
	s.messages = append(s.messages, msg)
}

// FindAny will return the first message
// where all filters return true. If no
// messages match we will return ErrNotFound.
func (s State) FindAny(filters ...Filter) (proto.Message, error) {
	var match bool
loop:
	for _, message := range s.messages {
		for _, filter := range filters {
			match = filter(message)
			if !match {
				continue loop
			}
		}
		if match {
			return message, nil
		}
	}
	return nil, ErrNotFound
}

// FindOne will a single message where all
// filters return true. If more than one
// message matches return ErrTooManyResults.
// If no messages match all filters we will
// return ErrNotFound.
func (s State) FindOne(filters ...Filter) (proto.Message, error) {
	var (
		position int
		match    bool
	)
loop:
	for i, message := range s.messages {
		for _, filter := range filters {
			if !filter(message) {
				continue loop
			}
		}
		// Already matched a message
		if match {
			return nil, ErrTooManyResults
		}
		// Mark this message as matched
		match = true
		position = i
	}
	if !match {
		return nil, ErrNotFound
	}
	return s.messages[position], nil
}

// FindMany will return as many messages
// in which all of the filters match true.
// If no messages match all the filters
// we will return an empty []proto.Message.
func (s State) FindMany(filters ...Filter) []proto.Message {
	matches := []proto.Message{}
loop:
	for _, message := range s.messages {
		for _, filter := range filters {
			if !filter(message) {
				continue loop
			}
		}
		matches = append(matches, message)
	}
	return matches
}
<|endoftext|>


<commit_before>package norm

import (
	"testing"
	"utf8"
)

// Test data is located in triedata_test.go, generated by maketesttables.
var testdata = testdataTrie

// Test cases for illegal runes.
type trietest struct {
	size  int
	bytes []byte
}

var tests = []trietest{
	// illegal runes
	{1, []byte{0x80}},
	{1, []byte{0xFF}},
	{1, []byte{t2, tx - 1}},
	{1, []byte{t2, t2}},
	{2, []byte{t3, tx, tx - 1}},
	{2, []byte{t3, tx, t2}},
	{1, []byte{t3, tx - 1, tx}},
	{3, []byte{t4, tx, tx, tx - 1}},
	{3, []byte{t4, tx, tx, t2}},
	{1, []byte{t4, t2, tx, tx - 1}},
	{2, []byte{t4, tx, t2, tx - 1}},

	// short runes
	{0, []byte{t2}},
	{0, []byte{t3, tx}},
	{0, []byte{t4, tx, tx}},
	{0, []byte{t5, tx, tx, tx}},
	{0, []byte{t6, tx, tx, tx, tx}},
}

func mkUtf8(rune int) ([]byte, int) {
	var b [utf8.UTFMax]byte
	sz := utf8.EncodeRune(b[:], rune)
	return b[:sz], sz
}

func TestLookup(t *testing.T) {
	for i, tt := range testRunes {
		b, szg := mkUtf8(tt)
		v, szt := testdata.lookup(b)
		if int(v) != i {
			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
		}
		if szt != szg {
			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
		}
	}
	for i, tt := range tests {
		v, sz := testdata.lookup(tt.bytes)
		if int(v) != 0 {
			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
		}
		if sz != tt.size {
			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
		}
	}
}

func TestLookupUnsafe(t *testing.T) {
	for i, tt := range testRunes {
		b, _ := mkUtf8(tt)
		v := testdata.lookupUnsafe(b)
		if int(v) != i {
			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
		}
	}
}

func TestLookupString(t *testing.T) {
	for i, tt := range testRunes {
		b, szg := mkUtf8(tt)
		v, szt := testdata.lookupString(string(b))
		if int(v) != i {
			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
		}
		if szt != szg {
			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
		}
	}
	for i, tt := range tests {
		v, sz := testdata.lookupString(string(tt.bytes))
		if int(v) != 0 {
			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
		}
		if sz != tt.size {
			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
		}
	}
}

func TestLookupStringUnsafe(t *testing.T) {
	for i, tt := range testRunes {
		b, _ := mkUtf8(tt)
		v := testdata.lookupStringUnsafe(string(b))
		if int(v) != i {
			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
		}
	}
}
<commit_msg>exp/norm: generate trie struct in triegen.go for better encapsulation.<commit_after>package norm

import (
	"testing"
	"utf8"
)

// Test data is located in triedata_test.go; generated by maketesttables.
var testdata = testdataTrie

// Test cases for illegal runes.
type trietest struct {
	size  int
	bytes []byte
}

var tests = []trietest{
	// illegal runes
	{1, []byte{0x80}},
	{1, []byte{0xFF}},
	{1, []byte{t2, tx - 1}},
	{1, []byte{t2, t2}},
	{2, []byte{t3, tx, tx - 1}},
	{2, []byte{t3, tx, t2}},
	{1, []byte{t3, tx - 1, tx}},
	{3, []byte{t4, tx, tx, tx - 1}},
	{3, []byte{t4, tx, tx, t2}},
	{1, []byte{t4, t2, tx, tx - 1}},
	{2, []byte{t4, tx, t2, tx - 1}},

	// short runes
	{0, []byte{t2}},
	{0, []byte{t3, tx}},
	{0, []byte{t4, tx, tx}},
	{0, []byte{t5, tx, tx, tx}},
	{0, []byte{t6, tx, tx, tx, tx}},
}

func mkUtf8(rune int) ([]byte, int) {
	var b [utf8.UTFMax]byte
	sz := utf8.EncodeRune(b[:], rune)
	return b[:sz], sz
}

func TestLookup(t *testing.T) {
	for i, tt := range testRunes {
		b, szg := mkUtf8(tt)
		v, szt := testdata.lookup(b)
		if int(v) != i {
			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
		}
		if szt != szg {
			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
		}
	}
	for i, tt := range tests {
		v, sz := testdata.lookup(tt.bytes)
		if int(v) != 0 {
			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
		}
		if sz != tt.size {
			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
		}
	}
}

func TestLookupUnsafe(t *testing.T) {
	for i, tt := range testRunes {
		b, _ := mkUtf8(tt)
		v := testdata.lookupUnsafe(b)
		if int(v) != i {
			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
		}
	}
}

func TestLookupString(t *testing.T) {
	for i, tt := range testRunes {
		b, szg := mkUtf8(tt)
		v, szt := testdata.lookupString(string(b))
		if int(v) != i {
			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
		}
		if szt != szg {
			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
		}
	}
	for i, tt := range tests {
		v, sz := testdata.lookupString(string(tt.bytes))
		if int(v) != 0 {
			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
		}
		if sz != tt.size {
			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
		}
	}
}

func TestLookupStringUnsafe(t *testing.T) {
	for i, tt := range testRunes {
		b, _ := mkUtf8(tt)
		v := testdata.lookupStringUnsafe(string(b))
		if int(v) != i {
			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stats

import (
	"math"
	"sort"
)

// A LocationHypothesis specifies the alternative hypothesis of a
// location test such as a t-test or a Mann-Whitney U-test. The
// default (zero) value is to test against the alternative hypothesis
// that they differ.
type LocationHypothesis int

//go:generate stringer -type LocationHypothesis

const (
	// LocationLess specifies the alternative hypothesis that the
	// location of the first sample is less than the second. This
	// is a one-tailed test.
	LocationLess LocationHypothesis = -1

	// LocationDiffers specifies the alternative hypothesis that
	// the locations of the two samples are not equal. This is a
	// two-tailed test.
	LocationDiffers LocationHypothesis = 0

	// LocationGreater specifies the alternative hypothesis that
	// the location of the first sample is greater than the
	// second. This is a one-tailed test.
	LocationGreater LocationHypothesis = 1
)

// A MannWhitneyUTestResult is the result of a Mann-Whitney U-test.
type MannWhitneyUTestResult struct {
	// N1 and N2 are the sizes of the input samples.
	N1, N2 int

	// U is the value of the Mann-Whitney U statistic for this
	// test, generalized by counting ties as 0.5.
	//
	// Given the Cartesian product of the two samples, this is the
	// number of pairs in which the value from the first sample is
	// greater than the value of the second, plus 0.5 times the
	// number of pairs where the values from the two samples are
	// equal. Hence, U is always an integer multiple of 0.5 (it is
	// a whole integer if there are no ties) in the range [0, N1*N2].
	//
	// U statistics always come in pairs, depending on which
	// sample is "first". The mirror U for the other sample can be
	// calculated as N1*N2 - U.
	//
	// There are many equivalent statistics with slightly
	// different definitions. The Wilcoxon (1945) W statistic
	// (generalized for ties) is U + (N1(N1+1))/2. It is also
	// common to use 2U to eliminate the half steps and Smid
	// (1956) uses N1*N2 - 2U to additionally center the
	// distribution.
	U float64

	// AltHypothesis specifies the alternative hypothesis tested
	// by this test against the null hypothesis that there is no
	// difference in the locations of the samples.
	AltHypothesis LocationHypothesis

	// P is the p-value of the Mann-Whitney test for the given
	// null hypothesis.
	P float64
}

// MannWhitneyExactLimit gives the largest sample size for which the
// exact U distribution will be used for the Mann-Whitney U-test.
//
// Using the exact distribution is necessary for small sample sizes
// because the distribution is highly irregular. However, computing
// the distribution for large sample sizes is both computationally
// expensive and unnecessary because it quickly approaches a normal
// approximation. Computing the distribution for two 50 value samples
// takes a few milliseconds on a 2014 laptop.
var MannWhitneyExactLimit = 50

// MannWhitneyTiesExactLimit gives the largest sample size for which
// the exact U distribution will be used for the Mann-Whitney U-test
// in the presence of ties.
//
// Computing this distribution is more expensive than computing the
// distribution without ties, so this is set lower. Computing this
// distribution for two 20 value samples takes about ten milliseconds
// on a 2014 laptop.
var MannWhitneyTiesExactLimit = 20

// MannWhitneyUTest performs a Mann-Whitney U-test [1,2] of the null
// hypothesis that two samples come from the same population against
// the alternative hypothesis that one sample tends to have larger or
// smaller values than the other.
//
// This is similar to a t-test, but unlike the t-test, the
// Mann-Whitney U-test is non-parametric (it does not assume a normal
// distribution). It has very slightly lower efficiency than the
// t-test on normal distributions.
//
// Computing the exact U distribution is expensive for large sample
// sizes, so this uses a normal approximation for sample sizes larger
// than MannWhitneyExactLimit if there are no ties or
// MannWhitneyTiesExactLimit if there are ties. This normal
// approximation uses both the tie correction and the continuity
// correction.
//
// This can fail with ErrSampleSize if either sample is empty or
// ErrSamplesEqual if all sample values are equal.
//
// This is also known as a Mann-Whitney-Wilcoxon test and is
// equivalent to the Wilcoxon rank-sum test, though the Wilcoxon
// rank-sum test differs in nomenclature.
//
// [1] Mann, Henry B.; Whitney, Donald R. (1947). "On a Test of
// Whether one of Two Random Variables is Stochastically Larger than
// the Other". Annals of Mathematical Statistics 18 (1): 50–60.
//
// [2] Klotz, J. H. (1966). "The Wilcoxon, Ties, and the Computer".
// Journal of the American Statistical Association 61 (315): 772-787.
func MannWhitneyUTest(x1, x2 []float64, alt LocationHypothesis) (*MannWhitneyUTestResult, error) {
	n1, n2 := len(x1), len(x2)
	if n1 == 0 || n2 == 0 {
		return nil, ErrSampleSize
	}

	// Compute the U statistic and tie vector T.
	x1 = append([]float64(nil), x1...)
	x2 = append([]float64(nil), x2...)
	sort.Float64s(x1)
	sort.Float64s(x2)
	merged, labels := labeledMerge(x1, x2)

	R1 := 0.0
	T, hasTies := []int{}, false
	for i := 0; i < len(merged); {
		rank1, nx1, v1 := i+1, 0, merged[i]
		// Consume samples that tie this sample (including itself).
		for ; i < len(merged) && merged[i] == v1; i++ {
			if labels[i] == 1 {
				nx1++
			}
		}
		// Assign all tied samples the average rank of the
		// samples, where merged[0] has rank 1.
		if nx1 != 0 {
			rank := float64(i+rank1) / 2
			R1 += rank * float64(nx1)
		}
		T = append(T, i-rank1+1)
		if i > rank1 {
			hasTies = true
		}
	}
	U1 := R1 - float64(n1*(n1+1))/2

	// Compute the smaller of U1 and U2
	U2 := float64(n1*n2) - U1
	Usmall := math.Min(U1, U2)

	var p float64
	if !hasTies && n1 <= MannWhitneyExactLimit && n2 <= MannWhitneyExactLimit ||
		hasTies && n1 <= MannWhitneyTiesExactLimit && n2 <= MannWhitneyTiesExactLimit {
		// Use exact U distribution. U1 will be an integer.
		if len(T) == 1 {
			// All values are equal. Test is meaningless.
			return nil, ErrSamplesEqual
		}

		dist := UDist{N1: n1, N2: n2, T: T}
		switch alt {
		case LocationDiffers:
			if U1 == U2 {
				// The distribution is symmetric about
				// Usmall. Since the distribution is
				// discrete, the CDF is discontinuous
				// and if simply double CDF(Usmall),
				// we'll double count the
				// (non-infinitesimal) probability
				// mass at Usmall. What we want is
				// just the integral of the whole CDF,
				// which is 1.
				p = 1
			} else {
				p = dist.CDF(Usmall) * 2
			}

		case LocationLess:
			p = dist.CDF(U1)

		case LocationGreater:
			p = 1 - dist.CDF(U1-1)
		}
	} else {
		// Use normal approximation (with tie and continuity
		// correction).
		t := tieCorrection(T)
		N := float64(n1 + n2)
		μ_U := float64(n1*n2) / 2
		σ_U := math.Sqrt(float64(n1*n2) * ((N + 1) - t/(N*(N-1))) / 12)
		if σ_U == 0 {
			return nil, ErrSamplesEqual
		}
		numer := U1 - μ_U
		// Perform continuity correction.
		switch alt {
		case LocationDiffers:
			numer -= sign(numer) * 0.5
		case LocationLess:
			numer += 0.5
		case LocationGreater:
			numer -= 0.5
		}
		z := numer / σ_U
		switch alt {
		case LocationDiffers:
			p = 2 * math.Min(StdNormal.CDF(z), 1-StdNormal.CDF(z))
		case LocationLess:
			p = StdNormal.CDF(z)
		case LocationGreater:
			p = 1 - StdNormal.CDF(z)
		}
	}

	return &MannWhitneyUTestResult{N1: n1, N2: n2, U: U1,
		AltHypothesis: alt, P: p}, nil
}

// labeledMerge merges sorted lists x1 and x2 into sorted list merged.
// labels[i] is 1 or 2 depending on whether merged[i] is a value from
// x1 or x2, respectively.
func labeledMerge(x1, x2 []float64) (merged []float64, labels []byte) {
	merged = make([]float64, len(x1)+len(x2))
	labels = make([]byte, len(x1)+len(x2))

	i, j, o := 0, 0, 0
	for i < len(x1) && j < len(x2) {
		if x1[i] < x2[j] {
			merged[o] = x1[i]
			labels[o] = 1
			i++
		} else {
			merged[o] = x2[j]
			labels[o] = 2
			j++
		}
		o++
	}
	for ; i < len(x1); i++ {
		merged[o] = x1[i]
		labels[o] = 1
		o++
	}
	for ; j < len(x2); j++ {
		merged[o] = x2[j]
		labels[o] = 2
		o++
	}
	return
}

// tieCorrection computes the tie correction factor Σ_j (t_j³ - t_j)
// where t_j is the number of ties in the j'th rank.
func tieCorrection(ties []int) float64 {
	t := 0
	for _, tie := range ties {
		t += tie*tie*tie - tie
	}
	return float64(t)
}
<commit_msg>stats: raise U test with ties limit to 25<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stats

import (
	"math"
	"sort"
)

// A LocationHypothesis specifies the alternative hypothesis of a
// location test such as a t-test or a Mann-Whitney U-test. The
// default (zero) value is to test against the alternative hypothesis
// that they differ.
type LocationHypothesis int

//go:generate stringer -type LocationHypothesis

const (
	// LocationLess specifies the alternative hypothesis that the
	// location of the first sample is less than the second. This
	// is a one-tailed test.
	LocationLess LocationHypothesis = -1

	// LocationDiffers specifies the alternative hypothesis that
	// the locations of the two samples are not equal. This is a
	// two-tailed test.
	LocationDiffers LocationHypothesis = 0

	// LocationGreater specifies the alternative hypothesis that
	// the location of the first sample is greater than the
	// second. This is a one-tailed test.
	LocationGreater LocationHypothesis = 1
)

// A MannWhitneyUTestResult is the result of a Mann-Whitney U-test.
type MannWhitneyUTestResult struct {
	// N1 and N2 are the sizes of the input samples.
	N1, N2 int

	// U is the value of the Mann-Whitney U statistic for this
	// test, generalized by counting ties as 0.5.
	//
	// Given the Cartesian product of the two samples, this is the
	// number of pairs in which the value from the first sample is
	// greater than the value of the second, plus 0.5 times the
	// number of pairs where the values from the two samples are
	// equal. Hence, U is always an integer multiple of 0.5 (it is
	// a whole integer if there are no ties) in the range [0, N1*N2].
	//
	// U statistics always come in pairs, depending on which
	// sample is "first". The mirror U for the other sample can be
	// calculated as N1*N2 - U.
	//
	// There are many equivalent statistics with slightly
	// different definitions. The Wilcoxon (1945) W statistic
	// (generalized for ties) is U + (N1(N1+1))/2. It is also
	// common to use 2U to eliminate the half steps and Smid
	// (1956) uses N1*N2 - 2U to additionally center the
	// distribution.
	U float64

	// AltHypothesis specifies the alternative hypothesis tested
	// by this test against the null hypothesis that there is no
	// difference in the locations of the samples.
	AltHypothesis LocationHypothesis

	// P is the p-value of the Mann-Whitney test for the given
	// null hypothesis.
	P float64
}

// MannWhitneyExactLimit gives the largest sample size for which the
// exact U distribution will be used for the Mann-Whitney U-test.
//
// Using the exact distribution is necessary for small sample sizes
// because the distribution is highly irregular. However, computing
// the distribution for large sample sizes is both computationally
// expensive and unnecessary because it quickly approaches a normal
// approximation. Computing the distribution for two 50 value samples
// takes a few milliseconds on a 2014 laptop.
var MannWhitneyExactLimit = 50

// MannWhitneyTiesExactLimit gives the largest sample size for which
// the exact U distribution will be used for the Mann-Whitney U-test
// in the presence of ties.
//
// Computing this distribution is more expensive than computing the
// distribution without ties, so this is set lower. Computing this
// distribution for two 25 value samples takes about ten milliseconds
// on a 2014 laptop.
var MannWhitneyTiesExactLimit = 25

// MannWhitneyUTest performs a Mann-Whitney U-test [1,2] of the null
// hypothesis that two samples come from the same population against
// the alternative hypothesis that one sample tends to have larger or
// smaller values than the other.
//
// This is similar to a t-test, but unlike the t-test, the
// Mann-Whitney U-test is non-parametric (it does not assume a normal
// distribution). It has very slightly lower efficiency than the
// t-test on normal distributions.
//
// Computing the exact U distribution is expensive for large sample
// sizes, so this uses a normal approximation for sample sizes larger
// than MannWhitneyExactLimit if there are no ties or
// MannWhitneyTiesExactLimit if there are ties. This normal
// approximation uses both the tie correction and the continuity
// correction.
//
// This can fail with ErrSampleSize if either sample is empty or
// ErrSamplesEqual if all sample values are equal.
//
// This is also known as a Mann-Whitney-Wilcoxon test and is
// equivalent to the Wilcoxon rank-sum test, though the Wilcoxon
// rank-sum test differs in nomenclature.
//
// [1] Mann, Henry B.; Whitney, Donald R. (1947). "On a Test of
// Whether one of Two Random Variables is Stochastically Larger than
// the Other". Annals of Mathematical Statistics 18 (1): 50–60.
//
// [2] Klotz, J. H. (1966). "The Wilcoxon, Ties, and the Computer".
// Journal of the American Statistical Association 61 (315): 772-787.
func MannWhitneyUTest(x1, x2 []float64, alt LocationHypothesis) (*MannWhitneyUTestResult, error) {
	n1, n2 := len(x1), len(x2)
	if n1 == 0 || n2 == 0 {
		return nil, ErrSampleSize
	}

	// Compute the U statistic and tie vector T.
	x1 = append([]float64(nil), x1...)
	x2 = append([]float64(nil), x2...)
	sort.Float64s(x1)
	sort.Float64s(x2)
	merged, labels := labeledMerge(x1, x2)

	R1 := 0.0
	T, hasTies := []int{}, false
	for i := 0; i < len(merged); {
		rank1, nx1, v1 := i+1, 0, merged[i]
		// Consume samples that tie this sample (including itself).
		for ; i < len(merged) && merged[i] == v1; i++ {
			if labels[i] == 1 {
				nx1++
			}
		}
		// Assign all tied samples the average rank of the
		// samples, where merged[0] has rank 1.
		if nx1 != 0 {
			rank := float64(i+rank1) / 2
			R1 += rank * float64(nx1)
		}
		T = append(T, i-rank1+1)
		if i > rank1 {
			hasTies = true
		}
	}
	U1 := R1 - float64(n1*(n1+1))/2

	// Compute the smaller of U1 and U2
	U2 := float64(n1*n2) - U1
	Usmall := math.Min(U1, U2)

	var p float64
	if !hasTies && n1 <= MannWhitneyExactLimit && n2 <= MannWhitneyExactLimit ||
		hasTies && n1 <= MannWhitneyTiesExactLimit && n2 <= MannWhitneyTiesExactLimit {
		// Use exact U distribution. U1 will be an integer.
		if len(T) == 1 {
			// All values are equal. Test is meaningless.
			return nil, ErrSamplesEqual
		}

		dist := UDist{N1: n1, N2: n2, T: T}
		switch alt {
		case LocationDiffers:
			if U1 == U2 {
				// The distribution is symmetric about
				// Usmall. Since the distribution is
				// discrete, the CDF is discontinuous
				// and if simply double CDF(Usmall),
				// we'll double count the
				// (non-infinitesimal) probability
				// mass at Usmall. What we want is
				// just the integral of the whole CDF,
				// which is 1.
				p = 1
			} else {
				p = dist.CDF(Usmall) * 2
			}

		case LocationLess:
			p = dist.CDF(U1)

		case LocationGreater:
			p = 1 - dist.CDF(U1-1)
		}
	} else {
		// Use normal approximation (with tie and continuity
		// correction).
		t := tieCorrection(T)
		N := float64(n1 + n2)
		μ_U := float64(n1*n2) / 2
		σ_U := math.Sqrt(float64(n1*n2) * ((N + 1) - t/(N*(N-1))) / 12)
		if σ_U == 0 {
			return nil, ErrSamplesEqual
		}
		numer := U1 - μ_U
		// Perform continuity correction.
		switch alt {
		case LocationDiffers:
			numer -= sign(numer) * 0.5
		case LocationLess:
			numer += 0.5
		case LocationGreater:
			numer -= 0.5
		}
		z := numer / σ_U
		switch alt {
		case LocationDiffers:
			p = 2 * math.Min(StdNormal.CDF(z), 1-StdNormal.CDF(z))
		case LocationLess:
			p = StdNormal.CDF(z)
		case LocationGreater:
			p = 1 - StdNormal.CDF(z)
		}
	}

	return &MannWhitneyUTestResult{N1: n1, N2: n2, U: U1,
		AltHypothesis: alt, P: p}, nil
}

// labeledMerge merges sorted lists x1 and x2 into sorted list merged.
// labels[i] is 1 or 2 depending on whether merged[i] is a value from
// x1 or x2, respectively.
func labeledMerge(x1, x2 []float64) (merged []float64, labels []byte) {
	merged = make([]float64, len(x1)+len(x2))
	labels = make([]byte, len(x1)+len(x2))

	i, j, o := 0, 0, 0
	for i < len(x1) && j < len(x2) {
		if x1[i] < x2[j] {
			merged[o] = x1[i]
			labels[o] = 1
			i++
		} else {
			merged[o] = x2[j]
			labels[o] = 2
			j++
		}
		o++
	}
	for ; i < len(x1); i++ {
		merged[o] = x1[i]
		labels[o] = 1
		o++
	}
	for ; j < len(x2); j++ {
		merged[o] = x2[j]
		labels[o] = 2
		o++
	}
	return
}

// tieCorrection computes the tie correction factor Σ_j (t_j³ - t_j)
// where t_j is the number of ties in the j'th rank.
func tieCorrection(ties []int) float64 {
	t := 0
	for _, tie := range ties {
		t += tie*tie*tie - tie
	}
	return float64(t)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"os"
	"sync"
	"syscall"
	"unsafe"
)

var (
	protoentLock sync.Mutex
	hostentLock  sync.Mutex
	serventLock  sync.Mutex
)

var (
	lookupPort = oldLookupPort
	lookupIP   = oldLookupIP
)

// lookupProtocol looks up IP protocol name and returns correspondent protocol number.
func lookupProtocol(name string) (proto int, err error) {
	protoentLock.Lock()
	defer protoentLock.Unlock()
	p, err := syscall.GetProtoByName(name)
	if err != nil {
		return 0, os.NewSyscallError("GetProtoByName", err)
	}
	return int(p.Proto), nil
}

func lookupHost(name string) (addrs []string, err error) {
	ips, err := LookupIP(name)
	if err != nil {
		return
	}
	addrs = make([]string, 0, len(ips))
	for _, ip := range ips {
		addrs = append(addrs, ip.String())
	}
	return
}

func oldLookupIP(name string) (addrs []IP, err error) {
	hostentLock.Lock()
	defer hostentLock.Unlock()
	h, err := syscall.GetHostByName(name)
	if err != nil {
		return nil, os.NewSyscallError("GetHostByName", err)
	}
	switch h.AddrType {
	case syscall.AF_INET:
		i := 0
		addrs = make([]IP, 100) // plenty of room to grow
		for p := (*[100](*[4]byte))(unsafe.Pointer(h.AddrList)); i < cap(addrs) && p[i] != nil; i++ {
			addrs[i] = IPv4(p[i][0], p[i][1], p[i][2], p[i][3])
		}
		addrs = addrs[0:i]
	default: // TODO(vcc): Implement non IPv4 address lookups.
		return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS)
	}
	return addrs, nil
}

func newLookupIP(name string) (addrs []IP, err error) {
	hints := syscall.AddrinfoW{
		Family:   syscall.AF_UNSPEC,
		Socktype: syscall.SOCK_STREAM,
		Protocol: syscall.IPPROTO_IP,
	}
	var result *syscall.AddrinfoW
	e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result)
	if e != nil {
		return nil, os.NewSyscallError("GetAddrInfoW", e)
	}
	defer syscall.FreeAddrInfoW(result)
	addrs = make([]IP, 0, 5)
	for ; result != nil; result = result.Next {
		addr := unsafe.Pointer(result.Addr)
		switch result.Family {
		case syscall.AF_INET:
			a := (*syscall.RawSockaddrInet4)(addr).Addr
			addrs = append(addrs, IPv4(a[0], a[1], a[2], a[3]))
		case syscall.AF_INET6:
			a := (*syscall.RawSockaddrInet6)(addr).Addr
			addrs = append(addrs, IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]})
		default:
			return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS)
		}
	}
	return addrs, nil
}

func oldLookupPort(network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	serventLock.Lock()
	defer serventLock.Unlock()
	s, err := syscall.GetServByName(service, network)
	if err != nil {
		return 0, os.NewSyscallError("GetServByName", err)
	}
	return int(syscall.Ntohs(s.Port)), nil
}

func newLookupPort(network, service string) (port int, err error) {
	var stype int32
	switch network {
	case "tcp4", "tcp6":
		stype = syscall.SOCK_STREAM
	case "udp4", "udp6":
		stype = syscall.SOCK_DGRAM
	}
	hints := syscall.AddrinfoW{
		Family:   syscall.AF_UNSPEC,
		Socktype: stype,
		Protocol: syscall.IPPROTO_IP,
	}
	var result *syscall.AddrinfoW
	e := syscall.GetAddrInfoW(nil, syscall.StringToUTF16Ptr(service), &hints, &result)
	if e != nil {
		return 0, os.NewSyscallError("GetAddrInfoW", e)
	}
	defer syscall.FreeAddrInfoW(result)
	if result == nil {
		return 0, os.NewSyscallError("LookupPort", syscall.EINVAL)
	}
	addr := unsafe.Pointer(result.Addr)
	switch result.Family {
	case syscall.AF_INET:
		a := (*syscall.RawSockaddrInet4)(addr)
		return int(syscall.Ntohs(a.Port)), nil
	case syscall.AF_INET6:
		a := (*syscall.RawSockaddrInet6)(addr)
		return int(syscall.Ntohs(a.Port)), nil
	}
	return 0, os.NewSyscallError("LookupPort", syscall.EINVAL)
}

func lookupCNAME(name string) (cname string, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_CNAME, 0, nil, &r, nil)
	if e != nil {
		return "", os.NewSyscallError("LookupCNAME", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	if r != nil && r.Type == syscall.DNS_TYPE_CNAME {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&r.Data[0]))
		cname = syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]) + "."
	}
	return
}

func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(target, syscall.DNS_TYPE_SRV, 0, nil, &r, nil)
	if e != nil {
		return "", nil, os.NewSyscallError("LookupSRV", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	addrs = make([]*SRV, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_SRV; p = p.Next {
		v := (*syscall.DNSSRVData)(unsafe.Pointer(&p.Data[0]))
		addrs = append(addrs, &SRV{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Target))[:]), v.Port, v.Priority, v.Weight})
	}
	byPriorityWeight(addrs).sort()
	return name, addrs, nil
}

func lookupMX(name string) (mx []*MX, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_MX, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupMX", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	mx = make([]*MX, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_MX; p = p.Next {
		v := (*syscall.DNSMXData)(unsafe.Pointer(&p.Data[0]))
		mx = append(mx, &MX{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.NameExchange))[:]) + ".", v.Preference})
	}
	byPref(mx).sort()
	return mx, nil
}

func lookupNS(name string) (ns []*NS, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_NS, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupNS", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	ns = make([]*NS, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_NS; p = p.Next {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0]))
		ns = append(ns, &NS{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]) + "."})
	}
	return ns, nil
}

func lookupTXT(name string) (txt []string, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_TEXT, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupTXT", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	txt = make([]string, 0, 10)
	if r != nil && r.Type == syscall.DNS_TYPE_TEXT {
		d := (*syscall.DNSTXTData)(unsafe.Pointer(&r.Data[0]))
		for _, v := range (*[1 << 10]*uint16)(unsafe.Pointer(&(d.StringArray[0])))[:d.StringCount] {
			s := syscall.UTF16ToString((*[1 << 20]uint16)(unsafe.Pointer(v))[:])
			txt = append(txt, s)
		}
	}
	return
}

func lookupAddr(addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return nil, err
	}
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(arpa, syscall.DNS_TYPE_PTR, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupAddr", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	name = make([]string, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_PTR; p = p.Next {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0]))
		name = append(name, syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]))
	}
	return name, nil
}
<commit_msg>net: prevent races during windows lookup calls<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"os"
	"runtime"
	"syscall"
	"unsafe"
)

var (
	lookupPort = oldLookupPort
	lookupIP   = oldLookupIP
)

func getprotobyname(name string) (proto int, err error) {
	p, err := syscall.GetProtoByName(name)
	if err != nil {
		return 0, os.NewSyscallError("GetProtoByName", err)
	}
	return int(p.Proto), nil
}

// lookupProtocol looks up IP protocol name and returns correspondent protocol number.
func lookupProtocol(name string) (proto int, err error) {
	// GetProtoByName return value is stored in thread local storage.
	// Start new os thread before the call to prevent races.
	type result struct {
		proto int
		err   error
	}
	ch := make(chan result)
	go func() {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		proto, err := getprotobyname(name)
		ch <- result{proto: proto, err: err}
	}()
	r := <-ch
	return r.proto, r.err
}

func lookupHost(name string) (addrs []string, err error) {
	ips, err := LookupIP(name)
	if err != nil {
		return
	}
	addrs = make([]string, 0, len(ips))
	for _, ip := range ips {
		addrs = append(addrs, ip.String())
	}
	return
}

func gethostbyname(name string) (addrs []IP, err error) {
	h, err := syscall.GetHostByName(name)
	if err != nil {
		return nil, os.NewSyscallError("GetHostByName", err)
	}
	switch h.AddrType {
	case syscall.AF_INET:
		i := 0
		addrs = make([]IP, 100) // plenty of room to grow
		for p := (*[100](*[4]byte))(unsafe.Pointer(h.AddrList)); i < cap(addrs) && p[i] != nil; i++ {
			addrs[i] = IPv4(p[i][0], p[i][1], p[i][2], p[i][3])
		}
		addrs = addrs[0:i]
	default: // TODO(vcc): Implement non IPv4 address lookups.
		return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS)
	}
	return addrs, nil
}

func oldLookupIP(name string) (addrs []IP, err error) {
	// GetHostByName return value is stored in thread local storage.
	// Start new os thread before the call to prevent races.
	type result struct {
		addrs []IP
		err   error
	}
	ch := make(chan result)
	go func() {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		addrs, err := gethostbyname(name)
		ch <- result{addrs: addrs, err: err}
	}()
	r := <-ch
	return r.addrs, r.err
}

func newLookupIP(name string) (addrs []IP, err error) {
	hints := syscall.AddrinfoW{
		Family:   syscall.AF_UNSPEC,
		Socktype: syscall.SOCK_STREAM,
		Protocol: syscall.IPPROTO_IP,
	}
	var result *syscall.AddrinfoW
	e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result)
	if e != nil {
		return nil, os.NewSyscallError("GetAddrInfoW", e)
	}
	defer syscall.FreeAddrInfoW(result)
	addrs = make([]IP, 0, 5)
	for ; result != nil; result = result.Next {
		addr := unsafe.Pointer(result.Addr)
		switch result.Family {
		case syscall.AF_INET:
			a := (*syscall.RawSockaddrInet4)(addr).Addr
			addrs = append(addrs, IPv4(a[0], a[1], a[2], a[3]))
		case syscall.AF_INET6:
			a := (*syscall.RawSockaddrInet6)(addr).Addr
			addrs = append(addrs, IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]})
		default:
			return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS)
		}
	}
	return addrs, nil
}

func getservbyname(network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	s, err := syscall.GetServByName(service, network)
	if err != nil {
		return 0, os.NewSyscallError("GetServByName", err)
	}
	return int(syscall.Ntohs(s.Port)), nil
}

func oldLookupPort(network, service string) (port int, err error) {
	// GetServByName return value is stored in thread local storage.
	// Start new os thread before the call to prevent races.
	type result struct {
		port int
		err  error
	}
	ch := make(chan result)
	go func() {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
		port, err := getservbyname(network, service)
		ch <- result{port: port, err: err}
	}()
	r := <-ch
	return r.port, r.err
}

func newLookupPort(network, service string) (port int, err error) {
	var stype int32
	switch network {
	case "tcp4", "tcp6":
		stype = syscall.SOCK_STREAM
	case "udp4", "udp6":
		stype = syscall.SOCK_DGRAM
	}
	hints := syscall.AddrinfoW{
		Family:   syscall.AF_UNSPEC,
		Socktype: stype,
		Protocol: syscall.IPPROTO_IP,
	}
	var result *syscall.AddrinfoW
	e := syscall.GetAddrInfoW(nil, syscall.StringToUTF16Ptr(service), &hints, &result)
	if e != nil {
		return 0, os.NewSyscallError("GetAddrInfoW", e)
	}
	defer syscall.FreeAddrInfoW(result)
	if result == nil {
		return 0, os.NewSyscallError("LookupPort", syscall.EINVAL)
	}
	addr := unsafe.Pointer(result.Addr)
	switch result.Family {
	case syscall.AF_INET:
		a := (*syscall.RawSockaddrInet4)(addr)
		return int(syscall.Ntohs(a.Port)), nil
	case syscall.AF_INET6:
		a := (*syscall.RawSockaddrInet6)(addr)
		return int(syscall.Ntohs(a.Port)), nil
	}
	return 0, os.NewSyscallError("LookupPort", syscall.EINVAL)
}

func lookupCNAME(name string) (cname string, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_CNAME, 0, nil, &r, nil)
	if e != nil {
		return "", os.NewSyscallError("LookupCNAME", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	if r != nil && r.Type == syscall.DNS_TYPE_CNAME {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&r.Data[0]))
		cname = syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]) + "."
	}
	return
}

func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(target, syscall.DNS_TYPE_SRV, 0, nil, &r, nil)
	if e != nil {
		return "", nil, os.NewSyscallError("LookupSRV", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	addrs = make([]*SRV, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_SRV; p = p.Next {
		v := (*syscall.DNSSRVData)(unsafe.Pointer(&p.Data[0]))
		addrs = append(addrs, &SRV{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Target))[:]), v.Port, v.Priority, v.Weight})
	}
	byPriorityWeight(addrs).sort()
	return name, addrs, nil
}

func lookupMX(name string) (mx []*MX, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_MX, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupMX", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	mx = make([]*MX, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_MX; p = p.Next {
		v := (*syscall.DNSMXData)(unsafe.Pointer(&p.Data[0]))
		mx = append(mx, &MX{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.NameExchange))[:]) + ".", v.Preference})
	}
	byPref(mx).sort()
	return mx, nil
}

func lookupNS(name string) (ns []*NS, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_NS, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupNS", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	ns = make([]*NS, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_NS; p = p.Next {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0]))
		ns = append(ns, &NS{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]) + "."})
	}
	return ns, nil
}

func lookupTXT(name string) (txt []string, err error) {
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(name, syscall.DNS_TYPE_TEXT, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupTXT", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	txt = make([]string, 0, 10)
	if r != nil && r.Type == syscall.DNS_TYPE_TEXT {
		d := (*syscall.DNSTXTData)(unsafe.Pointer(&r.Data[0]))
		for _, v := range (*[1 << 10]*uint16)(unsafe.Pointer(&(d.StringArray[0])))[:d.StringCount] {
			s := syscall.UTF16ToString((*[1 << 20]uint16)(unsafe.Pointer(v))[:])
			txt = append(txt, s)
		}
	}
	return
}

func lookupAddr(addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return nil, err
	}
	var r *syscall.DNSRecord
	e := syscall.DnsQuery(arpa, syscall.DNS_TYPE_PTR, 0, nil, &r, nil)
	if e != nil {
		return nil, os.NewSyscallError("LookupAddr", e)
	}
	defer syscall.DnsRecordListFree(r, 1)
	name = make([]string, 0, 10)
	for p := r; p != nil && p.Type == syscall.DNS_TYPE_PTR; p = p.Next {
		v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0]))
		name = append(name, syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]))
	}
	return name, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9

package net

import (
	"errors"
	"os"
	"runtime"
	"syscall"
	"testing"
)

var multicastListenerTests = []struct {
	net   string
	gaddr *UDPAddr
	flags Flags
	ipv6  bool // test with underlying AF_INET6 socket
}{
	// cf. RFC 4727: Experimental Values in IPv4, IPv6, ICMPv4, ICMPv6, UDP, and TCP Headers

	{"udp", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, FlagUp | FlagLoopback, false},
	{"udp", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, 0, false},
	{"udp", &UDPAddr{ParseIP("ff0e::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp", &UDPAddr{ParseIP("ff0e::114"), 12345}, 0, true},

	{"udp4", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, FlagUp | FlagLoopback, false},
	{"udp4", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, 0, false},

	{"udp6", &UDPAddr{ParseIP("ff01::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff01::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff02::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff02::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff04::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff04::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff05::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff05::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff08::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff08::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff0e::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff0e::114"), 12345}, 0, true},
}

// TestMulticastListener tests both single and double listen to a test
// listener with same address family, same group address and same port.
func TestMulticastListener(t *testing.T) {
	switch runtime.GOOS {
	case "netbsd", "openbsd", "plan9", "windows":
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	case "linux":
		if runtime.GOARCH == "arm" || runtime.GOARCH == "alpha" {
			t.Logf("skipping test on %q/%q", runtime.GOOS, runtime.GOARCH)
			return
		}
	}

	for _, tt := range multicastListenerTests {
		if tt.ipv6 && (!supportsIPv6 || os.Getuid() != 0) {
			continue
		}
		ifi, err := availMulticastInterface(t, tt.flags)
		if err != nil {
			continue
		}
		c1, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("First ListenMulticastUDP failed: %v", err)
		}
		checkMulticastListener(t, err, c1, tt.gaddr)
		c2, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("Second ListenMulticastUDP failed: %v", err)
		}
		checkMulticastListener(t, err, c2, tt.gaddr)
		c2.Close()
		switch c1.fd.family {
		case syscall.AF_INET:
			testIPv4MulticastSocketOptions(t, c1.fd, ifi)
		case syscall.AF_INET6:
			testIPv6MulticastSocketOptions(t, c1.fd, ifi)
		}
		c1.Close()
	}
}

func TestSimpleMulticastListener(t *testing.T) {
	switch runtime.GOOS {
	case "plan9":
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	case "windows":
		if testing.Short() || !*testExternal {
			t.Logf("skipping test on windows to avoid firewall")
			return
		}
	}

	for _, tt := range multicastListenerTests {
		if tt.ipv6 {
			continue
		}
		tt.flags = FlagUp | FlagMulticast // for windows testing
		ifi, err := availMulticastInterface(t, tt.flags)
		if err != nil {
			continue
		}
		c1, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("First ListenMulticastUDP failed: %v", err)
		}
		checkSimpleMulticastListener(t, err, c1, tt.gaddr)
		c2, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("Second ListenMulticastUDP failed: %v", err)
		}
		checkSimpleMulticastListener(t, err, c2, tt.gaddr)
		c2.Close()
		c1.Close()
	}
}

func checkMulticastListener(t *testing.T, err error, c *UDPConn, gaddr *UDPAddr) {
	if !multicastRIBContains(t, gaddr.IP) {
		t.Fatalf("%q not found in RIB", gaddr.String())
	}
	if c.LocalAddr().String() != gaddr.String() {
		t.Fatalf("LocalAddr returns %q, expected %q", c.LocalAddr().String(), gaddr.String())
	}
}

func checkSimpleMulticastListener(t *testing.T, err error, c *UDPConn, gaddr *UDPAddr) {
	if c.LocalAddr().String() != gaddr.String() {
		t.Fatalf("LocalAddr returns %q, expected %q", c.LocalAddr().String(), gaddr.String())
	}
}

func availMulticastInterface(t *testing.T, flags Flags) (*Interface, error) {
	var ifi *Interface
	if flags != Flags(0) {
		ift, err := Interfaces()
		if err != nil {
			t.Fatalf("Interfaces failed: %v", err)
		}
		for _, x := range ift {
			if x.Flags&flags == flags {
				ifi = &x
				break
			}
		}
		if ifi == nil {
			return nil, errors.New("an appropriate multicast interface not found")
		}
	}
	return ifi, nil
}

func multicastRIBContains(t *testing.T, ip IP) bool {
	ift, err := Interfaces()
	if err != nil {
		t.Fatalf("Interfaces failed: %v", err)
	}
	for _, ifi := range ift {
		ifmat, err := ifi.MulticastAddrs()
		if err != nil {
			t.Fatalf("MulticastAddrs failed: %v", err)
		}
		for _, ifma := range ifmat {
			if ifma.(*IPAddr).IP.Equal(ip) {
				return true
			}
		}
	}
	return false
}

func testIPv4MulticastSocketOptions(t *testing.T, fd *netFD, ifi *Interface) {
	_, err := ipv4MulticastInterface(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastInterface failed: %v", err)
	}
	if ifi != nil {
		err = setIPv4MulticastInterface(fd, ifi)
		if err != nil {
			t.Fatalf("setIPv4MulticastInterface failed: %v", err)
		}
	}
	_, err = ipv4MulticastTTL(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastTTL failed: %v", err)
	}
	err = setIPv4MulticastTTL(fd, 1)
	if err != nil {
		t.Fatalf("setIPv4MulticastTTL failed: %v", err)
	}
	_, err = ipv4MulticastLoopback(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastLoopback failed: %v", err)
	}
	err = setIPv4MulticastLoopback(fd, false)
	if err != nil {
		t.Fatalf("setIPv4MulticastLoopback failed: %v", err)
	}
}

func testIPv6MulticastSocketOptions(t *testing.T, fd *netFD, ifi *Interface) {
	_, err := ipv6MulticastInterface(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastInterface failed: %v", err)
	}
	if ifi != nil {
		err = setIPv6MulticastInterface(fd, ifi)
		if err != nil {
			t.Fatalf("setIPv6MulticastInterface failed: %v", err)
		}
	}
	_, err = ipv6MulticastHopLimit(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastHopLimit failed: %v", err)
	}
	err = setIPv6MulticastHopLimit(fd, 1)
	if err != nil {
		t.Fatalf("setIPv6MulticastHopLimit failed: %v", err)
	}
	_, err = ipv6MulticastLoopback(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastLoopback failed: %v", err)
	}
	err = setIPv6MulticastLoopback(fd, false)
	if err != nil {
		t.Fatalf("setIPv6MulticastLoopback failed: %v", err)
	}
}
<commit_msg>net: disable IPv6 multicast tests unless external IPv6 connection is present<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !plan9

package net

import (
	"errors"
	"os"
	"runtime"
	"syscall"
	"testing"
)

var multicastListenerTests = []struct {
	net   string
	gaddr *UDPAddr
	flags Flags
	ipv6  bool // test with underlying AF_INET6 socket
}{
	// cf. RFC 4727: Experimental Values in IPv4, IPv6, ICMPv4, ICMPv6, UDP, and TCP Headers

	{"udp", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, FlagUp | FlagLoopback, false},
	{"udp", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, 0, false},
	{"udp", &UDPAddr{ParseIP("ff0e::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp", &UDPAddr{ParseIP("ff0e::114"), 12345}, 0, true},

	{"udp4", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, FlagUp | FlagLoopback, false},
	{"udp4", &UDPAddr{IPv4(224, 0, 0, 254), 12345}, 0, false},

	{"udp6", &UDPAddr{ParseIP("ff01::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff01::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff02::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff02::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff04::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff04::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff05::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff05::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff08::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff08::114"), 12345}, 0, true},
	{"udp6", &UDPAddr{ParseIP("ff0e::114"), 12345}, FlagUp | FlagLoopback, true},
	{"udp6", &UDPAddr{ParseIP("ff0e::114"), 12345}, 0, true},
}

// TestMulticastListener tests both single and double listen to a test
// listener with same address family, same group address and same port.
func TestMulticastListener(t *testing.T) {
	switch runtime.GOOS {
	case "netbsd", "openbsd", "plan9", "windows":
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	case "linux":
		if runtime.GOARCH == "arm" || runtime.GOARCH == "alpha" {
			t.Logf("skipping test on %q/%q", runtime.GOOS, runtime.GOARCH)
			return
		}
	}

	for _, tt := range multicastListenerTests {
		if tt.ipv6 && (!*testIPv6 || !supportsIPv6 || os.Getuid() != 0) {
			continue
		}
		ifi, err := availMulticastInterface(t, tt.flags)
		if err != nil {
			continue
		}
		c1, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("First ListenMulticastUDP failed: %v", err)
		}
		checkMulticastListener(t, err, c1, tt.gaddr)
		c2, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("Second ListenMulticastUDP failed: %v", err)
		}
		checkMulticastListener(t, err, c2, tt.gaddr)
		c2.Close()
		switch c1.fd.family {
		case syscall.AF_INET:
			testIPv4MulticastSocketOptions(t, c1.fd, ifi)
		case syscall.AF_INET6:
			testIPv6MulticastSocketOptions(t, c1.fd, ifi)
		}
		c1.Close()
	}
}

func TestSimpleMulticastListener(t *testing.T) {
	switch runtime.GOOS {
	case "plan9":
		t.Logf("skipping test on %q", runtime.GOOS)
		return
	case "windows":
		if testing.Short() || !*testExternal {
			t.Logf("skipping test on windows to avoid firewall")
			return
		}
	}

	for _, tt := range multicastListenerTests {
		if tt.ipv6 {
			continue
		}
		tt.flags = FlagUp | FlagMulticast // for windows testing
		ifi, err := availMulticastInterface(t, tt.flags)
		if err != nil {
			continue
		}
		c1, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("First ListenMulticastUDP failed: %v", err)
		}
		checkSimpleMulticastListener(t, err, c1, tt.gaddr)
		c2, err := ListenMulticastUDP(tt.net, ifi, tt.gaddr)
		if err != nil {
			t.Fatalf("Second ListenMulticastUDP failed: %v", err)
		}
		checkSimpleMulticastListener(t, err, c2, tt.gaddr)
		c2.Close()
		c1.Close()
	}
}

func checkMulticastListener(t *testing.T, err error, c *UDPConn, gaddr *UDPAddr) {
	if !multicastRIBContains(t, gaddr.IP) {
		t.Fatalf("%q not found in RIB", gaddr.String())
	}
	if c.LocalAddr().String() != gaddr.String() {
		t.Fatalf("LocalAddr returns %q, expected %q", c.LocalAddr().String(), gaddr.String())
	}
}

func checkSimpleMulticastListener(t *testing.T, err error, c *UDPConn, gaddr *UDPAddr) {
	if c.LocalAddr().String() != gaddr.String() {
		t.Fatalf("LocalAddr returns %q, expected %q", c.LocalAddr().String(), gaddr.String())
	}
}

func availMulticastInterface(t *testing.T, flags Flags) (*Interface, error) {
	var ifi *Interface
	if flags != Flags(0) {
		ift, err := Interfaces()
		if err != nil {
			t.Fatalf("Interfaces failed: %v", err)
		}
		for _, x := range ift {
			if x.Flags&flags == flags {
				ifi = &x
				break
			}
		}
		if ifi == nil {
			return nil, errors.New("an appropriate multicast interface not found")
		}
	}
	return ifi, nil
}

func multicastRIBContains(t *testing.T, ip IP) bool {
	ift, err := Interfaces()
	if err != nil {
		t.Fatalf("Interfaces failed: %v", err)
	}
	for _, ifi := range ift {
		ifmat, err := ifi.MulticastAddrs()
		if err != nil {
			t.Fatalf("MulticastAddrs failed: %v", err)
		}
		for _, ifma := range ifmat {
			if ifma.(*IPAddr).IP.Equal(ip) {
				return true
			}
		}
	}
	return false
}

func testIPv4MulticastSocketOptions(t *testing.T, fd *netFD, ifi *Interface) {
	_, err := ipv4MulticastInterface(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastInterface failed: %v", err)
	}
	if ifi != nil {
		err = setIPv4MulticastInterface(fd, ifi)
		if err != nil {
			t.Fatalf("setIPv4MulticastInterface failed: %v", err)
		}
	}
	_, err = ipv4MulticastTTL(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastTTL failed: %v", err)
	}
	err = setIPv4MulticastTTL(fd, 1)
	if err != nil {
		t.Fatalf("setIPv4MulticastTTL failed: %v", err)
	}
	_, err = ipv4MulticastLoopback(fd)
	if err != nil {
		t.Fatalf("ipv4MulticastLoopback failed: %v", err)
	}
	err = setIPv4MulticastLoopback(fd, false)
	if err != nil {
		t.Fatalf("setIPv4MulticastLoopback failed: %v", err)
	}
}

func testIPv6MulticastSocketOptions(t *testing.T, fd *netFD, ifi *Interface) {
	_, err := ipv6MulticastInterface(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastInterface failed: %v", err)
	}
	if ifi != nil {
		err = setIPv6MulticastInterface(fd, ifi)
		if err != nil {
			t.Fatalf("setIPv6MulticastInterface failed: %v", err)
		}
	}
	_, err = ipv6MulticastHopLimit(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastHopLimit failed: %v", err)
	}
	err = setIPv6MulticastHopLimit(fd, 1)
	if err != nil {
		t.Fatalf("setIPv6MulticastHopLimit failed: %v", err)
	}
	_, err = ipv6MulticastLoopback(fd)
	if err != nil {
		t.Fatalf("ipv6MulticastLoopback failed: %v", err)
	}
	err = setIPv6MulticastLoopback(fd, false)
	if err != nil {
		t.Fatalf("setIPv6MulticastLoopback failed: %v", err)
	}
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.330"
<commit_msg>fnserver: 0.3.331 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.331"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.186"
<commit_msg>functions: 0.3.187 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.187"
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"fmt"
	"net"

	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/v1"
	testgroupetcd "k8s.io/kubernetes/examples/apiserver/rest"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/runtime/schema"
	"k8s.io/kubernetes/pkg/storage/storagebackend"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"

	// Install the testgroup API
	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install"
)

const (
	// Ports on which to run the server.
	// Explicitly setting these to a different value than the default values, to prevent this from clashing with a local cluster.
	InsecurePort = 8081
	SecurePort   = 6444
)

func newStorageFactory() genericapiserver.StorageFactory {
	config := storagebackend.Config{
		Prefix:     genericoptions.DefaultEtcdPathPrefix,
		ServerList: []string{"http://127.0.0.1:2379"},
	}
	storageFactory := genericapiserver.NewDefaultStorageFactory(config, "application/json", api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), genericapiserver.NewResourceConfig())

	return storageFactory
}

func NewServerRunOptions() *genericoptions.ServerRunOptions {
	serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions().WithSecureServingOptions()
	serverOptions.InsecurePort = InsecurePort
	return serverOptions
}

func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error {
	// Set ServiceClusterIPRange
	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
	serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
	serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
	genericvalidation.ValidateRunOptions(serverOptions)
	if errs := serverOptions.Etcd.Validate(); len(errs) > 0 {
		return utilerrors.NewAggregate(errs)
	}
	config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete()
	if err := config.MaybeGenerateServingCerts(); err != nil {
		// this wasn't treated as fatal for this process before
		fmt.Printf("Error creating cert: %v", err)
	}

	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
	s, err := config.New()
	if err != nil {
		return fmt.Errorf("Error in bringing up the server: %v", err)
	}

	groupVersion := v1.SchemeGroupVersion
	groupName := groupVersion.Group
	groupMeta, err := registered.Group(groupName)
	if err != nil {
		return fmt.Errorf("%v", err)
	}
	storageFactory := newStorageFactory()
	storageConfig, err := storageFactory.NewConfig(schema.GroupResource{Group: groupName, Resource: "testtype"})
	if err != nil {
		return fmt.Errorf("Unable to get storage config: %v", err)
	}

	restStorageMap := map[string]rest.Storage{
		"testtypes": testgroupetcd.NewREST(storageConfig, generic.UndecoratedStorage),
	}
	apiGroupInfo := genericapiserver.APIGroupInfo{
		GroupMeta: *groupMeta,
		VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
			groupVersion.Version: restStorageMap,
		},
		Scheme:               api.Scheme,
		NegotiatedSerializer: api.Codecs,
	}
	if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
		return fmt.Errorf("Error in installing API: %v", err)
	}
	s.PrepareRun().Run(stopCh)
	return nil
}
<commit_msg>split insecure serving options<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"fmt"
	"net"

	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/v1"
	testgroupetcd "k8s.io/kubernetes/examples/apiserver/rest"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/runtime/schema"
	"k8s.io/kubernetes/pkg/storage/storagebackend"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"

	// Install the testgroup API
	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install"
)

const (
	// Ports on which to run the server.
	// Explicitly setting these to a different value than the default values, to prevent this from clashing with a local cluster.
	InsecurePort = 8081
	SecurePort   = 6444
)

func newStorageFactory() genericapiserver.StorageFactory {
	config := storagebackend.Config{
		Prefix:     genericoptions.DefaultEtcdPathPrefix,
		ServerList: []string{"http://127.0.0.1:2379"},
	}
	storageFactory := genericapiserver.NewDefaultStorageFactory(config, "application/json", api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), genericapiserver.NewResourceConfig())

	return storageFactory
}

func NewServerRunOptions() *genericoptions.ServerRunOptions {
	serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions().WithSecureServingOptions().WithInsecureServingOptions()
	serverOptions.InsecureServing.BindPort = InsecurePort
	return serverOptions
}

func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error {
	// Set ServiceClusterIPRange
	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
	serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
	serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
	genericvalidation.ValidateRunOptions(serverOptions)
	if errs := serverOptions.Etcd.Validate(); len(errs) > 0 {
		return utilerrors.NewAggregate(errs)
	}
	config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete()
	if err := config.MaybeGenerateServingCerts(); err != nil {
		// this wasn't treated as fatal for this process before
		fmt.Printf("Error creating cert: %v", err)
	}

	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
	s, err := config.New()
	if err != nil {
		return fmt.Errorf("Error in bringing up the server: %v", err)
	}

	groupVersion := v1.SchemeGroupVersion
	groupName := groupVersion.Group
	groupMeta, err := registered.Group(groupName)
	if err != nil {
		return fmt.Errorf("%v", err)
	}
	storageFactory := newStorageFactory()
	storageConfig, err := storageFactory.NewConfig(schema.GroupResource{Group: groupName, Resource: "testtype"})
	if err != nil {
		return fmt.Errorf("Unable to get storage config: %v", err)
	}

	restStorageMap := map[string]rest.Storage{
		"testtypes": testgroupetcd.NewREST(storageConfig, generic.UndecoratedStorage),
	}
	apiGroupInfo := genericapiserver.APIGroupInfo{
		GroupMeta: *groupMeta,
		VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
			groupVersion.Version: restStorageMap,
		},
		Scheme:               api.Scheme,
		NegotiatedSerializer: api.Codecs,
	}
	if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
		return fmt.Errorf("Error in installing API: %v", err)
	}
	s.PrepareRun().Run(stopCh)
	return nil
}
<|endoftext|>