<commit_before>package main

import (
	"encoding/json"
	"errors"
	"github.com/gorilla/mux"
	"github.com/tarantool/go-tarantool"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
)

const SERVER_DEFAULT_CONFIG = "config.json"

type ServerConfig struct {
	TownsDataBase      string `json:"TownsDataBase"`
	CashPointsDataBase string `json:"CashPointsDataBase"`
	CertificateDir     string `json:"CertificateDir"`
	Port               uint64 `json:"Port"`
	UserLoginMinLength uint64 `json:"UserLoginMinLength"`
	UserPwdMinLength   uint64 `json:"UserPwdMinLength"`
	UseTLS             bool   `json:"UseTLS"`
	RedisHost          string `json:"RedisHost"`
	RedisScriptsDir    string `json:"RedisScriptsDir"`
	ReqResLogTTL       uint64 `json:"ReqResLogTTL"`
	UUID_TTL           uint64 `json:"UUID_TTL"`
	BanksIcoDir        string `json:"BanksIcoDir"`
	TestingMode        bool   `json:"TestingMode"`
	TntUser            string `json:"TntUser"`
	TntPass            string `json:"TntPass"`
	TntUrl             string `json:"TntUrl"`
}

type Message struct {
	Text string `json:"text"`
}

func checkConvertionUint(val uint32, err error, context string) uint32 {
	if err != nil {
		log.Printf("%s: uint conversion err => %v\n", context, err)
		return 0
	}
	return val
}

func logRequest(w http.ResponseWriter, r *http.Request, requestId int64, requestBody string/*, redisCliPool *pool.Pool*/) error {
	endpointStr := r.URL.Path
	if requestBody != "" {
		endpointStr = endpointStr + " =>"
		body := ""

// 		redisCli, err := redisCliPool.Get()
// 		if err != nil {
// 			log.Fatal("logRequest: cannot get redisCli from pool")
// 			return err
// 		}
// 		defer redisCliPool.Put(redisCli)

// 		if isTestingModeEnabled(redisCli) {
// 			prettyJson, err := jsonPrettify(requestBody)
// 			if err == nil {
// 				body = "\n" + prettyJson
// 			} else {
// 				body = " " + requestBody
// 			}
// 		} else {
			body = " " + requestBody
// 		}
		endpointStr = endpointStr + body
	}
	log.Printf("%s Request: %s %s", getRequestContexString(r), r.Method, endpointStr)
	return nil
}

func logResponse(w http.ResponseWriter, r *http.Request, requestId int64, responseBody string/*, redisCliPool *pool.Pool*/) error {
	endpointStr := r.URL.Path
	if responseBody != "" {
		endpointStr = endpointStr + " =>"
		body := ""

// 		redisCli, err := redisCliPool.Get()
// 		if err != nil {
// 			log.Fatal("logRequest: cannot get redisCli from pool")
// 			return err
// 		}
// 		defer redisCliPool.Put(redisCli)
//
// 		if isTestingModeEnabled(redisCli) {
// 			prettyJson, err := jsonPrettify(responseBody)
// 			if err == nil {
// 				body = "\n" + prettyJson
// 			} else {
// 				body = " " + responseBody
// 			}
// 		} else {
			body = " " + responseBody
// 		}
		endpointStr = endpointStr + body
	}
	log.Printf("%s: Response: %s %s", getRequestContexString(r), r.Method, endpointStr)
	return nil
}

func getRequestContexString(r *http.Request) string {
	return r.RemoteAddr
}

func getHandlerContextString(funcName string, args map[string]string) string {
	result := funcName + "("
	i := 0
	argsCount := len(args)
	for argName, argVal := range args {
		result = result + argName + "=" + argVal
		if i < argsCount-1 {
			result = result + ","
		}
		i++
	}
	result = result + ")"

	return result
}

func getRequestUserId(r *http.Request) (int64, error) {
	requestIdStr := r.Header.Get("Id")
	if requestIdStr == "" {
		return 0, errors.New(`Request header val "Id" is not set`)
	}
	requestId, err := strconv.ParseInt(requestIdStr, 10, 64)
	if err != nil {
		return 0, errors.New(`Request header val "Id" uint conversion failed: ` + requestIdStr)
	}
	return requestId, nil
}

func getRequestJsonStr(r *http.Request, context string) (string, error) {
	jsonStr, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Printf("%s => malformed json\n", context)
		return "", err
	}
	return string(jsonStr), nil
}

func prepareResponse(w http.ResponseWriter, r *http.Request) (bool, int64) {
	requestId, err := getRequestUserId(r)
	if err != nil {
		log.Printf("%s prepareResponse %v\n", getRequestContexString(r), err)
		w.WriteHeader(401)
		return false, 0
	}
	if requestId == 0 {
		log.Printf("%s prepareResponse unexpected requestId: %d\n", getRequestContexString(r), requestId)
		w.WriteHeader(401)
		return false, 0
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Id", strconv.FormatInt(requestId, 10))
	return true, requestId
}

func writeResponse(w http.ResponseWriter, r *http.Request, requestId int64, responseBody string/*, redisCliPool *pool.Pool*/) {
	io.WriteString(w, responseBody)
	go logResponse(w, r, requestId, responseBody/*, redisCliPool*/)
}

type EndpointCallback func(w http.ResponseWriter, r *http.Request)

func handlerPing(tnt *tarantool.Connection) (string, EndpointCallback) {
	return "/ping", func(w http.ResponseWriter, r *http.Request) {
		ok, requestId := prepareResponse(w, r)
		if ok == false {
			return
		}

		go logRequest(w, r, requestId, "")
		msg := &Message{Text: "pong"}
		jsonByteArr, _ := json.Marshal(msg)
		writeResponse(w, r, requestId, string(jsonByteArr))
	}
}

func main() {
	log.SetFlags(log.Flags() | log.Lmicroseconds)

	args := os.Args[1:]

	configFilePath := SERVER_DEFAULT_CONFIG
	if len(args) > 0 {
		configFilePath = args[0]
		log.Printf("Loading config file: %s\n", configFilePath)
	} else {
		log.Printf("Loading default config file: %s\n", configFilePath)
	}

	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
		log.Fatalf("No such config file: %s\n", configFilePath)
	}

	configFile, _ := os.Open(configFilePath)
	decoder := json.NewDecoder(configFile)
	serverConfig := ServerConfig{}
	err := decoder.Decode(&serverConfig)
	if err != nil {
		log.Fatalf("Failed to decode config file: %s\nError: %v\n", configFilePath, err)
		return
	}

	if serverConfig.TestingMode {
		log.Printf("WARNING: Server started is TESTING mode! Make sure it is not prod server.")
	}

	opts := tarantool.Opts{
		Reconnect: 1 * time.Second,
		MaxReconnects: 3,
		User: serverConfig.TntUser,
 		Pass: serverConfig.TntPass,
	}
	tnt, err := tarantool.Connect(serverConfig.TntUrl, opts)
	if err != nil {
		log.Fatal(err)
	}
	defer tnt.Close()

	router := mux.NewRouter()
	router.HandleFunc(handlerPing(tnt)).Methods("GET")
	router.HandleFunc(handlerCashpoint(tnt)).Methods("GET")
	router.HandleFunc(handlerCashpointCreate(tnt)).Methods("POST")
	router.HandleFunc(handlerCashpointsBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerTown(tnt)).Methods("GET")
	router.HandleFunc(handlerTownsBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerTownsList(tnt)).Methods("GET")
	router.HandleFunc(handlerBank(tnt)).Methods("GET")
	router.HandleFunc(handlerBankIco(serverConfig)).Methods("GET")
	router.HandleFunc(handlerBanksList(tnt)).Methods("GET")
	router.HandleFunc(handlerBanksBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerNearbyCashPoints(tnt)).Methods("POST")
	router.HandleFunc(handlerNearbyClusters(tnt)).Methods("POST")

	if serverConfig.TestingMode {
		router.HandleFunc(handlerCoordToQuadKey(tnt)).Methods("POST")
		router.HandleFunc(handlerQuadTreeBranch(tnt)).Methods("GET")
		router.HandleFunc(handlerCashpointDelete(tnt)).Methods("DELETE")
		router.HandleFunc(handlerSpaceMetrics(tnt)).Methods("GET")
	}

	port := strconv.FormatUint(serverConfig.Port, 10)
	log.Println("Listening port: " + port)

	server := &http.Server{
		Addr:           ":" + port,
		Handler:        router,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	err = server.ListenAndServe()
	if err != nil {
		log.Fatal(err)
	}
}<commit_msg>[C|Server] gofmt on cpsrv.go<commit_after>package main

import (
	"encoding/json"
	"errors"
	"github.com/gorilla/mux"
	"github.com/tarantool/go-tarantool"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
)

const SERVER_DEFAULT_CONFIG = "config.json"

type ServerConfig struct {
	TownsDataBase      string `json:"TownsDataBase"`
	CashPointsDataBase string `json:"CashPointsDataBase"`
	CertificateDir     string `json:"CertificateDir"`
	Port               uint64 `json:"Port"`
	UserLoginMinLength uint64 `json:"UserLoginMinLength"`
	UserPwdMinLength   uint64 `json:"UserPwdMinLength"`
	UseTLS             bool   `json:"UseTLS"`
	RedisHost          string `json:"RedisHost"`
	RedisScriptsDir    string `json:"RedisScriptsDir"`
	ReqResLogTTL       uint64 `json:"ReqResLogTTL"`
	UUID_TTL           uint64 `json:"UUID_TTL"`
	BanksIcoDir        string `json:"BanksIcoDir"`
	TestingMode        bool   `json:"TestingMode"`
	TntUser            string `json:"TntUser"`
	TntPass            string `json:"TntPass"`
	TntUrl             string `json:"TntUrl"`
}

type Message struct {
	Text string `json:"text"`
}

func checkConvertionUint(val uint32, err error, context string) uint32 {
	if err != nil {
		log.Printf("%s: uint conversion err => %v\n", context, err)
		return 0
	}
	return val
}

func logRequest(w http.ResponseWriter, r *http.Request, requestId int64, requestBody string) error {
	endpointStr := r.URL.Path
	if requestBody != "" {
		endpointStr = endpointStr + " =>"
		body := ""

		//if isTestingModeEnabled(redisCli) {
		//	prettyJson, err := jsonPrettify(requestBody)
		//	if err == nil {
		//		body = "\n" + prettyJson
		//	} else {
		//		body = " " + requestBody
		//	}
		//}
		body = " " + requestBody
		endpointStr = endpointStr + body
	}
	log.Printf("%s Request: %s %s", getRequestContexString(r), r.Method, endpointStr)
	return nil
}

func logResponse(w http.ResponseWriter, r *http.Request, requestId int64, responseBody string) error {
	endpointStr := r.URL.Path
	if responseBody != "" {
		endpointStr = endpointStr + " =>"
		body := ""

		//if isTestingModeEnabled(redisCli) {
		//	prettyJson, err := jsonPrettify(responseBody)
		//	if err == nil {
		//		body = "\n" + prettyJson
		//	} else {
		//		body = " " + responseBody
		//	}
		//
		body = " " + responseBody
		endpointStr = endpointStr + body
	}
	log.Printf("%s: Response: %s %s", getRequestContexString(r), r.Method, endpointStr)
	return nil
}

func getRequestContexString(r *http.Request) string {
	return r.RemoteAddr
}

func getHandlerContextString(funcName string, args map[string]string) string {
	result := funcName + "("
	i := 0
	argsCount := len(args)
	for argName, argVal := range args {
		result = result + argName + "=" + argVal
		if i < argsCount-1 {
			result = result + ","
		}
		i++
	}
	result = result + ")"

	return result
}

func getRequestUserId(r *http.Request) (int64, error) {
	requestIdStr := r.Header.Get("Id")
	if requestIdStr == "" {
		return 0, errors.New(`Request header val "Id" is not set`)
	}
	requestId, err := strconv.ParseInt(requestIdStr, 10, 64)
	if err != nil {
		return 0, errors.New(`Request header val "Id" uint conversion failed: ` + requestIdStr)
	}
	return requestId, nil
}

func getRequestJsonStr(r *http.Request, context string) (string, error) {
	jsonStr, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Printf("%s => malformed json\n", context)
		return "", err
	}
	return string(jsonStr), nil
}

func prepareResponse(w http.ResponseWriter, r *http.Request) (bool, int64) {
	requestId, err := getRequestUserId(r)
	if err != nil {
		log.Printf("%s prepareResponse %v\n", getRequestContexString(r), err)
		w.WriteHeader(401)
		return false, 0
	}
	if requestId == 0 {
		log.Printf("%s prepareResponse unexpected requestId: %d\n", getRequestContexString(r), requestId)
		w.WriteHeader(401)
		return false, 0
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Id", strconv.FormatInt(requestId, 10))
	return true, requestId
}

func writeResponse(w http.ResponseWriter, r *http.Request, requestId int64, responseBody string) {
	io.WriteString(w, responseBody)
	go logResponse(w, r, requestId, responseBody)
}

type EndpointCallback func(w http.ResponseWriter, r *http.Request)

func handlerPing(tnt *tarantool.Connection) (string, EndpointCallback) {
	return "/ping", func(w http.ResponseWriter, r *http.Request) {
		ok, requestId := prepareResponse(w, r)
		if ok == false {
			return
		}

		go logRequest(w, r, requestId, "")
		msg := &Message{Text: "pong"}
		jsonByteArr, _ := json.Marshal(msg)
		writeResponse(w, r, requestId, string(jsonByteArr))
	}
}

func main() {
	log.SetFlags(log.Flags() | log.Lmicroseconds)

	args := os.Args[1:]

	configFilePath := SERVER_DEFAULT_CONFIG
	if len(args) > 0 {
		configFilePath = args[0]
		log.Printf("Loading config file: %s\n", configFilePath)
	} else {
		log.Printf("Loading default config file: %s\n", configFilePath)
	}

	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
		log.Fatalf("No such config file: %s\n", configFilePath)
	}

	configFile, _ := os.Open(configFilePath)
	decoder := json.NewDecoder(configFile)
	serverConfig := ServerConfig{}
	err := decoder.Decode(&serverConfig)
	if err != nil {
		log.Fatalf("Failed to decode config file: %s\nError: %v\n", configFilePath, err)
		return
	}

	if serverConfig.TestingMode {
		log.Printf("WARNING: Server started is TESTING mode! Make sure it is not prod server.")
	}

	opts := tarantool.Opts{
		Reconnect:     1 * time.Second,
		MaxReconnects: 3,
		User:          serverConfig.TntUser,
		Pass:          serverConfig.TntPass,
	}
	tnt, err := tarantool.Connect(serverConfig.TntUrl, opts)
	if err != nil {
		log.Fatal(err)
	}
	defer tnt.Close()

	router := mux.NewRouter()
	router.HandleFunc(handlerPing(tnt)).Methods("GET")
	router.HandleFunc(handlerCashpoint(tnt)).Methods("GET")
	router.HandleFunc(handlerCashpointCreate(tnt)).Methods("POST")
	router.HandleFunc(handlerCashpointsBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerTown(tnt)).Methods("GET")
	router.HandleFunc(handlerTownsBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerTownsList(tnt)).Methods("GET")
	router.HandleFunc(handlerBank(tnt)).Methods("GET")
	router.HandleFunc(handlerBankIco(serverConfig)).Methods("GET")
	router.HandleFunc(handlerBanksList(tnt)).Methods("GET")
	router.HandleFunc(handlerBanksBatch(tnt)).Methods("POST")
	router.HandleFunc(handlerNearbyCashPoints(tnt)).Methods("POST")
	router.HandleFunc(handlerNearbyClusters(tnt)).Methods("POST")

	if serverConfig.TestingMode {
		router.HandleFunc(handlerCoordToQuadKey(tnt)).Methods("POST")
		router.HandleFunc(handlerQuadTreeBranch(tnt)).Methods("GET")
		router.HandleFunc(handlerCashpointDelete(tnt)).Methods("DELETE")
		router.HandleFunc(handlerSpaceMetrics(tnt)).Methods("GET")
	}

	port := strconv.FormatUint(serverConfig.Port, 10)
	log.Println("Listening port: " + port)

	server := &http.Server{
		Addr:           ":" + port,
		Handler:        router,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}

	err = server.ListenAndServe()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package ipam

import (
	"encoding/json"
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/ipamapi"
	"github.com/docker/libnetwork/types"
)

// SubnetKey is the pointer to the configured pools in each address space
type SubnetKey struct {
	AddressSpace string
	Subnet       string
	ChildSubnet  string
}

// PoolData contains the configured pool data
type PoolData struct {
	ParentKey SubnetKey
	Pool      *net.IPNet
	Range     *AddressRange `json:",omitempty"`
	RefCount  int
}

// addrSpace contains the pool configurations for the address space
type addrSpace struct {
	subnets  map[SubnetKey]*PoolData
	dbIndex  uint64
	dbExists bool
	id       string
	scope    string
	ds       datastore.DataStore
	alloc    *Allocator
	sync.Mutex
}

// AddressRange specifies first and last ip ordinal which
// identifies a range in a pool of addresses
type AddressRange struct {
	Sub        *net.IPNet
	Start, End uint64
}

// String returns the string form of the AddressRange object
func (r *AddressRange) String() string {
	return fmt.Sprintf("Sub: %s, range [%d, %d]", r.Sub, r.Start, r.End)
}

// MarshalJSON returns the JSON encoding of the Range object
func (r *AddressRange) MarshalJSON() ([]byte, error) {
	m := map[string]interface{}{
		"Sub":   r.Sub.String(),
		"Start": r.Start,
		"End":   r.End,
	}
	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the Range object
func (r *AddressRange) UnmarshalJSON(data []byte) error {
	m := map[string]interface{}{}
	err := json.Unmarshal(data, &m)
	if err != nil {
		return err
	}
	if r.Sub, err = types.ParseCIDR(m["Sub"].(string)); err != nil {
		return err
	}
	r.Start = uint64(m["Start"].(float64))
	r.End = uint64(m["End"].(float64))
	return nil
}

// String returns the string form of the SubnetKey object
func (s *SubnetKey) String() string {
	k := fmt.Sprintf("%s/%s", s.AddressSpace, s.Subnet)
	if s.ChildSubnet != "" {
		k = fmt.Sprintf("%s/%s", k, s.ChildSubnet)
	}
	return k
}

// FromString populates the SubnetKey object reading it from string
func (s *SubnetKey) FromString(str string) error {
	if str == "" || !strings.Contains(str, "/") {
		return types.BadRequestErrorf("invalid string form for subnetkey: %s", str)
	}

	p := strings.Split(str, "/")
	if len(p) != 3 && len(p) != 5 {
		return types.BadRequestErrorf("invalid string form for subnetkey: %s", str)
	}
	s.AddressSpace = p[0]
	s.Subnet = fmt.Sprintf("%s/%s", p[1], p[2])
	if len(p) == 5 {
		s.ChildSubnet = fmt.Sprintf("%s/%s", p[3], p[4])
	}

	return nil
}

// String returns the string form of the PoolData object
func (p *PoolData) String() string {
	return fmt.Sprintf("ParentKey: %s, Pool: %s, Range: %s, RefCount: %d",
		p.ParentKey.String(), p.Pool.String(), p.Range, p.RefCount)
}

// MarshalJSON returns the JSON encoding of the PoolData object
func (p *PoolData) MarshalJSON() ([]byte, error) {
	m := map[string]interface{}{
		"ParentKey": p.ParentKey,
		"RefCount":  p.RefCount,
	}
	if p.Pool != nil {
		m["Pool"] = p.Pool.String()
	}
	if p.Range != nil {
		m["Range"] = p.Range
	}
	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the PoolData object
func (p *PoolData) UnmarshalJSON(data []byte) error {
	var (
		err error
		t   struct {
			ParentKey SubnetKey
			Pool      string
			Range     *AddressRange `json:",omitempty"`
			RefCount  int
		}
	)

	if err = json.Unmarshal(data, &t); err != nil {
		return err
	}

	p.ParentKey = t.ParentKey
	p.Range = t.Range
	p.RefCount = t.RefCount
	if t.Pool != "" {
		if p.Pool, err = types.ParseCIDR(t.Pool); err != nil {
			return err
		}
	}

	return nil
}

// MarshalJSON returns the JSON encoding of the addrSpace object
func (aSpace *addrSpace) MarshalJSON() ([]byte, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	m := map[string]interface{}{
		"Scope": string(aSpace.scope),
	}

	if aSpace.subnets != nil {
		s := map[string]*PoolData{}
		for k, v := range aSpace.subnets {
			s[k.String()] = v
		}
		m["Subnets"] = s
	}

	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the addrSpace object
func (aSpace *addrSpace) UnmarshalJSON(data []byte) error {
	aSpace.Lock()
	defer aSpace.Unlock()

	m := map[string]interface{}{}
	err := json.Unmarshal(data, &m)
	if err != nil {
		return err
	}

	aSpace.scope = datastore.LocalScope
	s := m["Scope"].(string)
	if s == string(datastore.GlobalScope) {
		aSpace.scope = datastore.GlobalScope
	}

	if v, ok := m["Subnets"]; ok {
		sb, _ := json.Marshal(v)
		var s map[string]*PoolData
		err := json.Unmarshal(sb, &s)
		if err != nil {
			return err
		}
		for ks, v := range s {
			k := SubnetKey{}
			k.FromString(ks)
			aSpace.subnets[k] = v
		}
	}

	return nil
}

// CopyTo deep copies the pool data to the destination pooldata
func (p *PoolData) CopyTo(dstP *PoolData) error {
	dstP.ParentKey = p.ParentKey
	dstP.Pool = types.GetIPNetCopy(p.Pool)

	if p.Range != nil {
		dstP.Range = &AddressRange{}
		dstP.Range.Sub = types.GetIPNetCopy(p.Range.Sub)
		dstP.Range.Start = p.Range.Start
		dstP.Range.End = p.Range.End
	}

	dstP.RefCount = p.RefCount
	return nil
}

func (aSpace *addrSpace) CopyTo(o datastore.KVObject) error {
	aSpace.Lock()
	defer aSpace.Unlock()

	dstAspace := o.(*addrSpace)

	dstAspace.id = aSpace.id
	dstAspace.ds = aSpace.ds
	dstAspace.alloc = aSpace.alloc
	dstAspace.scope = aSpace.scope
	dstAspace.dbIndex = aSpace.dbIndex
	dstAspace.dbExists = aSpace.dbExists

	dstAspace.subnets = make(map[SubnetKey]*PoolData)
	for k, v := range aSpace.subnets {
		dstAspace.subnets[k] = &PoolData{}
		v.CopyTo(dstAspace.subnets[k])
	}

	return nil
}

func (aSpace *addrSpace) New() datastore.KVObject {
	aSpace.Lock()
	defer aSpace.Unlock()

	return &addrSpace{
		id:    aSpace.id,
		ds:    aSpace.ds,
		alloc: aSpace.alloc,
		scope: aSpace.scope,
	}
}

func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange, pdf bool) (func() error, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	// Check if already allocated
	if p, ok := aSpace.subnets[k]; ok {
		if pdf {
			return nil, types.InternalMaskableErrorf("predefined pool %s is already reserved", nw)
		}
		aSpace.incRefCount(p, 1)
		return func() error { return nil }, nil
	}

	// If master pool, check for overlap
	if ipr == nil {
		if aSpace.contains(k.AddressSpace, nw) {
			return nil, ipamapi.ErrPoolOverlap
		}
		// This is a new master pool, add it along with corresponding bitmask
		aSpace.subnets[k] = &PoolData{Pool: nw, RefCount: 1}
		return func() error { return aSpace.alloc.insertBitMask(k, nw) }, nil
	}

	// This is a new non-master pool
	p := &PoolData{
		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
		Pool:      nw,
		Range:     ipr,
		RefCount:  1,
	}
	aSpace.subnets[k] = p

	// Look for parent pool
	pp, ok := aSpace.subnets[p.ParentKey]
	if ok {
		aSpace.incRefCount(pp, 1)
		return func() error { return nil }, nil
	}

	// Parent pool does not exist, add it along with corresponding bitmask
	aSpace.subnets[p.ParentKey] = &PoolData{Pool: nw, RefCount: 1}
	return func() error { return aSpace.alloc.insertBitMask(p.ParentKey, nw) }, nil
}

func (aSpace *addrSpace) updatePoolDBOnRemoval(k SubnetKey) (func() error, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	p, ok := aSpace.subnets[k]
	if !ok {
		return nil, ipamapi.ErrBadPool
	}

	aSpace.incRefCount(p, -1)

	c := p
	for ok {
		if c.RefCount == 0 {
			delete(aSpace.subnets, k)
			if c.Range == nil {
				return func() error {
					bm, err := aSpace.alloc.retrieveBitmask(k, c.Pool)
					if err != nil {
						return types.InternalErrorf("could not find bitmask in datastore for pool %s removal: %v", k.String(), err)
					}
					return bm.Destroy()
				}, nil
			}
		}
		k = c.ParentKey
		c, ok = aSpace.subnets[k]
	}

	return func() error { return nil }, nil
}

func (aSpace *addrSpace) incRefCount(p *PoolData, delta int) {
	c := p
	ok := true
	for ok {
		c.RefCount += delta
		c, ok = aSpace.subnets[c.ParentKey]
	}
}

// Checks whether the passed subnet is a superset or subset of any of the subset in this config db
func (aSpace *addrSpace) contains(space string, nw *net.IPNet) bool {
	for k, v := range aSpace.subnets {
		if space == k.AddressSpace && k.ChildSubnet == "" {
			if nw.Contains(v.Pool.IP) || v.Pool.Contains(nw.IP) {
				return true
			}
		}
	}
	return false
}

func (aSpace *addrSpace) store() datastore.DataStore {
	aSpace.Lock()
	defer aSpace.Unlock()

	return aSpace.ds
}
<commit_msg>Add check for overlapping subnets<commit_after>package ipam

import (
	"encoding/json"
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/ipamapi"
	"github.com/docker/libnetwork/types"
)

// SubnetKey is the pointer to the configured pools in each address space
type SubnetKey struct {
	AddressSpace string
	Subnet       string
	ChildSubnet  string
}

// PoolData contains the configured pool data
type PoolData struct {
	ParentKey SubnetKey
	Pool      *net.IPNet
	Range     *AddressRange `json:",omitempty"`
	RefCount  int
}

// addrSpace contains the pool configurations for the address space
type addrSpace struct {
	subnets  map[SubnetKey]*PoolData
	dbIndex  uint64
	dbExists bool
	id       string
	scope    string
	ds       datastore.DataStore
	alloc    *Allocator
	sync.Mutex
}

// AddressRange specifies first and last ip ordinal which
// identifies a range in a pool of addresses
type AddressRange struct {
	Sub        *net.IPNet
	Start, End uint64
}

// String returns the string form of the AddressRange object
func (r *AddressRange) String() string {
	return fmt.Sprintf("Sub: %s, range [%d, %d]", r.Sub, r.Start, r.End)
}

// MarshalJSON returns the JSON encoding of the Range object
func (r *AddressRange) MarshalJSON() ([]byte, error) {
	m := map[string]interface{}{
		"Sub":   r.Sub.String(),
		"Start": r.Start,
		"End":   r.End,
	}
	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the Range object
func (r *AddressRange) UnmarshalJSON(data []byte) error {
	m := map[string]interface{}{}
	err := json.Unmarshal(data, &m)
	if err != nil {
		return err
	}
	if r.Sub, err = types.ParseCIDR(m["Sub"].(string)); err != nil {
		return err
	}
	r.Start = uint64(m["Start"].(float64))
	r.End = uint64(m["End"].(float64))
	return nil
}

// String returns the string form of the SubnetKey object
func (s *SubnetKey) String() string {
	k := fmt.Sprintf("%s/%s", s.AddressSpace, s.Subnet)
	if s.ChildSubnet != "" {
		k = fmt.Sprintf("%s/%s", k, s.ChildSubnet)
	}
	return k
}

// FromString populates the SubnetKey object reading it from string
func (s *SubnetKey) FromString(str string) error {
	if str == "" || !strings.Contains(str, "/") {
		return types.BadRequestErrorf("invalid string form for subnetkey: %s", str)
	}

	p := strings.Split(str, "/")
	if len(p) != 3 && len(p) != 5 {
		return types.BadRequestErrorf("invalid string form for subnetkey: %s", str)
	}
	s.AddressSpace = p[0]
	s.Subnet = fmt.Sprintf("%s/%s", p[1], p[2])
	if len(p) == 5 {
		s.ChildSubnet = fmt.Sprintf("%s/%s", p[3], p[4])
	}

	return nil
}

// String returns the string form of the PoolData object
func (p *PoolData) String() string {
	return fmt.Sprintf("ParentKey: %s, Pool: %s, Range: %s, RefCount: %d",
		p.ParentKey.String(), p.Pool.String(), p.Range, p.RefCount)
}

// MarshalJSON returns the JSON encoding of the PoolData object
func (p *PoolData) MarshalJSON() ([]byte, error) {
	m := map[string]interface{}{
		"ParentKey": p.ParentKey,
		"RefCount":  p.RefCount,
	}
	if p.Pool != nil {
		m["Pool"] = p.Pool.String()
	}
	if p.Range != nil {
		m["Range"] = p.Range
	}
	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the PoolData object
func (p *PoolData) UnmarshalJSON(data []byte) error {
	var (
		err error
		t   struct {
			ParentKey SubnetKey
			Pool      string
			Range     *AddressRange `json:",omitempty"`
			RefCount  int
		}
	)

	if err = json.Unmarshal(data, &t); err != nil {
		return err
	}

	p.ParentKey = t.ParentKey
	p.Range = t.Range
	p.RefCount = t.RefCount
	if t.Pool != "" {
		if p.Pool, err = types.ParseCIDR(t.Pool); err != nil {
			return err
		}
	}

	return nil
}

// MarshalJSON returns the JSON encoding of the addrSpace object
func (aSpace *addrSpace) MarshalJSON() ([]byte, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	m := map[string]interface{}{
		"Scope": string(aSpace.scope),
	}

	if aSpace.subnets != nil {
		s := map[string]*PoolData{}
		for k, v := range aSpace.subnets {
			s[k.String()] = v
		}
		m["Subnets"] = s
	}

	return json.Marshal(m)
}

// UnmarshalJSON decodes data into the addrSpace object
func (aSpace *addrSpace) UnmarshalJSON(data []byte) error {
	aSpace.Lock()
	defer aSpace.Unlock()

	m := map[string]interface{}{}
	err := json.Unmarshal(data, &m)
	if err != nil {
		return err
	}

	aSpace.scope = datastore.LocalScope
	s := m["Scope"].(string)
	if s == string(datastore.GlobalScope) {
		aSpace.scope = datastore.GlobalScope
	}

	if v, ok := m["Subnets"]; ok {
		sb, _ := json.Marshal(v)
		var s map[string]*PoolData
		err := json.Unmarshal(sb, &s)
		if err != nil {
			return err
		}
		for ks, v := range s {
			k := SubnetKey{}
			k.FromString(ks)
			aSpace.subnets[k] = v
		}
	}

	return nil
}

// CopyTo deep copies the pool data to the destination pooldata
func (p *PoolData) CopyTo(dstP *PoolData) error {
	dstP.ParentKey = p.ParentKey
	dstP.Pool = types.GetIPNetCopy(p.Pool)

	if p.Range != nil {
		dstP.Range = &AddressRange{}
		dstP.Range.Sub = types.GetIPNetCopy(p.Range.Sub)
		dstP.Range.Start = p.Range.Start
		dstP.Range.End = p.Range.End
	}

	dstP.RefCount = p.RefCount
	return nil
}

func (aSpace *addrSpace) CopyTo(o datastore.KVObject) error {
	aSpace.Lock()
	defer aSpace.Unlock()

	dstAspace := o.(*addrSpace)

	dstAspace.id = aSpace.id
	dstAspace.ds = aSpace.ds
	dstAspace.alloc = aSpace.alloc
	dstAspace.scope = aSpace.scope
	dstAspace.dbIndex = aSpace.dbIndex
	dstAspace.dbExists = aSpace.dbExists

	dstAspace.subnets = make(map[SubnetKey]*PoolData)
	for k, v := range aSpace.subnets {
		dstAspace.subnets[k] = &PoolData{}
		v.CopyTo(dstAspace.subnets[k])
	}

	return nil
}

func (aSpace *addrSpace) New() datastore.KVObject {
	aSpace.Lock()
	defer aSpace.Unlock()

	return &addrSpace{
		id:    aSpace.id,
		ds:    aSpace.ds,
		alloc: aSpace.alloc,
		scope: aSpace.scope,
	}
}

func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange, pdf bool) (func() error, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	// Check if already allocated
	if _, ok := aSpace.subnets[k]; ok {
		if pdf {
			return nil, types.InternalMaskableErrorf("predefined pool %s is already reserved", nw)
		}
		// This means the same pool is already allocated. updatePoolDBOnAdd is called when there
		// is request for a pool/subpool. It should ensure there is no overlap with existing pools
		return nil, ipamapi.ErrPoolOverlap
	}

	// If master pool, check for overlap
	if ipr == nil {
		if aSpace.contains(k.AddressSpace, nw) {
			return nil, ipamapi.ErrPoolOverlap
		}
		// This is a new master pool, add it along with corresponding bitmask
		aSpace.subnets[k] = &PoolData{Pool: nw, RefCount: 1}
		return func() error { return aSpace.alloc.insertBitMask(k, nw) }, nil
	}

	// This is a new non-master pool
	p := &PoolData{
		ParentKey: SubnetKey{AddressSpace: k.AddressSpace, Subnet: k.Subnet},
		Pool:      nw,
		Range:     ipr,
		RefCount:  1,
	}
	aSpace.subnets[k] = p

	// Look for parent pool
	pp, ok := aSpace.subnets[p.ParentKey]
	if ok {
		aSpace.incRefCount(pp, 1)
		return func() error { return nil }, nil
	}

	// Parent pool does not exist, add it along with corresponding bitmask
	aSpace.subnets[p.ParentKey] = &PoolData{Pool: nw, RefCount: 1}
	return func() error { return aSpace.alloc.insertBitMask(p.ParentKey, nw) }, nil
}

func (aSpace *addrSpace) updatePoolDBOnRemoval(k SubnetKey) (func() error, error) {
	aSpace.Lock()
	defer aSpace.Unlock()

	p, ok := aSpace.subnets[k]
	if !ok {
		return nil, ipamapi.ErrBadPool
	}

	aSpace.incRefCount(p, -1)

	c := p
	for ok {
		if c.RefCount == 0 {
			delete(aSpace.subnets, k)
			if c.Range == nil {
				return func() error {
					bm, err := aSpace.alloc.retrieveBitmask(k, c.Pool)
					if err != nil {
						return types.InternalErrorf("could not find bitmask in datastore for pool %s removal: %v", k.String(), err)
					}
					return bm.Destroy()
				}, nil
			}
		}
		k = c.ParentKey
		c, ok = aSpace.subnets[k]
	}

	return func() error { return nil }, nil
}

func (aSpace *addrSpace) incRefCount(p *PoolData, delta int) {
	c := p
	ok := true
	for ok {
		c.RefCount += delta
		c, ok = aSpace.subnets[c.ParentKey]
	}
}

// Checks whether the passed subnet is a superset or subset of any of the subset in this config db
func (aSpace *addrSpace) contains(space string, nw *net.IPNet) bool {
	for k, v := range aSpace.subnets {
		if space == k.AddressSpace && k.ChildSubnet == "" {
			if nw.Contains(v.Pool.IP) || v.Pool.Contains(nw.IP) {
				return true
			}
		}
	}
	return false
}

func (aSpace *addrSpace) store() datastore.DataStore {
	aSpace.Lock()
	defer aSpace.Unlock()

	return aSpace.ds
}
<|endoftext|>


<commit_before>package tsm1

import (
	"fmt"
	"log"
	"os"
	"sort"
	"sync"
)

var ErrCacheMemoryExceeded = fmt.Errorf("cache maximum memory size exceeded")
var ErrCacheInvalidCheckpoint = fmt.Errorf("invalid checkpoint")

// entry is a set of values and some metadata.
type entry struct {
	values   Values // All stored values.
	needSort bool   // true if the values are out of order and require deduping.
}

// newEntry returns a new instance of entry.
func newEntry() *entry {
	return &entry{}
}

// add adds the given values to the entry.
func (e *entry) add(values []Value) {
	// if there are existing values make sure they're all less than the first of
	// the new values being added
	l := len(e.values)
	if l != 0 {
		lastValTime := e.values[l-1].UnixNano()
		if lastValTime >= values[0].UnixNano() {
			e.needSort = true
		}
	}
	e.values = append(e.values, values...)

	// if there's only one value, we know it's sorted
	if len(values) <= 1 {
		return
	}

	// make sure the new values were in sorted order
	min := values[0].UnixNano()
	for _, v := range values[1:] {
		if min >= v.UnixNano() {
			e.needSort = true
			break
		}
	}
}

// deduplicate sorts and orders the entry's values. If values are already deduped and
// and sorted, the function does no work and simply returns.
func (e *entry) deduplicate() {
	if !e.needSort || len(e.values) == 0 {
		return
	}
	e.values = e.values.Deduplicate()
	e.needSort = false
}

// Cache maintains an in-memory store of Values for a set of keys.
type Cache struct {
	mu      sync.RWMutex
	store   map[string]*entry
	size    uint64
	maxSize uint64

	// snapshots are the cache objects that are currently being written to tsm files
	// they're kept in memory while flushing so they can be queried along with the cache.
	// they are read only and should never be modified
	snapshots     []*Cache
	snapshotsSize uint64
}

// NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory.
func NewCache(maxSize uint64) *Cache {
	return &Cache{
		maxSize: maxSize,
		store:   make(map[string]*entry),
	}
}

// Write writes the set of values for the key to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) Write(key string, values []Value) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Enough room in the cache?
	newSize := c.size + uint64(Values(values).Size())
	if c.maxSize > 0 && newSize+c.snapshotsSize > c.maxSize {
		return ErrCacheMemoryExceeded
	}

	c.write(key, values)
	c.size = newSize

	return nil
}

// WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) WriteMulti(values map[string][]Value) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	totalSz := 0
	for _, v := range values {
		totalSz += Values(v).Size()
	}

	// Enough room in the cache?
	newSize := c.size + uint64(totalSz)
	if c.maxSize > 0 && newSize+c.snapshotsSize > c.maxSize {
		return ErrCacheMemoryExceeded
	}

	for k, v := range values {
		c.write(k, v)
	}
	c.size = newSize

	return nil
}

// Snapshot will take a snapshot of the current cache, add it to the slice of caches that
// are being flushed, and reset the current cache with new values
func (c *Cache) Snapshot() *Cache {
	c.mu.Lock()
	defer c.mu.Unlock()

	snapshot := NewCache(c.maxSize)
	snapshot.store = c.store
	snapshot.size = c.size

	c.store = make(map[string]*entry)
	c.size = 0

	c.snapshots = append(c.snapshots, snapshot)
	c.snapshotsSize += snapshot.size

	return snapshot
}

// Deduplicate sorts the snapshot before returning it. The compactor and any queries
// coming in while it writes will need the values sorted
func (c *Cache) Deduplicate() {
	for _, e := range c.store {
		e.deduplicate()
	}
}

// ClearSnapshot will remove the snapshot cache from the list of flushing caches and
// adjust the size
func (c *Cache) ClearSnapshot(snapshot *Cache) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for i, cache := range c.snapshots {
		if cache == snapshot {
			c.snapshots = append(c.snapshots[:i], c.snapshots[i+1:]...)
			c.snapshotsSize -= snapshot.size
			break
		}
	}
}

// Size returns the number of point-calcuated bytes the cache currently uses.
func (c *Cache) Size() uint64 {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.size
}

// MaxSize returns the maximum number of bytes the cache may consume.
func (c *Cache) MaxSize() uint64 {
	return c.maxSize
}

// Keys returns a sorted slice of all keys under management by the cache.
func (c *Cache) Keys() []string {
	var a []string
	for k, _ := range c.store {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// Values returns a copy of all values, deduped and sorted, for the given key.
func (c *Cache) Values(key string) Values {
	c.mu.RLock()
	e := c.store[key]
	if e != nil && e.needSort {
		// Sorting is needed, so unlock and run the merge operation with
		// a write-lock. It is actually possible that the data will be
		// sorted by the time the merge runs, which would mean very occasionally
		// a write-lock will be held when only a read-lock is required.
		c.mu.RUnlock()
		return func() Values {
			c.mu.Lock()
			defer c.mu.Unlock()
			return c.merged(key)
		}()
	}

	// No sorting required for key, so just merge while continuing to hold read-lock.
	return func() Values {
		defer c.mu.RUnlock()
		return c.merged(key)
	}()
}

// Delete will remove the keys from the cache
func (c *Cache) Delete(keys []string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for _, k := range keys {
		delete(c.store, k)
	}
}

// merged returns a copy of hot and snapshot values. The copy will be merged, deduped, and
// sorted. It assumes all necessary locks have been taken. If the caller knows that the
// the hot source data for the key will not be changed, it is safe to call this function
// with a read-lock taken. Otherwise it must be called with a write-lock taken.
func (c *Cache) merged(key string) Values {
	e := c.store[key]
	if e == nil {
		if len(c.snapshots) == 0 {
			// No values in hot cache or snapshots.
			return nil
		}
	} else {
		e.deduplicate()
	}

	// Build the sequence of entries that will be returned, in the correct order.
	// Calculate the required size of the destination buffer.
	var entries []*entry
	sz := 0
	for _, s := range c.snapshots {
		e := s.store[key]
		if e != nil {
			entries = append(entries, e)
			sz += len(e.values)
		}
	}
	if e != nil {
		entries = append(entries, e)
		sz += len(e.values)
	}

	// Any entries? If not, return.
	if sz == 0 {
		return nil
	}

	// Create the buffer, and copy all hot values and snapshots. Individual
	// entries are sorted at this point, so now the code has to check if the
	// resultant buffer will be sorted from start to finish.
	var needSort bool
	values := make(Values, sz)
	n := 0
	for _, e := range entries {
		if !needSort && n > 0 {
			needSort = values[n-1].UnixNano() > e.values[0].UnixNano()
		}
		n += copy(values[n:], e.values)
	}

	if needSort {
		values = values.Deduplicate()
	}

	return values
}

// Store returns the underlying cache store. This is not goroutine safe!
// Protect access by using the Lock and Unlock functions on Cache.
func (c *Cache) Store() map[string]*entry {
	return c.store
}

func (c *Cache) Lock() {
	c.mu.Lock()
}

func (c *Cache) Unlock() {
	c.mu.Unlock()
}

// values returns the values for the key. It doesn't lock and assumes the data is
// already sorted. Should only be used in compact.go in the CacheKeyIterator
func (c *Cache) values(key string) Values {
	e := c.store[key]
	if e == nil {
		return nil
	}
	return e.values
}

// write writes the set of values for the key to the cache. This function assumes
// the lock has been taken and does not enforce the cache size limits.
func (c *Cache) write(key string, values []Value) {
	e, ok := c.store[key]
	if !ok {
		e = newEntry()
		c.store[key] = e
	}
	e.add(values)
}

// CacheLoader processes a set of WAL segment files, and loads a cache with the data
// contained within those files.  Processing of the supplied files take place in the
// order they exist in the files slice.
type CacheLoader struct {
	files []string

	Logger *log.Logger
}

// NewCacheLoader returns a new instance of a CacheLoader.
func NewCacheLoader(files []string) *CacheLoader {
	return &CacheLoader{
		files:  files,
		Logger: log.New(os.Stderr, "[cacheloader] ", log.LstdFlags),
	}
}

// Load returns a cache loaded with the data contained within the segment files.
// If, during reading of a segment file, corruption is encountered, that segment
// file is truncated up to and including the last valid byte, and processing
// continues with the next segment file.
func (cl *CacheLoader) Load(cache *Cache) error {
	for _, fn := range cl.files {
		if err := func() error {
			f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0666)
			if err != nil {
				return err
			}

			// Log some information about the segments.
			stat, err := os.Stat(f.Name())
			if err != nil {
				return err
			}
			cl.Logger.Printf("reading file %s, size %d", f.Name(), stat.Size())

			r := NewWALSegmentReader(f)
			defer r.Close()

			for r.Next() {
				entry, err := r.Read()
				if err != nil {
					n := r.Count()
					cl.Logger.Printf("file %s corrupt at position %d, truncating", f.Name(), n)
					if err := f.Truncate(n); err != nil {
						return err
					}
					break
				}

				switch t := entry.(type) {
				case *WriteWALEntry:
					if err := cache.WriteMulti(t.Values); err != nil {
						return err
					}
				case *DeleteWALEntry:
					cache.Delete(t.Keys)
				}
			}

			return nil
		}(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Reduce lock contention in Cache.WriteMulti<commit_after>package tsm1

import (
	"fmt"
	"log"
	"os"
	"sort"
	"sync"
)

var ErrCacheMemoryExceeded = fmt.Errorf("cache maximum memory size exceeded")
var ErrCacheInvalidCheckpoint = fmt.Errorf("invalid checkpoint")

// entry is a set of values and some metadata.
type entry struct {
	values   Values // All stored values.
	needSort bool   // true if the values are out of order and require deduping.
}

// newEntry returns a new instance of entry.
func newEntry() *entry {
	return &entry{}
}

// add adds the given values to the entry.
func (e *entry) add(values []Value) {
	// if there are existing values make sure they're all less than the first of
	// the new values being added
	l := len(e.values)
	if l != 0 {
		lastValTime := e.values[l-1].UnixNano()
		if lastValTime >= values[0].UnixNano() {
			e.needSort = true
		}
	}
	e.values = append(e.values, values...)

	// if there's only one value, we know it's sorted
	if len(values) <= 1 {
		return
	}

	// make sure the new values were in sorted order
	min := values[0].UnixNano()
	for _, v := range values[1:] {
		if min >= v.UnixNano() {
			e.needSort = true
			break
		}
	}
}

// deduplicate sorts and orders the entry's values. If values are already deduped and
// and sorted, the function does no work and simply returns.
func (e *entry) deduplicate() {
	if !e.needSort || len(e.values) == 0 {
		return
	}
	e.values = e.values.Deduplicate()
	e.needSort = false
}

// Cache maintains an in-memory store of Values for a set of keys.
type Cache struct {
	mu      sync.RWMutex
	store   map[string]*entry
	size    uint64
	maxSize uint64

	// snapshots are the cache objects that are currently being written to tsm files
	// they're kept in memory while flushing so they can be queried along with the cache.
	// they are read only and should never be modified
	snapshots     []*Cache
	snapshotsSize uint64
}

// NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory.
func NewCache(maxSize uint64) *Cache {
	return &Cache{
		maxSize: maxSize,
		store:   make(map[string]*entry),
	}
}

// Write writes the set of values for the key to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) Write(key string, values []Value) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Enough room in the cache?
	newSize := c.size + uint64(Values(values).Size())
	if c.maxSize > 0 && newSize+c.snapshotsSize > c.maxSize {
		return ErrCacheMemoryExceeded
	}

	c.write(key, values)
	c.size = newSize

	return nil
}

// WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe.
// It returns an error if the cache has exceeded its max size.
func (c *Cache) WriteMulti(values map[string][]Value) error {
	totalSz := 0
	for _, v := range values {
		totalSz += Values(v).Size()
	}

	// Enough room in the cache?
	c.mu.RLock()
	newSize := c.size + uint64(totalSz)
	if c.maxSize > 0 && newSize+c.snapshotsSize > c.maxSize {
		c.mu.RUnlock()
		return ErrCacheMemoryExceeded
	}
	c.mu.RUnlock()

	c.mu.Lock()
	for k, v := range values {
		c.write(k, v)
	}
	c.size = newSize
	c.mu.Unlock()

	return nil
}

// Snapshot will take a snapshot of the current cache, add it to the slice of caches that
// are being flushed, and reset the current cache with new values
func (c *Cache) Snapshot() *Cache {
	c.mu.Lock()
	defer c.mu.Unlock()

	snapshot := NewCache(c.maxSize)
	snapshot.store = c.store
	snapshot.size = c.size

	c.store = make(map[string]*entry)
	c.size = 0

	c.snapshots = append(c.snapshots, snapshot)
	c.snapshotsSize += snapshot.size

	return snapshot
}

// Deduplicate sorts the snapshot before returning it. The compactor and any queries
// coming in while it writes will need the values sorted
func (c *Cache) Deduplicate() {
	for _, e := range c.store {
		e.deduplicate()
	}
}

// ClearSnapshot will remove the snapshot cache from the list of flushing caches and
// adjust the size
func (c *Cache) ClearSnapshot(snapshot *Cache) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for i, cache := range c.snapshots {
		if cache == snapshot {
			c.snapshots = append(c.snapshots[:i], c.snapshots[i+1:]...)
			c.snapshotsSize -= snapshot.size
			break
		}
	}
}

// Size returns the number of point-calcuated bytes the cache currently uses.
func (c *Cache) Size() uint64 {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.size
}

// MaxSize returns the maximum number of bytes the cache may consume.
func (c *Cache) MaxSize() uint64 {
	return c.maxSize
}

// Keys returns a sorted slice of all keys under management by the cache.
func (c *Cache) Keys() []string {
	var a []string
	for k, _ := range c.store {
		a = append(a, k)
	}
	sort.Strings(a)
	return a
}

// Values returns a copy of all values, deduped and sorted, for the given key.
func (c *Cache) Values(key string) Values {
	c.mu.RLock()
	e := c.store[key]
	if e != nil && e.needSort {
		// Sorting is needed, so unlock and run the merge operation with
		// a write-lock. It is actually possible that the data will be
		// sorted by the time the merge runs, which would mean very occasionally
		// a write-lock will be held when only a read-lock is required.
		c.mu.RUnlock()
		return func() Values {
			c.mu.Lock()
			defer c.mu.Unlock()
			return c.merged(key)
		}()
	}

	// No sorting required for key, so just merge while continuing to hold read-lock.
	return func() Values {
		defer c.mu.RUnlock()
		return c.merged(key)
	}()
}

// Delete will remove the keys from the cache
func (c *Cache) Delete(keys []string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for _, k := range keys {
		delete(c.store, k)
	}
}

// merged returns a copy of hot and snapshot values. The copy will be merged, deduped, and
// sorted. It assumes all necessary locks have been taken. If the caller knows that the
// the hot source data for the key will not be changed, it is safe to call this function
// with a read-lock taken. Otherwise it must be called with a write-lock taken.
func (c *Cache) merged(key string) Values {
	e := c.store[key]
	if e == nil {
		if len(c.snapshots) == 0 {
			// No values in hot cache or snapshots.
			return nil
		}
	} else {
		e.deduplicate()
	}

	// Build the sequence of entries that will be returned, in the correct order.
	// Calculate the required size of the destination buffer.
	var entries []*entry
	sz := 0
	for _, s := range c.snapshots {
		e := s.store[key]
		if e != nil {
			entries = append(entries, e)
			sz += len(e.values)
		}
	}
	if e != nil {
		entries = append(entries, e)
		sz += len(e.values)
	}

	// Any entries? If not, return.
	if sz == 0 {
		return nil
	}

	// Create the buffer, and copy all hot values and snapshots. Individual
	// entries are sorted at this point, so now the code has to check if the
	// resultant buffer will be sorted from start to finish.
	var needSort bool
	values := make(Values, sz)
	n := 0
	for _, e := range entries {
		if !needSort && n > 0 {
			needSort = values[n-1].UnixNano() > e.values[0].UnixNano()
		}
		n += copy(values[n:], e.values)
	}

	if needSort {
		values = values.Deduplicate()
	}

	return values
}

// Store returns the underlying cache store. This is not goroutine safe!
// Protect access by using the Lock and Unlock functions on Cache.
func (c *Cache) Store() map[string]*entry {
	return c.store
}

func (c *Cache) Lock() {
	c.mu.Lock()
}

func (c *Cache) Unlock() {
	c.mu.Unlock()
}

// values returns the values for the key. It doesn't lock and assumes the data is
// already sorted. Should only be used in compact.go in the CacheKeyIterator
func (c *Cache) values(key string) Values {
	e := c.store[key]
	if e == nil {
		return nil
	}
	return e.values
}

// write writes the set of values for the key to the cache. This function assumes
// the lock has been taken and does not enforce the cache size limits.
func (c *Cache) write(key string, values []Value) {
	e, ok := c.store[key]
	if !ok {
		e = newEntry()
		c.store[key] = e
	}
	e.add(values)
}

// CacheLoader processes a set of WAL segment files, and loads a cache with the data
// contained within those files.  Processing of the supplied files take place in the
// order they exist in the files slice.
type CacheLoader struct {
	files []string

	Logger *log.Logger
}

// NewCacheLoader returns a new instance of a CacheLoader.
func NewCacheLoader(files []string) *CacheLoader {
	return &CacheLoader{
		files:  files,
		Logger: log.New(os.Stderr, "[cacheloader] ", log.LstdFlags),
	}
}

// Load returns a cache loaded with the data contained within the segment files.
// If, during reading of a segment file, corruption is encountered, that segment
// file is truncated up to and including the last valid byte, and processing
// continues with the next segment file.
func (cl *CacheLoader) Load(cache *Cache) error {
	for _, fn := range cl.files {
		if err := func() error {
			f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0666)
			if err != nil {
				return err
			}

			// Log some information about the segments.
			stat, err := os.Stat(f.Name())
			if err != nil {
				return err
			}
			cl.Logger.Printf("reading file %s, size %d", f.Name(), stat.Size())

			r := NewWALSegmentReader(f)
			defer r.Close()

			for r.Next() {
				entry, err := r.Read()
				if err != nil {
					n := r.Count()
					cl.Logger.Printf("file %s corrupt at position %d, truncating", f.Name(), n)
					if err := f.Truncate(n); err != nil {
						return err
					}
					break
				}

				switch t := entry.(type) {
				case *WriteWALEntry:
					if err := cache.WriteMulti(t.Values); err != nil {
						return err
					}
				case *DeleteWALEntry:
					cache.Delete(t.Keys)
				}
			}

			return nil
		}(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package consul_test

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"testing"
	"time"

	"golang.org/x/sys/unix"

	consulapi "github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/testutil"
	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/client/vaultclient"
	"github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
)

func testLogger() *log.Logger {
	if testing.Verbose() {
		return log.New(os.Stderr, "", log.LstdFlags)
	}
	return log.New(ioutil.Discard, "", 0)
}

// TestConsul_Integration asserts TaskRunner properly registers and deregisters
// services and checks with Consul using an embedded Consul agent.
func TestConsul_Integration(t *testing.T) {
	if _, ok := driver.BuiltinDrivers["mock_driver"]; !ok {
		t.Skip(`test requires mock_driver; run with "-tags nomad_test"`)
	}
	if testing.Short() {
		t.Skip("-short set; skipping")
	}
	if unix.Geteuid() != 0 {
		t.Skip("Must be run as root")
	}
	// Create an embedded Consul server
	testconsul := testutil.NewTestServerConfig(t, func(c *testutil.TestServerConfig) {
		// If -v wasn't specified squelch consul logging
		if !testing.Verbose() {
			c.Stdout = ioutil.Discard
			c.Stderr = ioutil.Discard
		}
	})
	defer testconsul.Stop()

	conf := config.DefaultConfig()
	conf.ConsulConfig.Addr = testconsul.HTTPAddr
	consulConfig, err := conf.ConsulConfig.ApiConfig()
	if err != nil {
		t.Fatalf("error generating consul config: %v", err)
	}

	conf.StateDir, err = ioutil.TempDir("", "nomadtest-consulstate")
	if err != nil {
		t.Fatalf("error creating temp dir: %v", err)
	}
	defer os.RemoveAll(conf.StateDir)
	conf.AllocDir, err = ioutil.TempDir("", "nomdtest-consulalloc")
	if err != nil {
		t.Fatalf("error creating temp dir: %v", err)
	}
	defer os.RemoveAll(conf.AllocDir)

	alloc := mock.Alloc()
	task := alloc.Job.TaskGroups[0].Tasks[0]
	task.Driver = "mock_driver"
	task.Config = map[string]interface{}{
		"run_for": "1h",
	}
	// Choose a port that shouldn't be in use
	task.Resources.Networks[0].ReservedPorts = []structs.Port{{Label: "http", Value: 3}}
	task.Services = []*structs.Service{
		{
			Name:      "httpd",
			PortLabel: "http",
			Tags:      []string{"nomad", "test", "http"},
			Checks: []*structs.ServiceCheck{
				{
					Name:      "httpd-http-check",
					Type:      "http",
					Path:      "/",
					Protocol:  "http",
					PortLabel: "http",
					Interval:  9000 * time.Hour,
					Timeout:   1, // fail as fast as possible
				},
				{
					Name:     "httpd-script-check",
					Type:     "script",
					Command:  "/bin/true",
					Interval: 10 * time.Second,
					Timeout:  10 * time.Second,
				},
			},
		},
		{
			Name:      "httpd2",
			PortLabel: "http",
			Tags:      []string{"test", "http2"},
		},
	}

	logger := testLogger()
	logUpdate := func(name, state string, event *structs.TaskEvent) {
		logger.Printf("[TEST] test.updater: name=%q state=%q event=%v", name, state, event)
	}
	allocDir := allocdir.NewAllocDir(logger, filepath.Join(conf.AllocDir, alloc.ID))
	if err := allocDir.Build(); err != nil {
		t.Fatalf("error building alloc dir: %v", err)
	}
	taskDir := allocDir.NewTaskDir(task.Name)
	vclient := vaultclient.NewMockVaultClient()
	consulClient, err := consulapi.NewClient(consulConfig)
	if err != nil {
		t.Fatalf("error creating consul client: %v", err)
	}
	serviceClient := consul.NewServiceClient(consulClient.Agent(), true, logger)
	defer serviceClient.Shutdown() // just-in-case cleanup
	consulRan := make(chan struct{})
	go func() {
		serviceClient.Run()
		close(consulRan)
	}()
	tr := client.NewTaskRunner(logger, conf, logUpdate, taskDir, alloc, task, vclient, serviceClient)
	tr.MarkReceived()
	go tr.Run()
	defer func() {
		// Make sure we always shutdown task runner when the test exits
		select {
		case <-tr.WaitCh():
			// Exited cleanly, no need to kill
		default:
			tr.Kill("", "", true) // just in case
		}
	}()

	// Block waiting for the service to appear
	catalog := consulClient.Catalog()
	res, meta, err := catalog.Service("httpd2", "test", nil)
	for i := 0; len(res) == 0 && i < 10; i++ {
		//Expected initial request to fail, do a blocking query
		res, meta, err = catalog.Service("httpd2", "test", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second})
		if err != nil {
			t.Fatalf("error querying for service: %v", err)
		}
	}
	if len(res) != 1 {
		t.Fatalf("expected 1 service but found %d:\n%#v", len(res), res)
	}
	res = res[:]

	// Assert the service with the checks exists
	for i := 0; len(res) == 0 && i < 10; i++ {
		res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second})
		if err != nil {
			t.Fatalf("error querying for service: %v", err)
		}
	}
	if len(res) != 1 {
		t.Fatalf("exepcted 1 service but found %d:\n%#v", len(res), res)
	}

	// Assert the script check passes (mock_driver script checks always
	// pass) after having time to run once
	time.Sleep(2 * time.Second)
	checks, _, err := consulClient.Health().Checks("httpd", nil)
	if err != nil {
		t.Fatalf("error querying checks: %v", err)
	}
	if expected := 2; len(checks) != expected {
		t.Fatalf("expected %d checks but found %d:\n%#v", expected, len(checks), checks)
	}
	for _, check := range checks {
		if expected := "httpd"; check.ServiceName != expected {
			t.Fatalf("expected checks to be for %q but found service name = %q", expected, check.ServiceName)
		}
		switch check.Name {
		case "httpd-http-check":
			// Port check should fail
			if expected := consulapi.HealthCritical; check.Status != expected {
				t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status)
			}
		case "httpd-script-check":
			// mock_driver script checks always succeed
			if expected := consulapi.HealthPassing; check.Status != expected {
				t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status)
			}
		default:
			t.Errorf("unexpected check %q with status %q", check.Name, check.Status)
		}
	}

	logger.Printf("[TEST] consul.test: killing task")

	// Kill the task
	tr.Kill("", "", false)

	select {
	case <-tr.WaitCh():
	case <-time.After(10 * time.Second):
		t.Fatalf("timed out waiting for Run() to exit")
	}

	// Shutdown Consul ServiceClient to ensure all pending operations complete
	if err := serviceClient.Shutdown(); err != nil {
		t.Errorf("error shutting down Consul ServiceClient: %v", err)
	}

	// Ensure Consul is clean
	services, _, err := catalog.Services(nil)
	if err != nil {
		t.Fatalf("error query services: %v", err)
	}
	if len(services) != 1 {
		t.Fatalf("expected only 1 service in Consul but found %d:\n%#v", len(services), services)
	}
	if _, ok := services["consul"]; !ok {
		t.Fatalf(`expected only the "consul" key in Consul but found: %#v`, services)
	}
}
<commit_msg>Fix consul test build on Windows<commit_after>package consul_test

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	consulapi "github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/testutil"
	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/client/vaultclient"
	"github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
)

func testLogger() *log.Logger {
	if testing.Verbose() {
		return log.New(os.Stderr, "", log.LstdFlags)
	}
	return log.New(ioutil.Discard, "", 0)
}

// TestConsul_Integration asserts TaskRunner properly registers and deregisters
// services and checks with Consul using an embedded Consul agent.
func TestConsul_Integration(t *testing.T) {
	if _, ok := driver.BuiltinDrivers["mock_driver"]; !ok {
		t.Skip(`test requires mock_driver; run with "-tags nomad_test"`)
	}
	if testing.Short() {
		t.Skip("-short set; skipping")
	}
	if runtime.GOOS != "windows" {
		if u, err := user.Current(); err == nil && u.Uid != "0" {
			t.Skip("Must be run as root")
		}
	}
	// Create an embedded Consul server
	testconsul := testutil.NewTestServerConfig(t, func(c *testutil.TestServerConfig) {
		// If -v wasn't specified squelch consul logging
		if !testing.Verbose() {
			c.Stdout = ioutil.Discard
			c.Stderr = ioutil.Discard
		}
	})
	defer testconsul.Stop()

	conf := config.DefaultConfig()
	conf.ConsulConfig.Addr = testconsul.HTTPAddr
	consulConfig, err := conf.ConsulConfig.ApiConfig()
	if err != nil {
		t.Fatalf("error generating consul config: %v", err)
	}

	conf.StateDir, err = ioutil.TempDir("", "nomadtest-consulstate")
	if err != nil {
		t.Fatalf("error creating temp dir: %v", err)
	}
	defer os.RemoveAll(conf.StateDir)
	conf.AllocDir, err = ioutil.TempDir("", "nomdtest-consulalloc")
	if err != nil {
		t.Fatalf("error creating temp dir: %v", err)
	}
	defer os.RemoveAll(conf.AllocDir)

	alloc := mock.Alloc()
	task := alloc.Job.TaskGroups[0].Tasks[0]
	task.Driver = "mock_driver"
	task.Config = map[string]interface{}{
		"run_for": "1h",
	}
	// Choose a port that shouldn't be in use
	task.Resources.Networks[0].ReservedPorts = []structs.Port{{Label: "http", Value: 3}}
	task.Services = []*structs.Service{
		{
			Name:      "httpd",
			PortLabel: "http",
			Tags:      []string{"nomad", "test", "http"},
			Checks: []*structs.ServiceCheck{
				{
					Name:      "httpd-http-check",
					Type:      "http",
					Path:      "/",
					Protocol:  "http",
					PortLabel: "http",
					Interval:  9000 * time.Hour,
					Timeout:   1, // fail as fast as possible
				},
				{
					Name:     "httpd-script-check",
					Type:     "script",
					Command:  "/bin/true",
					Interval: 10 * time.Second,
					Timeout:  10 * time.Second,
				},
			},
		},
		{
			Name:      "httpd2",
			PortLabel: "http",
			Tags:      []string{"test", "http2"},
		},
	}

	logger := testLogger()
	logUpdate := func(name, state string, event *structs.TaskEvent) {
		logger.Printf("[TEST] test.updater: name=%q state=%q event=%v", name, state, event)
	}
	allocDir := allocdir.NewAllocDir(logger, filepath.Join(conf.AllocDir, alloc.ID))
	if err := allocDir.Build(); err != nil {
		t.Fatalf("error building alloc dir: %v", err)
	}
	taskDir := allocDir.NewTaskDir(task.Name)
	vclient := vaultclient.NewMockVaultClient()
	consulClient, err := consulapi.NewClient(consulConfig)
	if err != nil {
		t.Fatalf("error creating consul client: %v", err)
	}
	serviceClient := consul.NewServiceClient(consulClient.Agent(), true, logger)
	defer serviceClient.Shutdown() // just-in-case cleanup
	consulRan := make(chan struct{})
	go func() {
		serviceClient.Run()
		close(consulRan)
	}()
	tr := client.NewTaskRunner(logger, conf, logUpdate, taskDir, alloc, task, vclient, serviceClient)
	tr.MarkReceived()
	go tr.Run()
	defer func() {
		// Make sure we always shutdown task runner when the test exits
		select {
		case <-tr.WaitCh():
			// Exited cleanly, no need to kill
		default:
			tr.Kill("", "", true) // just in case
		}
	}()

	// Block waiting for the service to appear
	catalog := consulClient.Catalog()
	res, meta, err := catalog.Service("httpd2", "test", nil)
	for i := 0; len(res) == 0 && i < 10; i++ {
		//Expected initial request to fail, do a blocking query
		res, meta, err = catalog.Service("httpd2", "test", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second})
		if err != nil {
			t.Fatalf("error querying for service: %v", err)
		}
	}
	if len(res) != 1 {
		t.Fatalf("expected 1 service but found %d:\n%#v", len(res), res)
	}
	res = res[:]

	// Assert the service with the checks exists
	for i := 0; len(res) == 0 && i < 10; i++ {
		res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second})
		if err != nil {
			t.Fatalf("error querying for service: %v", err)
		}
	}
	if len(res) != 1 {
		t.Fatalf("exepcted 1 service but found %d:\n%#v", len(res), res)
	}

	// Assert the script check passes (mock_driver script checks always
	// pass) after having time to run once
	time.Sleep(2 * time.Second)
	checks, _, err := consulClient.Health().Checks("httpd", nil)
	if err != nil {
		t.Fatalf("error querying checks: %v", err)
	}
	if expected := 2; len(checks) != expected {
		t.Fatalf("expected %d checks but found %d:\n%#v", expected, len(checks), checks)
	}
	for _, check := range checks {
		if expected := "httpd"; check.ServiceName != expected {
			t.Fatalf("expected checks to be for %q but found service name = %q", expected, check.ServiceName)
		}
		switch check.Name {
		case "httpd-http-check":
			// Port check should fail
			if expected := consulapi.HealthCritical; check.Status != expected {
				t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status)
			}
		case "httpd-script-check":
			// mock_driver script checks always succeed
			if expected := consulapi.HealthPassing; check.Status != expected {
				t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status)
			}
		default:
			t.Errorf("unexpected check %q with status %q", check.Name, check.Status)
		}
	}

	logger.Printf("[TEST] consul.test: killing task")

	// Kill the task
	tr.Kill("", "", false)

	select {
	case <-tr.WaitCh():
	case <-time.After(10 * time.Second):
		t.Fatalf("timed out waiting for Run() to exit")
	}

	// Shutdown Consul ServiceClient to ensure all pending operations complete
	if err := serviceClient.Shutdown(); err != nil {
		t.Errorf("error shutting down Consul ServiceClient: %v", err)
	}

	// Ensure Consul is clean
	services, _, err := catalog.Services(nil)
	if err != nil {
		t.Fatalf("error query services: %v", err)
	}
	if len(services) != 1 {
		t.Fatalf("expected only 1 service in Consul but found %d:\n%#v", len(services), services)
	}
	if _, ok := services["consul"]; !ok {
		t.Fatalf(`expected only the "consul" key in Consul but found: %#v`, services)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package console

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger/loglevel"
	"io"
	"os"
)

const (
	LogLevelDebug = "Debug"
	LogLevelInfo  = "Info"
	LogLevelWarn  = "Warn"
	LogLevelError = "Error"
	LogLevelFatal = "Fatal"
)

func New(loglevel loglevel.LogLevel) *ConsoleLogger {
	return &ConsoleLogger{
		output: os.Stdout,
		level:  loglevel,
	}
}

type ConsoleLogger struct {
	output io.Writer
	level  loglevel.LogLevel
}

func (logger *ConsoleLogger) SetOutput(w io.Writer) {
	logger.output = w
}

func (logger *ConsoleLogger) Debug(format string, v ...interface{}) {
	if logger.level > loglevel.Debug {
		return
	}

	logger.print(LogLevelDebug, format, v)
}

func (logger *ConsoleLogger) Info(format string, v ...interface{}) {
	if logger.level > loglevel.Info {
		return
	}

	logger.print(LogLevelInfo, format, v)
}

func (logger *ConsoleLogger) Warn(format string, v ...interface{}) {
	if logger.level > loglevel.Warn {
		return
	}

	logger.print(LogLevelWarn, format, v)
}

func (logger *ConsoleLogger) Error(format string, v ...interface{}) {
	if logger.level > loglevel.Error {
		return
	}

	logger.print(LogLevelError, format, v)
}

func (logger *ConsoleLogger) Fatal(format string, v ...interface{}) {
	logger.print(LogLevelFatal, format, v)
	os.Exit(1)
}

func (logger *ConsoleLogger) print(level, format string, v ...interface{}) {
	if v != nil {
		fmt.Fprintf(logger.output, fmt.Sprintf("%s: \t%s\n", level, format), v)
	} else {
		fmt.Fprintf(logger.output, fmt.Sprintf("%s: \t%s\n", level, format))
	}
}
<commit_msg>Logger improvement: Handle empty parameter lists<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package console

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger/loglevel"
	"io"
	"os"
)

const (
	LogLevelDebug = "Debug"
	LogLevelInfo  = "Info"
	LogLevelWarn  = "Warn"
	LogLevelError = "Error"
	LogLevelFatal = "Fatal"
)

func New(loglevel loglevel.LogLevel) *ConsoleLogger {
	return &ConsoleLogger{
		output: os.Stdout,
		level:  loglevel,
	}
}

type ConsoleLogger struct {
	output io.Writer
	level  loglevel.LogLevel
}

func (logger *ConsoleLogger) SetOutput(w io.Writer) {
	logger.output = w
}

func (logger *ConsoleLogger) Debug(format string, v ...interface{}) {
	if logger.level > loglevel.Debug {
		return
	}

	logger.print(LogLevelDebug, format, v)
}

func (logger *ConsoleLogger) Info(format string, v ...interface{}) {
	if logger.level > loglevel.Info {
		return
	}

	logger.print(LogLevelInfo, format, v)
}

func (logger *ConsoleLogger) Warn(format string, v ...interface{}) {
	if logger.level > loglevel.Warn {
		return
	}

	logger.print(LogLevelWarn, format, v)
}

func (logger *ConsoleLogger) Error(format string, v ...interface{}) {
	if logger.level > loglevel.Error {
		return
	}

	logger.print(LogLevelError, format, v)
}

func (logger *ConsoleLogger) Fatal(format string, v ...interface{}) {
	logger.print(LogLevelFatal, format, v)
	os.Exit(1)
}

func (logger *ConsoleLogger) print(level, format string, v []interface{}) {
	if len(v) > 0 {
		fmt.Fprintf(logger.output, fmt.Sprintf("%s: \t%s\n", level, format), v)
	} else {
		fmt.Fprintf(logger.output, fmt.Sprintf("%s: \t%s\n", level, format))
	}
}
<|endoftext|>


<commit_before>package ui

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/tarfs"
	"github.com/getlantern/waitforserver"
	"github.com/skratchdot/open-golang/open"
)

const (
	LocalUIDir = "../../../lantern-ui/app"
)

var (
	log = golog.LoggerFor("flashlight.ui")

	l            net.Listener
	fs           *tarfs.FileSystem
	Translations *tarfs.FileSystem
	server       *http.Server
	uiaddr       string

	externalUrl    = "https://www.facebook.com/manototv/" // this string is going to be changed by Makefile
	openedExternal = false
	r              = http.NewServeMux()
)

func init() {
	// Assume the default directory containing UI assets is
	// a sibling directory to this file's directory.
	localResourcesPath := ""
	_, curDir, _, ok := runtime.Caller(1)
	if !ok {
		log.Errorf("Unable to determine caller directory")
	} else {
		localResourcesPath = filepath.Join(curDir, LocalUIDir)
		absLocalResourcesPath, err := filepath.Abs(localResourcesPath)
		if err != nil {
			absLocalResourcesPath = localResourcesPath
		}
		log.Debugf("Creating tarfs filesystem that prefers local resources at %v", absLocalResourcesPath)
	}

	var err error
	fs, err = tarfs.New(Resources, localResourcesPath)
	if err != nil {
		// Panicking here because this shouldn't happen at runtime unless the
		// resources were incorrectly embedded.
		panic(fmt.Errorf("Unable to open tarfs filesystem: %v", err))
	}
	Translations = fs.SubDir("locale")
}

func Handle(p string, handler http.Handler) string {
	r.Handle(p, handler)
	return uiaddr + p
}

func Start(addr string) error {
	var err error
	l, err = net.Listen("tcp", addr)
	if err != nil {
		return fmt.Errorf("Unable to listen at %v: %v", addr, l)
	}

	r.Handle("/", http.FileServer(fs))

	server = &http.Server{
		Handler:  r,
		ErrorLog: log.AsStdLogger(),
	}
	go func() {
		err := server.Serve(l)
		if err != nil {
			log.Errorf("Error serving: %v", err)
		}
	}()
	uiaddr = fmt.Sprintf("http://%v", l.Addr().String())
	log.Debugf("UI available at %v", uiaddr)

	return nil
}

// Show opens the UI in a browser. It will wait for the UI addr come up for at most 3 seconds
func Show() {
	go func() {
		addr, _ := url.Parse(uiaddr)
		if err := waitforserver.WaitForServer("tcp", addr.Host, 3*time.Second); err != nil {
			log.Error(err)
			return
		}
		open.Run(uiaddr)
		if externalUrl != "NO"+"_URL" && !openedExternal {
			time.Sleep(4 * time.Second)
			open.Run(externalUrl)
			openedExternal = true
		}
	}()
}
<commit_msg>Additional error checks and logs #2719<commit_after>package ui

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/tarfs"
	"github.com/getlantern/waitforserver"
	"github.com/skratchdot/open-golang/open"
)

const (
	LocalUIDir = "../../../lantern-ui/app"
)

var (
	log = golog.LoggerFor("flashlight.ui")

	l            net.Listener
	fs           *tarfs.FileSystem
	Translations *tarfs.FileSystem
	server       *http.Server
	uiaddr       string

	externalUrl    = "https://www.facebook.com/manototv/" // this string is going to be changed by Makefile
	openedExternal = false
	r              = http.NewServeMux()
)

func init() {
	// Assume the default directory containing UI assets is
	// a sibling directory to this file's directory.
	localResourcesPath := ""
	_, curDir, _, ok := runtime.Caller(1)
	if !ok {
		log.Errorf("Unable to determine caller directory")
	} else {
		localResourcesPath = filepath.Join(curDir, LocalUIDir)
		absLocalResourcesPath, err := filepath.Abs(localResourcesPath)
		if err != nil {
			absLocalResourcesPath = localResourcesPath
		}
		log.Debugf("Creating tarfs filesystem that prefers local resources at %v", absLocalResourcesPath)
	}

	var err error
	fs, err = tarfs.New(Resources, localResourcesPath)
	if err != nil {
		// Panicking here because this shouldn't happen at runtime unless the
		// resources were incorrectly embedded.
		panic(fmt.Errorf("Unable to open tarfs filesystem: %v", err))
	}
	Translations = fs.SubDir("locale")
}

func Handle(p string, handler http.Handler) string {
	r.Handle(p, handler)
	return uiaddr + p
}

func Start(addr string) error {
	var err error
	l, err = net.Listen("tcp", addr)
	if err != nil {
		return fmt.Errorf("Unable to listen at %v: %v", addr, l)
	}

	r.Handle("/", http.FileServer(fs))

	server = &http.Server{
		Handler:  r,
		ErrorLog: log.AsStdLogger(),
	}
	go func() {
		err := server.Serve(l)
		if err != nil {
			log.Errorf("Error serving: %v", err)
		}
	}()
	uiaddr = fmt.Sprintf("http://%v", l.Addr().String())
	log.Debugf("UI available at %v", uiaddr)

	return nil
}

// Show opens the UI in a browser. It will wait for the UI addr come up for at most 3 seconds
func Show() {
	go func() {
		addr, er := url.Parse(uiaddr)
		if er != nil {
			log.Errorf("Could not parse url `%v` with error `%v`", uiaddr, er)
			return
		}

		if err := waitforserver.WaitForServer("tcp", addr.Host, 10*time.Second); err != nil {
			log.Errorf("Error waiting for server: %v", err)
			return
		}
		open.Run(uiaddr)
		if externalUrl != "NO"+"_URL" && !openedExternal {
			time.Sleep(4 * time.Second)
			open.Run(externalUrl)
			openedExternal = true
		}
	}()
}
<|endoftext|>


<commit_before>package concurrent

import (
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type LockTestObject struct {
	t    *testing.T
	lock *sync.Mutex
	cond *TimeoutCond
}

func NewLockTestObject(t *testing.T) *LockTestObject {
	lock := new(sync.Mutex)
	return &LockTestObject{t: t, lock: lock, cond: NewTimeoutCond(lock)}
}

func (o *LockTestObject) lockAndWaitWithTimeout(timeout time.Duration) (time.Duration, bool) {
	o.lock.Lock()
	defer o.lock.Unlock()
	return o.cond.WaitWithTimeout(timeout)
}

func (o *LockTestObject) lockAndWait() bool {
	o.lock.Lock()
	defer o.lock.Unlock()
	o.t.Log("lockAndWait")
	return o.cond.Wait()
}

func (o *LockTestObject) lockAndSignal() {
	o.lock.Lock()
	defer o.lock.Unlock()
	o.t.Log("lockAndNotify")
	o.cond.Signal()
}

func (o *LockTestObject) lockHasWaiters() bool {
	o.lock.Lock()
	defer o.lock.Unlock()
	return o.cond.HasWaiters()
}

func TestTimeoutCondWait(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWait")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	go func() {
		obj.lockAndWait()
		wait.Done()
	}()
	time.Sleep(50 * time.Millisecond)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
}

func TestTimeoutCondWaitTimeout(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeout")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(1)
	go func() {
		obj.lockAndWaitWithTimeout(2 * time.Second)
		wait.Done()
	}()
	wait.Wait()
}

func TestTimeoutCondWaitTimeoutNotify(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeoutNotify")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	ch := make(chan int, 1)
	timeout := 2000
	go func() {
		begin := currentTimeMillis()
		obj.lockAndWaitWithTimeout(time.Duration(timeout) * time.Millisecond)
		end := currentTimeMillis()
		ch <- int((end - begin))
		wait.Done()
	}()
	sleep(200)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
	time := <-ch
	close(ch)
	assert.True(t, time < timeout)
	assert.True(t, time >= 200)
}

func sleep(millisecond int) {
	time.Sleep(time.Duration(millisecond) * time.Millisecond)
}

func currentTimeMillis() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func TestTimeoutCondWaitTimeoutRemain(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeoutRemain")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	ch := make(chan time.Duration, 1)
	timeout := 2000 * time.Millisecond
	go func() {
		remainTimeout, _ := obj.lockAndWaitWithTimeout(timeout)
		ch <- remainTimeout
		wait.Done()
	}()
	sleep(200)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
	remainTimeout := <-ch
	close(ch)
	assert.True(t, remainTimeout < timeout, "expect remainTimeout %v < %v", remainTimeout, timeout)
	assert.True(t, remainTimeout >= 200*time.Millisecond, "expect remainTimeout %v >= 200 millisecond", remainTimeout)
}

func TestTimeoutCondHasWaiters(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondHasWaiters")
	obj := NewLockTestObject(t)
	waitersCount := 2
	ch := make(chan struct{}, waitersCount)
	for i := 0; i < 2; i++ {
		go func() {
			obj.lockAndWait()
			ch <- struct{}{}
		}()
	}
	time.Sleep(50 * time.Millisecond)
	assert.True(t, obj.lockHasWaiters(), "Should have waiters")

	obj.lockAndSignal()
	<-ch
	assert.True(t, obj.lockHasWaiters(), "Should still have waiters")

	obj.lockAndSignal()
	<-ch
	assert.False(t, obj.lockHasWaiters(), "Should no longer have waiters")
}
func TestInterrupted(t *testing.T) {
	t.Parallel()

	t.Log("TestInterrupted")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	count := 5
	wait.Add(5)
	ch := make(chan bool, 5)
	for i := 0; i < count; i++ {
		go func() {
			ch <- obj.lockAndWait()
			wait.Done()
		}()
	}
	sleep(100)
	go func() { obj.cond.Interrupt() }()
	wait.Wait()
	for i := 0; i < count; i++ {
		b := <-ch
		assert.True(t, b, "expect %v interrupted bug get false", i)
	}
}

func TestInterruptedWithTimeout(t *testing.T) {
	t.Parallel()

	t.Log("TestInterruptedWithTimeout")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	count := 5
	wait.Add(5)
	ch := make(chan bool, 5)
	timeout := 1000 * time.Millisecond
	for i := 0; i < count; i++ {
		go func() {
			_, interrupted := obj.lockAndWaitWithTimeout(timeout)
			ch <- interrupted
			wait.Done()
		}()
	}
	sleep(100)
	go func() { obj.cond.Interrupt() }()
	wait.Wait()
	for i := 0; i < count; i++ {
		b := <-ch
		assert.True(t, b, "expect %v interrupted bug get false", i)
	}
}

func TestSignalNoWait(t *testing.T) {
	t.Parallel()

	obj := NewLockTestObject(t)
	obj.cond.Signal()
}
<commit_msg>Additional tests<commit_after>package concurrent

import (
	"math"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"github.com/stretchr/testify/assert"
)

type LockTestObject struct {
	t    *testing.T
	lock *sync.Mutex
	cond *TimeoutCond
}

func NewLockTestObject(t *testing.T) *LockTestObject {
	lock := new(sync.Mutex)
	return &LockTestObject{t: t, lock: lock, cond: NewTimeoutCond(lock)}
}

func (o *LockTestObject) lockAndWaitWithTimeout(timeout time.Duration) (time.Duration, bool) {
	o.lock.Lock()
	defer o.lock.Unlock()
	return o.cond.WaitWithTimeout(timeout)
}

func (o *LockTestObject) lockAndWait() bool {
	o.lock.Lock()
	defer o.lock.Unlock()
	o.t.Log("lockAndWait")
	return o.cond.Wait()
}

func (o *LockTestObject) lockAndSignal() {
	o.lock.Lock()
	defer o.lock.Unlock()
	o.t.Log("lockAndNotify")
	o.cond.Signal()
}

func (o *LockTestObject) hasWaiters() bool {
	return o.cond.HasWaiters()
}

func TestTimeoutCondWait(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWait")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	go func() {
		obj.lockAndWait()
		wait.Done()
	}()
	time.Sleep(50 * time.Millisecond)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
}

func TestTimeoutCondWaitTimeout(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeout")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(1)
	go func() {
		obj.lockAndWaitWithTimeout(2 * time.Second)
		wait.Done()
	}()
	wait.Wait()
}

func TestTimeoutCondWaitTimeoutNotify(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeoutNotify")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	ch := make(chan int, 1)
	timeout := 2000
	go func() {
		begin := currentTimeMillis()
		obj.lockAndWaitWithTimeout(time.Duration(timeout) * time.Millisecond)
		end := currentTimeMillis()
		ch <- int((end - begin))
		wait.Done()
	}()
	sleep(200)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
	time := <-ch
	close(ch)
	assert.True(t, time < timeout)
	assert.True(t, time >= 200)
}

func sleep(millisecond int) {
	time.Sleep(time.Duration(millisecond) * time.Millisecond)
}

func currentTimeMillis() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func TestTimeoutCondWaitTimeoutRemain(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondWaitTimeoutRemain")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	wait.Add(2)
	ch := make(chan time.Duration, 1)
	timeout := 2000 * time.Millisecond
	go func() {
		remainTimeout, _ := obj.lockAndWaitWithTimeout(timeout)
		ch <- remainTimeout
		wait.Done()
	}()
	sleep(200)
	go func() {
		obj.lockAndSignal()
		wait.Done()
	}()
	wait.Wait()
	remainTimeout := <-ch
	close(ch)
	assert.True(t, remainTimeout < timeout, "expect remainTimeout %v < %v", remainTimeout, timeout)
	assert.True(t, remainTimeout >= 200*time.Millisecond, "expect remainTimeout %v >= 200 millisecond", remainTimeout)
}

func TestTimeoutCondHasWaiters(t *testing.T) {
	t.Parallel()

	t.Log("TestTimeoutCondHasWaiters")
	obj := NewLockTestObject(t)
	waitersCount := 2
	ch := make(chan struct{}, waitersCount)
	for i := 0; i < 2; i++ {
		go func() {
			obj.lockAndWait()
			ch <- struct{}{}
		}()
	}
	time.Sleep(50 * time.Millisecond)
	assert.True(t, obj.hasWaiters(), "Should have waiters")

	obj.lockAndSignal()
	<-ch
	assert.True(t, obj.hasWaiters(), "Should still have waiters")

	obj.lockAndSignal()
	<-ch
	assert.False(t, obj.hasWaiters(), "Should no longer have waiters")
}

func TestTooManyWaiters(t *testing.T) {
	t.Parallel()

	obj := NewLockTestObject(t)
	obj.cond.hasWaiters = math.MaxUint64

	require.Panics(t, func() { obj.lockAndWait() })
}

func TestRemoveWaiterUsedIncorrectly(t *testing.T) {
	t.Parallel()

	cond := NewTimeoutCond(&sync.Mutex{})
	require.Panics(t, cond.removeWaiter)
}

func TestInterrupted(t *testing.T) {
	t.Parallel()

	t.Log("TestInterrupted")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	count := 5
	wait.Add(5)
	ch := make(chan bool, 5)
	for i := 0; i < count; i++ {
		go func() {
			ch <- obj.lockAndWait()
			wait.Done()
		}()
	}
	sleep(100)
	go func() { obj.cond.Interrupt() }()
	wait.Wait()
	for i := 0; i < count; i++ {
		b := <-ch
		assert.True(t, b, "expect %v interrupted bug get false", i)
	}
}

func TestInterruptedWithTimeout(t *testing.T) {
	t.Parallel()

	t.Log("TestInterruptedWithTimeout")
	obj := NewLockTestObject(t)
	wait := sync.WaitGroup{}
	count := 5
	wait.Add(5)
	ch := make(chan bool, 5)
	timeout := 1000 * time.Millisecond
	for i := 0; i < count; i++ {
		go func() {
			_, interrupted := obj.lockAndWaitWithTimeout(timeout)
			ch <- interrupted
			wait.Done()
		}()
	}
	sleep(100)
	go func() { obj.cond.Interrupt() }()
	wait.Wait()
	for i := 0; i < count; i++ {
		b := <-ch
		assert.True(t, b, "expect %v interrupted bug get false", i)
	}
}

func TestSignalNoWait(t *testing.T) {
	t.Parallel()

	obj := NewLockTestObject(t)
	obj.cond.Signal()
}
<|endoftext|>