<commit_before>package main

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

const delimiter string = "::::"

var runecount = utf8.RuneCountInString

func fail(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
func failOn(b bool, message string) {
	if b {
		fail(errors.New(message))
	}
}

type Stack struct {
	s   []string
	dir string
}

func NewStack(capacity int) *Stack {
	s := make([]string, 0, capacity)
	return &Stack{s, ""}
}

func (s *Stack) Push(dirname string) {
	s.s = append(s.s, dirname)
	s.dir = filepath.Join(s.s...)
}

func (s *Stack) Pop() {
	failOn(len(s.s) <= 0, "Invalid directory state. Corrupted database?")
	i := len(s.s) - 1
	s.s = s.s[:i]
	s.dir = filepath.Join(s.s...)
}

func (s Stack) Dir() string {
	return s.dir
}

func keyval(line string) (string, string) {
	i := strings.Index(line, ":")
	if i == -1 || i == len(line)-1 {
		return line, ""
	}
	return line[:i], line[i+2:]
}

type Track struct {
	Album    string
	Artist   string
	Date     string
	Filename string
	Genre    string
	Path     string
	Time     string
	Title    string
}

func (t *Track) Set(key, value string) {
	switch key {
	case "Album":
		t.Album = value
	case "Artist":
		t.Artist = value
	case "Date":
		t.Date = value
	case "Genre":
		t.Genre = value
	case "Time":
		t.Time = formatDurationString(value)
	case "Title":
		t.Title = value
	}
}

func formatDurationString(str string) string {
	duration, _ := time.ParseDuration(str + "s")
	zero := time.Time{}
	format := zero.Add(duration).Format("04:05")
	if duration > time.Hour {
		format = fmt.Sprintf("%d:%s", int(duration.Hours()), format)
	}
	return "(" + format + ")"
}

func spaceBetween(left, right string, maxchars int) string {
	n := maxchars - runecount(left) - runecount(right)
	return left + strings.Repeat(" ", n) + right
}

func withoutExt(path string) string {
	basename := filepath.Base(path)
	return strings.TrimSuffix(basename, filepath.Ext(basename))
}

func truncate(s string, max int, suffix string) string {
	max -= runecount(suffix)
	if max < 0 {
		panic("suffix length greater than max chars")
	}
	trunc := false
	count := 0
	out := make([]rune, 0, max)
	for _, r := range s {
		if count >= max {
			trunc = true
			break
		}
		out = append(out, r)
		count += 1
	}
	result := string(out)
	if trunc {
		result += suffix
	}
	return result
}

func trackFormatter() func(*Track) string {
	cmd := exec.Command("stty", "size")
	cmd.Stdin = os.Stdin
	out, err := cmd.Output()
	fail(err)
	var height, width int
	_, err = fmt.Sscanf(string(out), "%d %d\n", &height, &width)
	fail(err)
	contentLen := width - 5 // remove 5 for fzf display
	return func(t *Track) string {
		str := t.Artist + " - " + t.Title
		str = strings.TrimPrefix(str, " - ")
		if str == "" {
			str = withoutExt(t.Filename)
		}
		if t.Album != "" {
			str += " {" + t.Album + "}"
		}
		str = truncate(str, contentLen-len(t.Time), "..")
		str = spaceBetween(str, t.Time, contentLen)
		str = strings.Replace(str, delimiter, "", -1)
		return str + delimiter + t.Path
	}
}

func groupByArtist(tracks []*Track) []*Track {
	// group by artist, then shuffle to stop same order, but keep artist together
	artists := map[string][]*Track{}
	for _, t := range tracks {
		artists[t.Artist] = append(artists[t.Artist], t)
	}
	shuffled := make([]*Track, len(tracks))
	i := 0
	for _, tracks := range artists {
		for _, t := range tracks {
			shuffled[i] = t
			i += 1
		}
	}
	return shuffled
}

func fzfcmd() *exec.Cmd {
	bind := "--bind=ctrl-k:kill-line,enter:execute(mpd-fzf-play {})"
	fzf := exec.Command("fzf", "--no-hscroll", "--exact", bind)
	fzf.Stderr = os.Stderr
	return fzf
}

func parse(scan *bufio.Scanner) []*Track {

	tracks, track := []*Track{}, new(Track)
	dirstack := NewStack(256)

	for scan.Scan() {
		key, value := keyval(scan.Text())
		switch key {
		case "directory":
			dirstack.Push(value)
		case "end":
			dirstack.Pop()
		case "Artist", "Album", "Date", "Genre", "Time", "Title":
			track.Set(key, value)
		case "song_begin":
			track.Filename = value
			track.Path = filepath.Join(dirstack.Dir(), track.Filename)
		case "song_end":
			tracks = append(tracks, track)
			track = new(Track)
		}
	}
	fail(scan.Err())
	return tracks
}

func expandUser(path, home string) string {
	if path[:2] == "~/" {
		path = strings.Replace(path, "~", home, 1)
	}
	return path
}

func findDbFile() string {
	usr, err := user.Current()
	fail(err)
	home := usr.HomeDir
	paths := []string{
		filepath.Join(os.Getenv("XDG_CONFIG_HOME"), "/mpd/mpd.conf"),
		filepath.Join(home, ".config", "/mpd/mpd.conf"),
		filepath.Join(home, ".mpdconf"),
		"/etc/mpd.conf",
	}
	var f *os.File
	var confpath string
	for _, path := range paths {
		f, err = os.Open(path)
		if err == nil {
			confpath = path
			break
		}
	}
	failOn(f == nil, "No config file found")

	expDb := regexp.MustCompile(`^\s*db_file\s*"([^"]+)"`)
	scan := bufio.NewScanner(f)
	var dbFile string
	for scan.Scan() {
		m := expDb.FindStringSubmatch(scan.Text())
		if m != nil {
			dbFile = expandUser(m[1], home)
		}
	}
	fail(scan.Err())
	fail(f.Close())
	failOn(dbFile == "", fmt.Sprintf("Could not find 'db_file' in configuration file '%s'", confpath))
	return dbFile
}

func main() {
	dbFile := findDbFile()
	format := trackFormatter()

	f, err := os.Open(dbFile)
	fail(err)
	gz, err := gzip.NewReader(f)
	fail(err)

	scan := bufio.NewScanner(gz)
	tracks := groupByArtist(parse(scan))

	fail(f.Close())
	fail(gz.Close())

	fzf := fzfcmd()
	in, _ := fzf.StdinPipe()
	fail(fzf.Start())
	for _, t := range tracks {
		fmt.Fprintln(in, format(t))
	}
	fail(in.Close())
	fzf.Wait()
}
<commit_msg>Return empty string on failed time parse<commit_after>package main

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

const delimiter string = "::::"

var runecount = utf8.RuneCountInString

func fail(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
func failOn(b bool, message string) {
	if b {
		fail(errors.New(message))
	}
}

type Stack struct {
	s   []string
	dir string
}

func NewStack(capacity int) *Stack {
	s := make([]string, 0, capacity)
	return &Stack{s, ""}
}

func (s *Stack) Push(dirname string) {
	s.s = append(s.s, dirname)
	s.dir = filepath.Join(s.s...)
}

func (s *Stack) Pop() {
	failOn(len(s.s) <= 0, "Invalid directory state. Corrupted database?")
	i := len(s.s) - 1
	s.s = s.s[:i]
	s.dir = filepath.Join(s.s...)
}

func (s Stack) Dir() string {
	return s.dir
}

func keyval(line string) (string, string) {
	i := strings.Index(line, ":")
	if i == -1 || i == len(line)-1 {
		return line, ""
	}
	return line[:i], line[i+2:]
}

type Track struct {
	Album    string
	Artist   string
	Date     string
	Filename string
	Genre    string
	Path     string
	Time     string
	Title    string
}

func (t *Track) Set(key, value string) {
	switch key {
	case "Album":
		t.Album = value
	case "Artist":
		t.Artist = value
	case "Date":
		t.Date = value
	case "Genre":
		t.Genre = value
	case "Time":
		t.Time = formatDurationString(value)
	case "Title":
		t.Title = value
	}
}

func formatDurationString(str string) string {
	duration, err := time.ParseDuration(str + "s")
	if err != nil {
		return ""
	}
	zero := time.Time{}
	format := zero.Add(duration).Format("04:05")
	if duration > time.Hour {
		format = fmt.Sprintf("%d:%s", int(duration.Hours()), format)
	}
	return "(" + format + ")"
}

func spaceBetween(left, right string, maxchars int) string {
	n := maxchars - runecount(left) - runecount(right)
	return left + strings.Repeat(" ", n) + right
}

func withoutExt(path string) string {
	basename := filepath.Base(path)
	return strings.TrimSuffix(basename, filepath.Ext(basename))
}

func truncate(s string, max int, suffix string) string {
	max -= runecount(suffix)
	if max < 0 {
		panic("suffix length greater than max chars")
	}
	trunc := false
	count := 0
	out := make([]rune, 0, max)
	for _, r := range s {
		if count >= max {
			trunc = true
			break
		}
		out = append(out, r)
		count += 1
	}
	result := string(out)
	if trunc {
		result += suffix
	}
	return result
}

func trackFormatter() func(*Track) string {
	cmd := exec.Command("stty", "size")
	cmd.Stdin = os.Stdin
	out, err := cmd.Output()
	fail(err)
	var height, width int
	_, err = fmt.Sscanf(string(out), "%d %d\n", &height, &width)
	fail(err)
	contentLen := width - 5 // remove 5 for fzf display
	return func(t *Track) string {
		str := t.Artist + " - " + t.Title
		str = strings.TrimPrefix(str, " - ")
		if str == "" {
			str = withoutExt(t.Filename)
		}
		if t.Album != "" {
			str += " {" + t.Album + "}"
		}
		str = truncate(str, contentLen-len(t.Time), "..")
		str = spaceBetween(str, t.Time, contentLen)
		str = strings.Replace(str, delimiter, "", -1)
		return str + delimiter + t.Path
	}
}

func groupByArtist(tracks []*Track) []*Track {
	// group by artist, then shuffle to stop same order, but keep artist together
	artists := map[string][]*Track{}
	for _, t := range tracks {
		artists[t.Artist] = append(artists[t.Artist], t)
	}
	shuffled := make([]*Track, len(tracks))
	i := 0
	for _, tracks := range artists {
		for _, t := range tracks {
			shuffled[i] = t
			i += 1
		}
	}
	return shuffled
}

func fzfcmd() *exec.Cmd {
	bind := "--bind=ctrl-k:kill-line,enter:execute(mpd-fzf-play {})"
	fzf := exec.Command("fzf", "--no-hscroll", "--exact", bind)
	fzf.Stderr = os.Stderr
	return fzf
}

func parse(scan *bufio.Scanner) []*Track {

	tracks, track := []*Track{}, new(Track)
	dirstack := NewStack(256)

	for scan.Scan() {
		key, value := keyval(scan.Text())
		switch key {
		case "directory":
			dirstack.Push(value)
		case "end":
			dirstack.Pop()
		case "Artist", "Album", "Date", "Genre", "Time", "Title":
			track.Set(key, value)
		case "song_begin":
			track.Filename = value
			track.Path = filepath.Join(dirstack.Dir(), track.Filename)
		case "song_end":
			tracks = append(tracks, track)
			track = new(Track)
		}
	}
	fail(scan.Err())
	return tracks
}

func expandUser(path, home string) string {
	if path[:2] == "~/" {
		path = strings.Replace(path, "~", home, 1)
	}
	return path
}

func findDbFile() string {
	usr, err := user.Current()
	fail(err)
	home := usr.HomeDir
	paths := []string{
		filepath.Join(os.Getenv("XDG_CONFIG_HOME"), "/mpd/mpd.conf"),
		filepath.Join(home, ".config", "/mpd/mpd.conf"),
		filepath.Join(home, ".mpdconf"),
		"/etc/mpd.conf",
	}
	var f *os.File
	var confpath string
	for _, path := range paths {
		f, err = os.Open(path)
		if err == nil {
			confpath = path
			break
		}
	}
	failOn(f == nil, "No config file found")

	expDb := regexp.MustCompile(`^\s*db_file\s*"([^"]+)"`)
	scan := bufio.NewScanner(f)
	var dbFile string
	for scan.Scan() {
		m := expDb.FindStringSubmatch(scan.Text())
		if m != nil {
			dbFile = expandUser(m[1], home)
		}
	}
	fail(scan.Err())
	fail(f.Close())
	failOn(dbFile == "", fmt.Sprintf("Could not find 'db_file' in configuration file '%s'", confpath))
	return dbFile
}

func main() {
	dbFile := findDbFile()
	format := trackFormatter()

	f, err := os.Open(dbFile)
	fail(err)
	gz, err := gzip.NewReader(f)
	fail(err)

	scan := bufio.NewScanner(gz)
	tracks := groupByArtist(parse(scan))

	fail(f.Close())
	fail(gz.Close())

	fzf := fzfcmd()
	in, _ := fzf.StdinPipe()
	fail(fzf.Start())
	for _, t := range tracks {
		fmt.Fprintln(in, format(t))
	}
	fail(in.Close())
	fzf.Wait()
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

// Package nodetable implements high performance GC optimized Node lookup table
// for Nitro index storage. This table is not thread-safe
//
// Golang map does not need to pay the cost of GC scans if you have native
// fixed size types for both key and value. We use two tables for the node
// lookup table implementation. Fast table and slow table. Fast table stores
// maps crc32(key) to a uint64. Value is a pointer to a skiplist node. Highest
// bit is used to indicate whether there is any hash collision for the crc32
// key used. If the bit is set, that means we need to lookup second table,
// which is the slow table. Slow table has multiple entries which are mapped
// by the same crc32 key.
package nodetable

import "unsafe"
import "fmt"
import "github.com/t3rm1n4l/nitro/skiplist"

var emptyResult ntResult

const approxItemSize = 42

var dbInstances *skiplist.Skiplist

func init() {
	dbInstances = skiplist.New()
}

// EqualKeyFn implements key equality check
type EqualKeyFn func(unsafe.Pointer, []byte) bool

// HashFn implements 32bit hash function on a string
type HashFn func([]byte) uint32

// NodeTable describes lookup table
type NodeTable struct {
	fastHT      map[uint32]uint64
	slowHT      map[uint32][]uint64
	fastHTCount uint64
	slowHTCount uint64
	conflicts   uint64

	hash     HashFn
	keyEqual EqualKeyFn

	res ntResult
}

// CompareNodeTable implements comparator for nodetable instances
func CompareNodeTable(a, b unsafe.Pointer) int {
	return int(uintptr(a)) - int(uintptr(b))
}

const (
	ntNotFound    = 0x00
	ntFoundInFast = 0x01
	ntFoundInSlow = 0x03
	ntFoundMask   = 0x01
)

type ntResult struct {
	status         int
	hash           uint32
	hasConflict    bool
	fastHTHasEntry bool
	fastHTValue    uint64
	slowHTValues   []uint64
	slowHTPos      int
}

// New creates a nodetable instance
func New(hfn HashFn, kfn EqualKeyFn) *NodeTable {
	nt := &NodeTable{
		fastHT:   make(map[uint32]uint64),
		slowHT:   make(map[uint32][]uint64),
		hash:     hfn,
		keyEqual: kfn,
	}

	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	dbInstances.Insert(unsafe.Pointer(nt), CompareNodeTable, buf, &dbInstances.Stats)

	return nt
}

// Stats returns nodetable statistics
func (nt *NodeTable) Stats() string {
	return fmt.Sprintf("\nFastHTCount = %d\n"+
		"SlowHTCount = %d\n"+
		"Conflicts   = %d\n"+
		"MemoryInUse = %d\n",
		nt.fastHTCount, nt.slowHTCount, nt.conflicts, nt.MemoryInUse())
}

// MemoryInUse returns memory used by nodetable instance
func (nt *NodeTable) MemoryInUse() int64 {
	return int64(approxItemSize * (nt.fastHTCount + nt.slowHTCount))
}

// Get returns node pointer for the lookup key
func (nt *NodeTable) Get(key []byte) unsafe.Pointer {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		if res.status == ntFoundInFast {
			return decodePointer(res.fastHTValue)
		}
		return decodePointer(res.slowHTValues[res.slowHTPos])
	}

	return nil
}

// Update inserts or replaces an existing entry
func (nt *NodeTable) Update(key []byte, nptr unsafe.Pointer) (updated bool, oldPtr unsafe.Pointer) {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		// Found key, replace old pointer value with new one
		updated = true
		if res.status == ntFoundInFast {
			oldPtr = decodePointer(res.fastHTValue)
			nt.fastHT[res.hash] = encodePointer(nptr, res.hasConflict)
		} else {
			oldPtr = decodePointer(res.slowHTValues[res.slowHTPos])
			res.slowHTValues[res.slowHTPos] = encodePointer(nptr, true)
		}
	} else {
		// Insert new key
		updated = false
		newSlowValue := res.fastHTHasEntry && !res.hasConflict
		// Key needs to be inserted into slowHT
		if res.hasConflict || newSlowValue {
			slowHTValues := nt.slowHT[res.hash]
			slowHTValues = append(slowHTValues, encodePointer(nptr, false))
			nt.slowHT[res.hash] = slowHTValues
			// There is an entry already in the fastHT for same crc32 hash
			// We have inserted first entry into the slowHT. Now mark conflict bit.
			if newSlowValue {
				nt.fastHT[res.hash] = encodePointer(decodePointer(nt.fastHT[res.hash]), true)
				nt.conflicts++
			}
			nt.slowHTCount++
		} else {
			// Insert new item into fastHT
			nt.fastHT[res.hash] = encodePointer(nptr, false)
			nt.fastHTCount++
		}
	}

	return
}

// Remove an item from the nodetable
func (nt *NodeTable) Remove(key []byte) (success bool, nptr unsafe.Pointer) {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		success = true
		if res.status == ntFoundInFast {
			nptr = decodePointer(res.fastHTValue)
			// Key needs to be removed from fastHT. For that we need to move
			// an item present in slowHT and overwrite fastHT entry.
			if res.hasConflict {
				slowHTValues := nt.slowHT[res.hash]
				v := slowHTValues[0] // New fastHT candidate
				slowHTValues = append([]uint64(nil), slowHTValues[1:]...)
				nt.slowHTCount--

				var conflict bool
				if len(slowHTValues) == 0 {
					delete(nt.slowHT, res.hash)
					nt.conflicts--
				} else {
					conflict = true
					nt.slowHT[res.hash] = slowHTValues
				}

				nt.fastHT[res.hash] = encodePointer(decodePointer(v), conflict)
			} else {
				delete(nt.fastHT, res.hash)
				nt.fastHTCount--
			}
		} else {
			nptr = decodePointer(res.slowHTValues[res.slowHTPos])
			// Remove key from slowHT
			newSlowValue := append([]uint64(nil), res.slowHTValues[:res.slowHTPos]...)
			if res.slowHTPos+1 != len(res.slowHTValues) {
				newSlowValue = append(newSlowValue, res.slowHTValues[:res.slowHTPos+1]...)
			}
			nt.slowHTCount--

			if len(newSlowValue) == 0 {
				delete(nt.slowHT, res.hash)
				nt.fastHT[res.hash] = encodePointer(decodePointer(nt.fastHT[res.hash]), false)
				nt.conflicts--
			}
		}
	}
	return
}

func decodePointer(v uint64) unsafe.Pointer {
	var x uintptr
	if unsafe.Sizeof(x) == 8 {
		ptr := uintptr(v & ^(uint64(1) << 63))
		return unsafe.Pointer(ptr)
	}
	return unsafe.Pointer(uintptr(v & 0xffffffff))
}

func encodePointer(p unsafe.Pointer, hasConflict bool) uint64 {
	v := uint64(uintptr(p))
	if hasConflict {
		v |= 1 << 63
	}

	return v
}

func (nt *NodeTable) hasConflict(v uint64) bool {
	return v>>63 == 1
}

func (nt *NodeTable) isEqual(key []byte, v uint64) bool {
	p := decodePointer(v)
	return nt.keyEqual(p, key)
}

func (nt *NodeTable) find(key []byte) (res *ntResult) {
	nt.res = emptyResult
	res = &nt.res
	res.status = ntNotFound
	h := nt.hash(key)
	res.hash = h

	v, ok := nt.fastHT[h]
	res.fastHTHasEntry = ok
	if ok {
		res.hasConflict = nt.hasConflict(v)
		if nt.isEqual(key, v) {
			res.status = ntFoundInFast
			res.fastHTValue = v
			return
		}

		if res.hasConflict {
			if vs, ok := nt.slowHT[h]; ok {
				for i, v := range vs {
					if nt.isEqual(key, v) {
						res.slowHTPos = i
						res.slowHTValues = vs
						res.status = ntFoundInSlow
						return
					}
				}
			}
		}
	}

	return
}

// Close destroys the nodetable
func (nt *NodeTable) Close() {
	nt.fastHTCount = 0
	nt.slowHTCount = 0
	nt.conflicts = 0
	nt.fastHT = make(map[uint32]uint64)
	nt.slowHT = make(map[uint32][]uint64)

	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	dbInstances.Delete(unsafe.Pointer(nt), CompareNodeTable, buf, &dbInstances.Stats)
}

// MemoryInUse returns total memory used by nodetables in a process
func MemoryInUse() (sz int64) {
	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	iter := dbInstances.NewIterator(CompareNodeTable, buf)
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
		db := (*NodeTable)(iter.Get())
		sz += db.MemoryInUse()
	}

	return
}
<commit_msg>MB-20141 nitro/nodetable: Fix bug in remove operation using slowHT<commit_after>// Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

// Package nodetable implements high performance GC optimized Node lookup table
// for Nitro index storage. This table is not thread-safe
//
// Golang map does not need to pay the cost of GC scans if you have native
// fixed size types for both key and value. We use two tables for the node
// lookup table implementation. Fast table and slow table. Fast table stores
// maps crc32(key) to a uint64. Value is a pointer to a skiplist node. Highest
// bit is used to indicate whether there is any hash collision for the crc32
// key used. If the bit is set, that means we need to lookup second table,
// which is the slow table. Slow table has multiple entries which are mapped
// by the same crc32 key.
package nodetable

import "unsafe"
import "fmt"
import "github.com/t3rm1n4l/nitro/skiplist"

var emptyResult ntResult

const approxItemSize = 42

var dbInstances *skiplist.Skiplist

func init() {
	dbInstances = skiplist.New()
}

// EqualKeyFn implements key equality check
type EqualKeyFn func(unsafe.Pointer, []byte) bool

// HashFn implements 32bit hash function on a string
type HashFn func([]byte) uint32

// NodeTable describes lookup table
type NodeTable struct {
	fastHT      map[uint32]uint64
	slowHT      map[uint32][]uint64
	fastHTCount uint64
	slowHTCount uint64
	conflicts   uint64

	hash     HashFn
	keyEqual EqualKeyFn

	res ntResult
}

// CompareNodeTable implements comparator for nodetable instances
func CompareNodeTable(a, b unsafe.Pointer) int {
	return int(uintptr(a)) - int(uintptr(b))
}

const (
	ntNotFound    = 0x00
	ntFoundInFast = 0x01
	ntFoundInSlow = 0x03
	ntFoundMask   = 0x01
)

type ntResult struct {
	status         int
	hash           uint32
	hasConflict    bool
	fastHTHasEntry bool
	fastHTValue    uint64
	slowHTValues   []uint64
	slowHTPos      int
}

// New creates a nodetable instance
func New(hfn HashFn, kfn EqualKeyFn) *NodeTable {
	nt := &NodeTable{
		fastHT:   make(map[uint32]uint64),
		slowHT:   make(map[uint32][]uint64),
		hash:     hfn,
		keyEqual: kfn,
	}

	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	dbInstances.Insert(unsafe.Pointer(nt), CompareNodeTable, buf, &dbInstances.Stats)

	return nt
}

// Stats returns nodetable statistics
func (nt *NodeTable) Stats() string {
	return fmt.Sprintf("\nFastHTCount = %d\n"+
		"SlowHTCount = %d\n"+
		"Conflicts   = %d\n"+
		"MemoryInUse = %d\n",
		nt.fastHTCount, nt.slowHTCount, nt.conflicts, nt.MemoryInUse())
}

// MemoryInUse returns memory used by nodetable instance
func (nt *NodeTable) MemoryInUse() int64 {
	return int64(approxItemSize * (nt.fastHTCount + nt.slowHTCount))
}

// Get returns node pointer for the lookup key
func (nt *NodeTable) Get(key []byte) unsafe.Pointer {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		if res.status == ntFoundInFast {
			return decodePointer(res.fastHTValue)
		}
		return decodePointer(res.slowHTValues[res.slowHTPos])
	}

	return nil
}

// Update inserts or replaces an existing entry
func (nt *NodeTable) Update(key []byte, nptr unsafe.Pointer) (updated bool, oldPtr unsafe.Pointer) {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		// Found key, replace old pointer value with new one
		updated = true
		if res.status == ntFoundInFast {
			oldPtr = decodePointer(res.fastHTValue)
			nt.fastHT[res.hash] = encodePointer(nptr, res.hasConflict)
		} else {
			oldPtr = decodePointer(res.slowHTValues[res.slowHTPos])
			res.slowHTValues[res.slowHTPos] = encodePointer(nptr, true)
		}
	} else {
		// Insert new key
		updated = false
		newSlowValue := res.fastHTHasEntry && !res.hasConflict
		// Key needs to be inserted into slowHT
		if res.hasConflict || newSlowValue {
			slowHTValues := nt.slowHT[res.hash]
			slowHTValues = append(slowHTValues, encodePointer(nptr, false))
			nt.slowHT[res.hash] = slowHTValues
			// There is an entry already in the fastHT for same crc32 hash
			// We have inserted first entry into the slowHT. Now mark conflict bit.
			if newSlowValue {
				nt.fastHT[res.hash] = encodePointer(decodePointer(nt.fastHT[res.hash]), true)
				nt.conflicts++
			}
			nt.slowHTCount++
		} else {
			// Insert new item into fastHT
			nt.fastHT[res.hash] = encodePointer(nptr, false)
			nt.fastHTCount++
		}
	}

	return
}

// Remove an item from the nodetable
func (nt *NodeTable) Remove(key []byte) (success bool, nptr unsafe.Pointer) {
	res := nt.find(key)
	if res.status&ntFoundMask == ntFoundMask {
		success = true
		if res.status == ntFoundInFast {
			nptr = decodePointer(res.fastHTValue)
			// Key needs to be removed from fastHT. For that we need to move
			// an item present in slowHT and overwrite fastHT entry.
			if res.hasConflict {
				slowHTValues := nt.slowHT[res.hash]
				v := slowHTValues[0] // New fastHT candidate
				slowHTValues = append([]uint64(nil), slowHTValues[1:]...)
				nt.slowHTCount--

				var conflict bool
				if len(slowHTValues) == 0 {
					delete(nt.slowHT, res.hash)
					nt.conflicts--
				} else {
					conflict = true
					nt.slowHT[res.hash] = slowHTValues
				}

				nt.fastHT[res.hash] = encodePointer(decodePointer(v), conflict)
			} else {
				delete(nt.fastHT, res.hash)
				nt.fastHTCount--
			}
		} else {
			nptr = decodePointer(res.slowHTValues[res.slowHTPos])
			// Remove key from slowHT
			newSlowValue := append([]uint64(nil), res.slowHTValues[:res.slowHTPos]...)
			if res.slowHTPos+1 != len(res.slowHTValues) {
				newSlowValue = append(newSlowValue, res.slowHTValues[res.slowHTPos+1:]...)
			}
			nt.slowHTCount--

			if len(newSlowValue) == 0 {
				delete(nt.slowHT, res.hash)
				nt.fastHT[res.hash] = encodePointer(decodePointer(nt.fastHT[res.hash]), false)
				nt.conflicts--
			} else {
				nt.slowHT[res.hash] = newSlowValue
			}
		}
	}
	return
}

func decodePointer(v uint64) unsafe.Pointer {
	var x uintptr
	if unsafe.Sizeof(x) == 8 {
		ptr := uintptr(v & ^(uint64(1) << 63))
		return unsafe.Pointer(ptr)
	}
	return unsafe.Pointer(uintptr(v & 0xffffffff))
}

func encodePointer(p unsafe.Pointer, hasConflict bool) uint64 {
	v := uint64(uintptr(p))
	if hasConflict {
		v |= 1 << 63
	}

	return v
}

func (nt *NodeTable) hasConflict(v uint64) bool {
	return v>>63 == 1
}

func (nt *NodeTable) isEqual(key []byte, v uint64) bool {
	p := decodePointer(v)
	return nt.keyEqual(p, key)
}

func (nt *NodeTable) find(key []byte) (res *ntResult) {
	nt.res = emptyResult
	res = &nt.res
	res.status = ntNotFound
	h := nt.hash(key)
	res.hash = h

	v, ok := nt.fastHT[h]
	res.fastHTHasEntry = ok
	if ok {
		res.hasConflict = nt.hasConflict(v)
		if nt.isEqual(key, v) {
			res.status = ntFoundInFast
			res.fastHTValue = v
			return
		}

		if res.hasConflict {
			if vs, ok := nt.slowHT[h]; ok {
				for i, v := range vs {
					if nt.isEqual(key, v) {
						res.slowHTPos = i
						res.slowHTValues = vs
						res.status = ntFoundInSlow
						return
					}
				}
			}
		}
	}

	return
}

// Close destroys the nodetable
func (nt *NodeTable) Close() {
	nt.fastHTCount = 0
	nt.slowHTCount = 0
	nt.conflicts = 0
	nt.fastHT = make(map[uint32]uint64)
	nt.slowHT = make(map[uint32][]uint64)

	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	dbInstances.Delete(unsafe.Pointer(nt), CompareNodeTable, buf, &dbInstances.Stats)
}

// MemoryInUse returns total memory used by nodetables in a process
func MemoryInUse() (sz int64) {
	buf := dbInstances.MakeBuf()
	defer dbInstances.FreeBuf(buf)
	iter := dbInstances.NewIterator(CompareNodeTable, buf)
	for iter.SeekFirst(); iter.Valid(); iter.Next() {
		db := (*NodeTable)(iter.Get())
		sz += db.MemoryInUse()
	}

	return
}
<|endoftext|>


<commit_before>package siri

import (
	"bytes"
	"log"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/jbowtie/gokogiri"
	"github.com/jbowtie/gokogiri/xml"
)

type XMLCheckStatusRequest struct {
	node *xml.XmlNode

	messageIdentifier string
	requestorRef      string
	requestTimestamp  time.Time
}

type SIRICheckStatusRequest struct {
	MessageIdentifier string
	RequestorRef      string
	RequestTimestamp  time.Time
}

const SIRIRequestTemplate = `<ns7:CheckStatus xmlns:ns2="http://www.siri.org.uk/siri" xmlns:ns3="http://www.ifopt.org.uk/acsb" xmlns:ns4="http://www.ifopt.org.uk/ifopt" xmlns:ns5="http://datex2.eu/schema/2_0RC1/2_0" xmlns:ns6="http://scma/siri" xmlns:ns7="http://wsdl.siri.org.uk">
	<Request>
		<ns2:RequestTimestamp>{{.RequestTimestamp.Format "2006-01-02T15:04:05.000Z07:00"}}</ns2:RequestTimestamp>
		<ns2:RequestorRef>{{.RequestorRef}}</ns2:RequestorRef>
		<ns2:MessageIdentifier>{{.MessageIdentifier}}</ns2:MessageIdentifier>
	</Request>
	<RequestExtension/>
</ns7:CheckStatus>`

func NewXMLCheckStatusRequest(node *xml.XmlNode) *XMLCheckStatusRequest {
	return &XMLCheckStatusRequest{node: node}
}

func NewXMLCheckStatusRequestFromContent(content []byte) (*XMLCheckStatusRequest, error) {
	doc, err := gokogiri.ParseXml(content)
	if err != nil {
		return nil, err
	}
	request := NewXMLCheckStatusRequest(doc.Root().XmlNode)
	finalizer := func(request *XMLCheckStatusRequest) {
		doc.Free()
	}
	runtime.SetFinalizer(request, finalizer)
	return request, nil
}

func NewSIRICheckStatusRequest(RequestorRef string, RequestTimestamp time.Time, MessageIdentifier string) *SIRICheckStatusRequest {
	return &SIRICheckStatusRequest{RequestorRef: RequestorRef, RequestTimestamp: RequestTimestamp, MessageIdentifier: MessageIdentifier}
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) RequestorRef() string {
	if request.requestorRef == "" {
		nodes, _ := request.node.Search("//*[local-name()='RequestorRef']")
		request.requestorRef = strings.TrimSpace(nodes[0].Content())
	}
	return request.requestorRef
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) RequestTimestamp() time.Time {
	if request.requestTimestamp.IsZero() {
		nodes, _ := request.node.Search("//*[local-name()='RequestTimestamp']")
		t, _ := time.Parse("2006-01-02T15:04:05.000Z07:00", strings.TrimSpace(nodes[0].Content()))
		request.requestTimestamp = t
	}
	return request.requestTimestamp
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) MessageIdentifier() string {
	if request.messageIdentifier == "" {
		nodes, _ := request.node.Search("//*[local-name()='MessageIdentifier']")
		request.messageIdentifier = strings.TrimSpace(nodes[0].Content())
	}
	return request.messageIdentifier
}

// TODO : Handle errors
func (request *SIRICheckStatusRequest) BuildXML() string {
	var buffer bytes.Buffer
	var siriRequest = template.Must(template.New("siriRequest").Parse(SIRIRequestTemplate))
	if err := siriRequest.Execute(&buffer, request); err != nil {
		log.Fatal(err)
	}
	return buffer.String()
}
<commit_msg>Fixe RequestTimestamp (according to xml sample). Refs #1629<commit_after>package siri

import (
	"bytes"
	"log"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/jbowtie/gokogiri"
	"github.com/jbowtie/gokogiri/xml"
)

type XMLCheckStatusRequest struct {
	node *xml.XmlNode

	messageIdentifier string
	requestorRef      string
	requestTimestamp  time.Time
}

type SIRICheckStatusRequest struct {
	MessageIdentifier string
	RequestorRef      string
	RequestTimestamp  time.Time
}

const SIRIRequestTemplate = `<ns7:CheckStatus xmlns:ns2="http://www.siri.org.uk/siri" xmlns:ns3="http://www.ifopt.org.uk/acsb" xmlns:ns4="http://www.ifopt.org.uk/ifopt" xmlns:ns5="http://datex2.eu/schema/2_0RC1/2_0" xmlns:ns6="http://scma/siri" xmlns:ns7="http://wsdl.siri.org.uk">
	<Request>
		<ns2:RequestTimestamp>{{.RequestTimestamp.Format "2006-01-02T15:04:05.000Z"}}</ns2:RequestTimestamp>
		<ns2:RequestorRef>{{.RequestorRef}}</ns2:RequestorRef>
		<ns2:MessageIdentifier>{{.MessageIdentifier}}</ns2:MessageIdentifier>
	</Request>
	<RequestExtension/>
</ns7:CheckStatus>`

func NewXMLCheckStatusRequest(node *xml.XmlNode) *XMLCheckStatusRequest {
	return &XMLCheckStatusRequest{node: node}
}

func NewXMLCheckStatusRequestFromContent(content []byte) (*XMLCheckStatusRequest, error) {
	doc, err := gokogiri.ParseXml(content)
	if err != nil {
		return nil, err
	}
	request := NewXMLCheckStatusRequest(doc.Root().XmlNode)
	finalizer := func(request *XMLCheckStatusRequest) {
		doc.Free()
	}
	runtime.SetFinalizer(request, finalizer)
	return request, nil
}

func NewSIRICheckStatusRequest(RequestorRef string, RequestTimestamp time.Time, MessageIdentifier string) *SIRICheckStatusRequest {
	return &SIRICheckStatusRequest{RequestorRef: RequestorRef, RequestTimestamp: RequestTimestamp, MessageIdentifier: MessageIdentifier}
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) RequestorRef() string {
	if request.requestorRef == "" {
		nodes, _ := request.node.Search("//*[local-name()='RequestorRef']")
		request.requestorRef = strings.TrimSpace(nodes[0].Content())
	}
	return request.requestorRef
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) RequestTimestamp() time.Time {
	if request.requestTimestamp.IsZero() {
		nodes, _ := request.node.Search("//*[local-name()='RequestTimestamp']")
		t, _ := time.Parse("2006-01-02T15:04:05.000Z07:00", strings.TrimSpace(nodes[0].Content()))
		request.requestTimestamp = t
	}
	return request.requestTimestamp
}

// TODO : Handle errors
func (request *XMLCheckStatusRequest) MessageIdentifier() string {
	if request.messageIdentifier == "" {
		nodes, _ := request.node.Search("//*[local-name()='MessageIdentifier']")
		request.messageIdentifier = strings.TrimSpace(nodes[0].Content())
	}
	return request.messageIdentifier
}

// TODO : Handle errors
func (request *SIRICheckStatusRequest) BuildXML() string {
	var buffer bytes.Buffer
	var siriRequest = template.Must(template.New("siriRequest").Parse(SIRIRequestTemplate))
	if err := siriRequest.Execute(&buffer, request); err != nil {
		log.Fatal(err)
	}
	return buffer.String()
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package data

import (
	"appengine"
	"appengine/datastore"
	"errors"
	"math/rand"
	"net"
)

var (
	ErrNoMatchingSliverTool = errors.New("No matching SliverTool found.")
	ErrNoMatchingSite       = errors.New("No matching Site found.")
)

// GetSliverTools returns a list of all SliverTools.
func GetSliverTools(c appengine.Context) ([]*SliverTool, error) {
	q := datastore.NewQuery("SliverTool")
	var slivers []*SliverTool
	if err := QueryData(c, "SliverTools", q, slivers); err != nil {
		return nil, err
	}
	return slivers, nil
}

// GetSliverToolsWithToolID returns a list of SliverTools which run an M-Lab
// tool with ID, toolID.
func GetSliverToolsWithToolID(c appengine.Context, toolID string) ([]*SliverTool, error) {
	q := datastore.NewQuery("SliverTool").Filter("tool_id =", toolID)
	var slivers []*SliverTool
	if err := QueryData(c, toolID, q, slivers); err != nil {
		return nil, err
	}
	return slivers, nil
}

// GetRandomSliverFromSite returns a randomly selected SliverTool from a list of
// SliverTools which run an M-Lab tool with ID toolID on an M-Lab site with ID
// siteID.
func GetRandomSliverFromSite(c appengine.Context, toolID, siteID string) (*SliverTool, error) {
	slivers, err := GetSliverToolsWithToolID(c, toolID)
	if err != nil {
		return nil, err
	}

	slivers = FilterOnline(slivers)                     // Filter out offline slivers
	siteslivers := make([]*SliverTool, 0, len(slivers)) // Get Slivers of required Site ID
	for _, s := range slivers {
		if s.SiteID == siteID {
			siteslivers = append(siteslivers, s)
		}
	}

	if len(siteslivers) == 0 {
		return nil, ErrNoMatchingSliverTool
	}

	idx := rand.Int() % len(siteslivers) // Get random index
	return siteslivers[idx], nil
}

// GetSiteWithSiteID returns a Site which matches a provided site ID.
func GetSiteWithSiteID(c appengine.Context, siteID string) (*Site, error) {
	q := datastore.NewQuery("Site").Filter("site_id =", siteID)
	var sites []*Site
	if err := QueryData(c, siteID, q, sites); err != nil {
		return nil, err
	}
	if len(sites) == 0 {
		return nil, ErrNoMatchingSite
	}
	return sites[0], nil
}

// GetSliverToolWithIP returns a SliverTool which matches a provided IP.
func GetSliverToolWithIP(c appengine.Context, ip net.IP) (*SliverTool, error) {
	slivers, err := GetSliverTools(c)
	if err != nil {
		return nil, err
	}

	ipStr := ip.String()
	isIPv4 := ip.To4() != nil
	for _, s := range slivers {
		if isIPv4 && s.SliverIPv4 == ipStr {
			return s, nil
		} else if !isIPv4 && s.SliverIPv6 == ipStr {
			return s, nil
		}
	}
	return nil, ErrNoMatchingSliverTool
}
<commit_msg>data: More comments<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package data

import (
	"appengine"
	"appengine/datastore"
	"errors"
	"math/rand"
	"net"
)

var (
	ErrNoMatchingSliverTool = errors.New("No matching SliverTool found.")
	ErrNoMatchingSite       = errors.New("No matching Site found.")
)

// GetSliverTools returns a list of all SliverTools.
func GetSliverTools(c appengine.Context) ([]*SliverTool, error) {
	q := datastore.NewQuery("SliverTool")
	var slivers []*SliverTool
	if err := QueryData(c, "SliverTools", q, slivers); err != nil {
		return nil, err
	}
	return slivers, nil
}

// GetSliverToolsWithToolID returns a list of SliverTools which run an M-Lab
// tool with ID, toolID.
func GetSliverToolsWithToolID(c appengine.Context, toolID string) ([]*SliverTool, error) {
	q := datastore.NewQuery("SliverTool").Filter("tool_id =", toolID)
	var slivers []*SliverTool
	if err := QueryData(c, toolID, q, slivers); err != nil {
		return nil, err
	}
	return slivers, nil
}

// GetRandomSliverFromSite returns a randomly selected online SliverTool from a
// list of SliverTools which run an M-Lab tool with ID toolID on an M-Lab site
// with ID siteID.
func GetRandomSliverFromSite(c appengine.Context, toolID, siteID string) (*SliverTool, error) {
	slivers, err := GetSliverToolsWithToolID(c, toolID)
	if err != nil {
		return nil, err
	}

	slivers = FilterOnline(slivers)                     // Filter out offline slivers
	siteslivers := make([]*SliverTool, 0, len(slivers)) // Get Slivers of required Site ID
	for _, s := range slivers {
		if s.SiteID == siteID {
			siteslivers = append(siteslivers, s)
		}
	}

	if len(siteslivers) == 0 {
		return nil, ErrNoMatchingSliverTool
	}

	idx := rand.Int() % len(siteslivers) // Get random index
	return siteslivers[idx], nil
}

// GetSiteWithSiteID returns a Site which matches a provided site ID.
func GetSiteWithSiteID(c appengine.Context, siteID string) (*Site, error) {
	q := datastore.NewQuery("Site").Filter("site_id =", siteID)
	var sites []*Site
	if err := QueryData(c, siteID, q, sites); err != nil {
		return nil, err
	}
	if len(sites) == 0 {
		return nil, ErrNoMatchingSite
	}
	return sites[0], nil
}

// GetSliverToolWithIP returns a SliverTool which matches a provided IP.
// TODO(seon.wook): Make more efficient. Possibly create and memcache map of IPs
// to *SliverTool.
func GetSliverToolWithIP(c appengine.Context, ip net.IP) (*SliverTool, error) {
	slivers, err := GetSliverTools(c)
	if err != nil {
		return nil, err
	}

	ipStr := ip.String()
	isIPv4 := ip.To4() != nil
	for _, s := range slivers {
		if isIPv4 && s.SliverIPv4 == ipStr {
			return s, nil
		} else if !isIPv4 && s.SliverIPv6 == ipStr {
			return s, nil
		}
	}
	return nil, ErrNoMatchingSliverTool
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte              // Gateway's Identifier
	alti     int                  // GPS altitude in RX meters
	ackr     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	txnb     uint                 // Number of packets emitted
	adapters []io.ReadWriteCloser // List of downlink adapters
}

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}
	return &Forwarder{
		Id:       id,
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
	}, nil
}

// Forward dispatch a packet to all connected routers.
func (fwd *Forwarder) Forward(packet semtech.Packet) error {
	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd *Forwarder) Flush() []semtech.Packet {
	return nil
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	var ackr float64
	if fwd.txnb != 0 {
		ackr = float64(fwd.ackr) / float64(fwd.txnb)
	}

	return semtech.Stat{
		Ackr: &ackr,
		Alti: pointer.Int(fwd.alti),
		Dwnb: pointer.Uint(fwd.dwnb),
		Lati: pointer.Float64(fwd.lati),
		Long: pointer.Float64(fwd.long),
		Rxfw: pointer.Uint(fwd.rxfw),
		Rxnb: pointer.Uint(fwd.rxnb),
		Rxok: pointer.Uint(fwd.rxnb),
		Time: pointer.Time(time.Now()),
		Txnb: pointer.Uint(fwd.txnb),
	}
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd *Forwarder) Stop() error {
	var errors []error
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}
	return nil
}
<commit_msg>[simulators.gateway] Add basic forwarding<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte              // Gateway's Identifier
	alti     int                  // GPS altitude in RX meters
	upnb     uint                 // Number of upstream datagrams sent
	ackn     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	txnb     uint                 // Number of packets emitted
	adapters []io.ReadWriteCloser // List of downlink adapters
	packets  []semtech.Packet     // Downlink packets received
}

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}
	return &Forwarder{
		Id:       id,
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
	}, nil
}

// Forward dispatch a packet to all connected routers.
func (fwd *Forwarder) Forward(packet semtech.Packet) error {
	if packet.Identifier != semtech.PUSH_DATA {
		return fmt.Errorf("Unable to forward with identifier %x", packet.Identifier)
	}

	raw, err := semtech.Marshal(packet)
	if err != nil {
		return err
	}

	for _, adapter := range fwd.adapters {
		n, err := adapter.Write(raw)
		if err != nil {
			return err
		}
		if n < len(raw) {
			return fmt.Errorf("Packet was too long")
		}
	}

	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd *Forwarder) Flush() []semtech.Packet {
	return nil
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	var ackr float64
	if fwd.upnb != 0 {
		ackr = float64(fwd.ackn) / float64(fwd.upnb)
	}

	return semtech.Stat{
		Ackr: &ackr,
		Alti: pointer.Int(fwd.alti),
		Dwnb: pointer.Uint(fwd.dwnb),
		Lati: pointer.Float64(fwd.lati),
		Long: pointer.Float64(fwd.long),
		Rxfw: pointer.Uint(fwd.rxfw),
		Rxnb: pointer.Uint(fwd.rxnb),
		Rxok: pointer.Uint(fwd.rxnb),
		Time: pointer.Time(time.Now()),
		Txnb: pointer.Uint(fwd.txnb),
	}
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd *Forwarder) Stop() error {
	var errors []error
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}
	return nil
}
<|endoftext|>


<commit_before>package gerrit

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func (g GerritInstance) GetChangeInformation(changeID string) (*ChangeInfo, error) {
	urlToCall := fmt.Sprintf("%s/changes/%s/?o=CURRENT_REVISION", g.getAPIUrl(false), changeID)
	log.Printf("> Calling %s\n", urlToCall)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", urlToCall, nil)
	req.SetBasicAuth(g.Username, g.Password)
	req.Header.Add("Content-Type", "application/json;charset=UTF-8")

	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Printf("> Change-details for change id \"%s\" received", changeID)

	} else {
		log.Printf("> Call success, but the status code doesn`t match ~200: %s - %s", resp.Status, err)
		return nil, errors.New("Call success, but the status code doesn`t match ~200")
	}

	var change ChangeInfo

	respBody, err := ioutil.ReadAll(resp.Body)
	// Every Gerrit response starts with ")]}'"
	jsonBody := string(respBody)[4:]

	err = json.Unmarshal([]byte(jsonBody), &change)
	if err != nil {
		return nil, err
	}

	return &change, nil
}

// https://review.typo3.org/Documentation/rest-api-changes.html#set-review
func (g GerritInstance) PostCommentOnChangeset(m *Message, vote int, msg string) {
	log.Printf("> Start posting review for %s (%s)", m.Change.URL, m.Patchset.Ref)

	changeID := m.Change.ID
	revisionID := m.Patchset.Revision
	urlToCall := fmt.Sprintf("%s/changes/%s/revisions/%s/review", g.getAPIUrl(true), changeID, revisionID)

	log.Printf("> Calling %s", urlToCall)

	bodyStruct := &ReviewInput{
		Message: msg,
		Labels: map[string]int{
			// Code-Review
			"Verified": vote,
		},
	}

	body, _ := json.Marshal(bodyStruct)

	client := &http.Client{}
	req, _ := http.NewRequest("POST", urlToCall, strings.NewReader(string(body)))
	req.SetBasicAuth(g.Username, g.Password)
	req.Header.Add("Content-Type", "application/json;charset=UTF-8")

	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		log.Println("> Call failed", err)
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Printf("> Call success: %s", resp.Status)
	} else {
		log.Printf("> Call success, but the status code doesn`t match ~200: %s", resp.Status)
	}
}
<commit_msg>Removed printing a error, because err is nil at every time (see if statement a few lines before)<commit_after>package gerrit

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func (g GerritInstance) GetChangeInformation(changeID string) (*ChangeInfo, error) {
	urlToCall := fmt.Sprintf("%s/changes/%s/?o=CURRENT_REVISION", g.getAPIUrl(false), changeID)
	log.Printf("> Calling %s\n", urlToCall)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", urlToCall, nil)
	req.SetBasicAuth(g.Username, g.Password)
	req.Header.Add("Content-Type", "application/json;charset=UTF-8")

	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Printf("> Change-details for change id \"%s\" received", changeID)

	} else {
		log.Printf("> Call success, but the status code doesn`t match ~200: %s", resp.Status)
		return nil, errors.New("Call success, but the status code doesn`t match ~200")
	}

	var change ChangeInfo

	respBody, err := ioutil.ReadAll(resp.Body)
	// Every Gerrit response starts with ")]}'"
	jsonBody := string(respBody)[4:]

	err = json.Unmarshal([]byte(jsonBody), &change)
	if err != nil {
		return nil, err
	}

	return &change, nil
}

// https://review.typo3.org/Documentation/rest-api-changes.html#set-review
func (g GerritInstance) PostCommentOnChangeset(m *Message, vote int, msg string) {
	log.Printf("> Start posting review for %s (%s)", m.Change.URL, m.Patchset.Ref)

	changeID := m.Change.ID
	revisionID := m.Patchset.Revision
	urlToCall := fmt.Sprintf("%s/changes/%s/revisions/%s/review", g.getAPIUrl(true), changeID, revisionID)

	log.Printf("> Calling %s", urlToCall)

	bodyStruct := &ReviewInput{
		Message: msg,
		Labels: map[string]int{
			// Code-Review
			"Verified": vote,
		},
	}

	body, _ := json.Marshal(bodyStruct)

	client := &http.Client{}
	req, _ := http.NewRequest("POST", urlToCall, strings.NewReader(string(body)))
	req.SetBasicAuth(g.Username, g.Password)
	req.Header.Add("Content-Type", "application/json;charset=UTF-8")

	resp, err := client.Do(req)
	defer resp.Body.Close()

	if err != nil {
		log.Println("> Call failed", err)
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Printf("> Call success: %s", resp.Status)
	} else {
		log.Printf("> Call success, but the status code doesn`t match ~200: %s", resp.Status)
	}
}
<|endoftext|>


<commit_before>package uploader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Uploader interface {
	Upload(u *url.URL, filename string, r *http.Request) (*http.Response, *url.URL, error)
	Poll(u *url.URL, res *http.Response, closeChan <-chan bool, interval time.Duration) error
}

type httpUploader struct {
	baseUrl *url.URL
	client  *http.Client
}

func New(baseUrl *url.URL, transport http.RoundTripper) Uploader {
	return &httpUploader{
		baseUrl: baseUrl,
		client: &http.Client{
			Transport: transport,
		},
	}
}

const contentMD5Header = "Content-MD5"

func (u *httpUploader) Upload(primaryUrl *url.URL, filename string, r *http.Request) (*http.Response, *url.URL, error) {
	if r.ContentLength <= 0 {
		return &http.Response{StatusCode: http.StatusLengthRequired}, nil, fmt.Errorf("Missing Content Length")
	}
	defer r.Body.Close()

	uploadReq, err := newMultipartRequestFromReader(r.ContentLength, r.Body, filename)
	if err != nil {
		return nil, nil, err
	}

	uploadReq.Header.Set(contentMD5Header, r.Header.Get(contentMD5Header))

	// try the fast path
	uploadReq.URL = primaryUrl
	if primaryUrl.User != nil {
		if password, set := primaryUrl.User.Password(); set {
			uploadReq.SetBasicAuth(primaryUrl.User.Username(), password)
		}
	}

	rsp, err := u.do(uploadReq)
	if err == nil {
		return rsp, uploadReq.URL, err
	}

	// not a connect (dial) error
	var nestedErr error = err
	if urlErr, ok := err.(*url.Error); ok {
		nestedErr = urlErr.Err
	}
	if netErr, ok := nestedErr.(*net.OpError); !ok || netErr.Op != "dial" {
		return rsp, nil, err
	}

	// try the slow path
	uploadReq, err = newMultipartRequestFromReader(r.ContentLength, r.Body, filename)
	if err != nil {
		return nil, nil, err
	}

	uploadReq.Header.Set(contentMD5Header, r.Header.Get(contentMD5Header))

	uploadReq.URL.Scheme = u.baseUrl.Scheme
	uploadReq.URL.Host = u.baseUrl.Host
	if u.baseUrl.User != nil {
		if password, set := u.baseUrl.User.Password(); set {
			uploadReq.URL.User = u.baseUrl.User
			uploadReq.SetBasicAuth(u.baseUrl.User.Username(), password)
		}
	}
	uploadReq.URL.Path = primaryUrl.Path
	uploadReq.URL.RawQuery = primaryUrl.RawQuery

	rsp, err = u.do(uploadReq)
	return rsp, uploadReq.URL, err
}

func (u *httpUploader) do(req *http.Request) (*http.Response, error) {
	rsp, err := u.client.Do(req)
	req.Body.Close()
	if err != nil {
		return nil, err
	}

	switch rsp.StatusCode {
	case http.StatusOK, http.StatusCreated:
		return rsp, nil
	}

	respBody, _ := ioutil.ReadAll(rsp.Body)
	rsp.Body.Close()
	return rsp, fmt.Errorf("status code: %d\n%s", rsp.StatusCode, string(respBody))
}

// CLOUD CONTROLLER POLLING HELPERS
// The following code is not specific to uploading droplets.
// As soon as we talk to cloud controller in more than one place,
// this code should be extracted into a library

const (
	JOB_QUEUED   = "queued"
	JOB_RUNNING  = "running"
	JOB_FAILED   = "failed"
	JOB_FINISHED = "finished"
)

func (u *httpUploader) Poll(target *url.URL, res *http.Response, closeChan <-chan bool, interval time.Duration) error {
	body, err := u.parsePollingResponse(res)
	if err != nil {
		return err
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		switch body.Entity.Status {
		case JOB_QUEUED, JOB_RUNNING:
		case JOB_FINISHED:
			return nil
		case JOB_FAILED:
			return fmt.Errorf("upload job failed")
		default:
			return fmt.Errorf("unknown job status: %s", body.Entity.Status)
		}

		select {
		case <-ticker.C:
			pollingUrl, err := url.Parse(body.Metadata.Url)
			if err != nil {
				return err
			}

			if pollingUrl.Host == "" {
				pollingUrl.Scheme = target.Scheme
				pollingUrl.Host = target.Host
			}
			res, err := u.client.Get(pollingUrl.String())
			if err != nil {
				return err
			}

			body, err = u.parsePollingResponse(res)
			if err != nil {
				return err
			}
		case <-closeChan:
			return fmt.Errorf("upstream request was cancelled")
		}
	}
}

func (u *httpUploader) parsePollingResponse(res *http.Response) (pollingResponseBody, error) {
	body := pollingResponseBody{}
	err := json.NewDecoder(res.Body).Decode(&body)
	res.Body.Close()
	return body, err
}

type pollingResponseBody struct {
	Metadata struct {
		Url string
	}
	Entity struct {
		Status string
	}
}
<commit_msg>Rename target to fallbackURL in Poll [#82415708]<commit_after>package uploader

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Uploader interface {
	Upload(u *url.URL, filename string, r *http.Request) (*http.Response, *url.URL, error)
	Poll(u *url.URL, res *http.Response, closeChan <-chan bool, interval time.Duration) error
}

type httpUploader struct {
	baseUrl *url.URL
	client  *http.Client
}

func New(baseUrl *url.URL, transport http.RoundTripper) Uploader {
	return &httpUploader{
		baseUrl: baseUrl,
		client: &http.Client{
			Transport: transport,
		},
	}
}

const contentMD5Header = "Content-MD5"

func (u *httpUploader) Upload(primaryUrl *url.URL, filename string, r *http.Request) (*http.Response, *url.URL, error) {
	if r.ContentLength <= 0 {
		return &http.Response{StatusCode: http.StatusLengthRequired}, nil, fmt.Errorf("Missing Content Length")
	}
	defer r.Body.Close()

	uploadReq, err := newMultipartRequestFromReader(r.ContentLength, r.Body, filename)
	if err != nil {
		return nil, nil, err
	}

	uploadReq.Header.Set(contentMD5Header, r.Header.Get(contentMD5Header))

	// try the fast path
	uploadReq.URL = primaryUrl
	if primaryUrl.User != nil {
		if password, set := primaryUrl.User.Password(); set {
			uploadReq.SetBasicAuth(primaryUrl.User.Username(), password)
		}
	}

	rsp, err := u.do(uploadReq)
	if err == nil {
		return rsp, uploadReq.URL, err
	}

	// not a connect (dial) error
	var nestedErr error = err
	if urlErr, ok := err.(*url.Error); ok {
		nestedErr = urlErr.Err
	}
	if netErr, ok := nestedErr.(*net.OpError); !ok || netErr.Op != "dial" {
		return rsp, nil, err
	}

	// try the slow path
	uploadReq, err = newMultipartRequestFromReader(r.ContentLength, r.Body, filename)
	if err != nil {
		return nil, nil, err
	}

	uploadReq.Header.Set(contentMD5Header, r.Header.Get(contentMD5Header))

	uploadReq.URL.Scheme = u.baseUrl.Scheme
	uploadReq.URL.Host = u.baseUrl.Host
	if u.baseUrl.User != nil {
		if password, set := u.baseUrl.User.Password(); set {
			uploadReq.URL.User = u.baseUrl.User
			uploadReq.SetBasicAuth(u.baseUrl.User.Username(), password)
		}
	}
	uploadReq.URL.Path = primaryUrl.Path
	uploadReq.URL.RawQuery = primaryUrl.RawQuery

	rsp, err = u.do(uploadReq)
	return rsp, uploadReq.URL, err
}

func (u *httpUploader) do(req *http.Request) (*http.Response, error) {
	rsp, err := u.client.Do(req)
	req.Body.Close()
	if err != nil {
		return nil, err
	}

	switch rsp.StatusCode {
	case http.StatusOK, http.StatusCreated:
		return rsp, nil
	}

	respBody, _ := ioutil.ReadAll(rsp.Body)
	rsp.Body.Close()
	return rsp, fmt.Errorf("status code: %d\n%s", rsp.StatusCode, string(respBody))
}

// CLOUD CONTROLLER POLLING HELPERS
// The following code is not specific to uploading droplets.
// As soon as we talk to cloud controller in more than one place,
// this code should be extracted into a library

const (
	JOB_QUEUED   = "queued"
	JOB_RUNNING  = "running"
	JOB_FAILED   = "failed"
	JOB_FINISHED = "finished"
)

func (u *httpUploader) Poll(fallbackURL *url.URL, res *http.Response, closeChan <-chan bool, interval time.Duration) error {
	body, err := u.parsePollingResponse(res)
	if err != nil {
		return err
	}

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		switch body.Entity.Status {
		case JOB_QUEUED, JOB_RUNNING:
		case JOB_FINISHED:
			return nil
		case JOB_FAILED:
			return fmt.Errorf("upload job failed")
		default:
			return fmt.Errorf("unknown job status: %s", body.Entity.Status)
		}

		select {
		case <-ticker.C:
			pollingUrl, err := url.Parse(body.Metadata.Url)
			if err != nil {
				return err
			}

			if pollingUrl.Host == "" {
				pollingUrl.Scheme = fallbackURL.Scheme
				pollingUrl.Host = fallbackURL.Host
			}
			res, err := u.client.Get(pollingUrl.String())
			if err != nil {
				return err
			}

			body, err = u.parsePollingResponse(res)
			if err != nil {
				return err
			}
		case <-closeChan:
			return fmt.Errorf("upstream request was cancelled")
		}
	}
}

func (u *httpUploader) parsePollingResponse(res *http.Response) (pollingResponseBody, error) {
	body := pollingResponseBody{}
	err := json.NewDecoder(res.Body).Decode(&body)
	res.Body.Close()
	return body, err
}

type pollingResponseBody struct {
	Metadata struct {
		Url string
	}
	Entity struct {
		Status string
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"time"

	"net"

	"sync"

	"os"

	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goworld/common"
	"github.com/xiaonanln/goworld/components/dispatcher/dispatcher_client"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/consts"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/opmon"
	"github.com/xiaonanln/goworld/proto"
)

type GateService struct {
	listenAddr        string
	clientProxies     map[common.ClientID]*ClientProxy
	clientProxiesLock sync.RWMutex
	packetQueue       sync_queue.SyncQueue

	filterTreesLock sync.Mutex
	filterTrees     map[string]*FilterTree
}

func newGateService() *GateService {
	return &GateService{
		//packetQueue: make(chan packetQueueItem, consts.DISPATCHER_CLIENT_PACKET_QUEUE_SIZE),
		clientProxies: map[common.ClientID]*ClientProxy{},
		packetQueue:   sync_queue.NewSyncQueue(),
		filterTrees:   map[string]*FilterTree{},
	}
}

func (gs *GateService) run() {
	cfg := config.GetServer(serverid)
	gwlog.Info("Compress connection: %v", cfg.CompressConnection)
	gs.listenAddr = fmt.Sprintf("%s:%d", cfg.Ip, cfg.Port)
	go netutil.ServeForever(gs.handlePacketRoutine)
	netutil.ServeTCPForever(gs.listenAddr, gs)
}

func (gs *GateService) String() string {
	return fmt.Sprintf("GateService<%s>", gs.listenAddr)
}

func (gs *GateService) ServeTCPConnection(conn net.Conn) {
	cfg := config.GetServer(serverid)
	cp := newClientProxy(conn, cfg)

	gs.clientProxiesLock.Lock()
	gs.clientProxies[cp.clientid] = cp
	gs.clientProxiesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientConnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.ServeTCPConnection: client %s connected", gs, cp)
	}
	cp.serve()
}

func (gs *GateService) onClientProxyClose(cp *ClientProxy) {
	gs.clientProxiesLock.Lock()
	delete(gs.clientProxies, cp.clientid)
	gs.clientProxiesLock.Unlock()

	gs.filterTreesLock.Lock()
	for key, val := range cp.filterProps {
		ft := gs.filterTrees[key]
		if ft != nil {
			if consts.DEBUG_FILTER_PROP {
				gwlog.Debug("DROP CLIENT %s FILTER PROP: %s = %s", cp, key, val)
			}
			ft.Remove(cp.clientid, val)
		}
	}
	gs.filterTreesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.onClientProxyClose: client %s disconnected", gs, cp)
	}
}

func (gs *GateService) HandleDispatcherClientPacket(msgtype proto.MsgType_t, packet *netutil.Packet) {
	if consts.DEBUG_PACKETS {
		gwlog.Debug("%s.HandleDispatcherClientPacket: msgtype=%v, packet(%d)=%v", gs, msgtype, packet.GetPayloadLen(), packet.Payload())
	}

	if msgtype >= proto.MT_REDIRECT_TO_GATEPROXY_MSG_TYPE_START && msgtype <= proto.MT_REDIRECT_TO_GATEPROXY_MSG_TYPE_STOP {
		_ = packet.ReadUint16() // sid
		clientid := packet.ReadClientID()

		gs.clientProxiesLock.RLock()
		clientproxy := gs.clientProxies[clientid]
		gs.clientProxiesLock.RUnlock()

		if clientproxy != nil {
			if msgtype == proto.MT_SET_CLIENTPROXY_FILTER_PROP {
				gs.handleSetClientFilterProp(clientproxy, packet)
			} else if msgtype == proto.MT_CLEAR_CLIENTPROXY_FILTER_PROPS {
				gs.handleClearClientFilterProps(clientproxy, packet)
			} else {
				// message types that should be redirected to client proxy
				clientproxy.SendPacket(packet)
			}
		} else {
			// client already disconnected, but the game service seems not knowing it, so tell it
			dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(clientid)
		}
	} else if msgtype == proto.MT_CALL_FILTERED_CLIENTS {
		gs.handleCallFilteredClientProxies(packet)
	} else {
		gwlog.Panicf("%s: unknown msg type: %d", gs, msgtype)
		if consts.DEBUG_MODE {
			os.Exit(2)
		}
	}
}

func (gs *GateService) handleSetClientFilterProp(clientproxy *ClientProxy, packet *netutil.Packet) {
	gwlog.Debug("%s.handleSetClientFilterProp: clientproxy=%s", gs, clientproxy)
	key := packet.ReadVarStr()
	val := packet.ReadVarStr()
	clientid := clientproxy.clientid

	gs.filterTreesLock.Lock()
	ft, ok := gs.filterTrees[key]
	if !ok {
		ft = NewFilterTree()
		gs.filterTrees[key] = ft
	}

	oldVal, ok := clientproxy.filterProps[key]
	if ok {
		if consts.DEBUG_FILTER_PROP {
			gwlog.Debug("REMOVE CLIENT %s FILTER PROP: %s = %s", clientproxy, key, val)
		}
		ft.Remove(clientid, oldVal)
	}
	clientproxy.filterProps[key] = val
	ft.Insert(clientid, val)
	gs.filterTreesLock.Unlock()

	if consts.DEBUG_FILTER_PROP {
		gwlog.Debug("SET CLIENT %s FILTER PROP: %s = %s", clientproxy, key, val)
	}
}

func (gs *GateService) handleClearClientFilterProps(clientproxy *ClientProxy, packet *netutil.Packet) {
	gwlog.Debug("%s.handleClearClientFilterProps: clientproxy=%s", gs, clientproxy)
	clientid := clientproxy.clientid

	gs.filterTreesLock.Lock()

	for key, val := range clientproxy.filterProps {
		ft, ok := gs.filterTrees[key]
		if !ok {
			continue
		}
		ft.Remove(clientid, val)
	}
	gs.filterTreesLock.Unlock()

	if consts.DEBUG_FILTER_PROP {
		gwlog.Debug("CLEAR CLIENT %s FILTER PROPS", clientproxy)
	}
}

func (gs *GateService) handleCallFilteredClientProxies(packet *netutil.Packet) {
	key := packet.ReadVarStr()
	val := packet.ReadVarStr()

	gs.filterTreesLock.Lock()
	gs.clientProxiesLock.RLock()

	ft := gs.filterTrees[key]
	if ft != nil {
		ft.Visit(val, func(clientid common.ClientID) {
			//// visit all clientids and
			clientproxy := gs.clientProxies[clientid]
			if clientproxy != nil {
				clientproxy.SendPacket(packet)
			}
		})
	}

	gs.clientProxiesLock.RUnlock()
	gs.filterTreesLock.Unlock()
}

func (gs *GateService) handlePacketRoutine() {
	for {
		item := gs.packetQueue.Pop().(packetQueueItem)
		op := opmon.StartOperation("GateServiceHandlePacket")
		gs.HandleDispatcherClientPacket(item.msgtype, item.packet)
		op.Finish(time.Millisecond * 100)
		item.packet.Release()
	}
}
<commit_msg>gate service terminate<commit_after>package server

import (
	"fmt"
	"time"

	"net"

	"sync"

	"os"

	"github.com/xiaonanln/go-xnsyncutil/xnsyncutil"
	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goworld/common"
	"github.com/xiaonanln/goworld/components/dispatcher/dispatcher_client"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/consts"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/opmon"
	"github.com/xiaonanln/goworld/proto"
)

type GateService struct {
	listenAddr        string
	clientProxies     map[common.ClientID]*ClientProxy
	clientProxiesLock sync.RWMutex
	packetQueue       sync_queue.SyncQueue

	filterTreesLock sync.Mutex
	filterTrees     map[string]*FilterTree
	terminated      xnsyncutil.AtomicBool
}

func newGateService() *GateService {
	return &GateService{
		//packetQueue: make(chan packetQueueItem, consts.DISPATCHER_CLIENT_PACKET_QUEUE_SIZE),
		clientProxies: map[common.ClientID]*ClientProxy{},
		packetQueue:   sync_queue.NewSyncQueue(),
		filterTrees:   map[string]*FilterTree{},
	}
}

func (gs *GateService) run() {
	cfg := config.GetServer(serverid)
	gwlog.Info("Compress connection: %v", cfg.CompressConnection)
	gs.listenAddr = fmt.Sprintf("%s:%d", cfg.Ip, cfg.Port)
	go netutil.ServeForever(gs.handlePacketRoutine)
	netutil.ServeTCPForever(gs.listenAddr, gs)
}

func (gs *GateService) String() string {
	return fmt.Sprintf("GateService<%s>", gs.listenAddr)
}

func (gs *GateService) ServeTCPConnection(conn net.Conn) {
	cfg := config.GetServer(serverid)
	cp := newClientProxy(conn, cfg)

	gs.clientProxiesLock.Lock()
	gs.clientProxies[cp.clientid] = cp
	gs.clientProxiesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientConnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.ServeTCPConnection: client %s connected", gs, cp)
	}
	cp.serve()
}

func (gs *GateService) onClientProxyClose(cp *ClientProxy) {
	gs.clientProxiesLock.Lock()
	delete(gs.clientProxies, cp.clientid)
	gs.clientProxiesLock.Unlock()

	gs.filterTreesLock.Lock()
	for key, val := range cp.filterProps {
		ft := gs.filterTrees[key]
		if ft != nil {
			if consts.DEBUG_FILTER_PROP {
				gwlog.Debug("DROP CLIENT %s FILTER PROP: %s = %s", cp, key, val)
			}
			ft.Remove(cp.clientid, val)
		}
	}
	gs.filterTreesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.onClientProxyClose: client %s disconnected", gs, cp)
	}
}

func (gs *GateService) HandleDispatcherClientPacket(msgtype proto.MsgType_t, packet *netutil.Packet) {
	if consts.DEBUG_PACKETS {
		gwlog.Debug("%s.HandleDispatcherClientPacket: msgtype=%v, packet(%d)=%v", gs, msgtype, packet.GetPayloadLen(), packet.Payload())
	}

	if msgtype >= proto.MT_REDIRECT_TO_GATEPROXY_MSG_TYPE_START && msgtype <= proto.MT_REDIRECT_TO_GATEPROXY_MSG_TYPE_STOP {
		_ = packet.ReadUint16() // sid
		clientid := packet.ReadClientID()

		gs.clientProxiesLock.RLock()
		clientproxy := gs.clientProxies[clientid]
		gs.clientProxiesLock.RUnlock()

		if clientproxy != nil {
			if msgtype == proto.MT_SET_CLIENTPROXY_FILTER_PROP {
				gs.handleSetClientFilterProp(clientproxy, packet)
			} else if msgtype == proto.MT_CLEAR_CLIENTPROXY_FILTER_PROPS {
				gs.handleClearClientFilterProps(clientproxy, packet)
			} else {
				// message types that should be redirected to client proxy
				clientproxy.SendPacket(packet)
			}
		} else {
			// client already disconnected, but the game service seems not knowing it, so tell it
			dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(clientid)
		}
	} else if msgtype == proto.MT_CALL_FILTERED_CLIENTS {
		gs.handleCallFilteredClientProxies(packet)
	} else {
		gwlog.Panicf("%s: unknown msg type: %d", gs, msgtype)
		if consts.DEBUG_MODE {
			os.Exit(2)
		}
	}
}

func (gs *GateService) handleSetClientFilterProp(clientproxy *ClientProxy, packet *netutil.Packet) {
	gwlog.Debug("%s.handleSetClientFilterProp: clientproxy=%s", gs, clientproxy)
	key := packet.ReadVarStr()
	val := packet.ReadVarStr()
	clientid := clientproxy.clientid

	gs.filterTreesLock.Lock()
	ft, ok := gs.filterTrees[key]
	if !ok {
		ft = NewFilterTree()
		gs.filterTrees[key] = ft
	}

	oldVal, ok := clientproxy.filterProps[key]
	if ok {
		if consts.DEBUG_FILTER_PROP {
			gwlog.Debug("REMOVE CLIENT %s FILTER PROP: %s = %s", clientproxy, key, val)
		}
		ft.Remove(clientid, oldVal)
	}
	clientproxy.filterProps[key] = val
	ft.Insert(clientid, val)
	gs.filterTreesLock.Unlock()

	if consts.DEBUG_FILTER_PROP {
		gwlog.Debug("SET CLIENT %s FILTER PROP: %s = %s", clientproxy, key, val)
	}
}

func (gs *GateService) handleClearClientFilterProps(clientproxy *ClientProxy, packet *netutil.Packet) {
	gwlog.Debug("%s.handleClearClientFilterProps: clientproxy=%s", gs, clientproxy)
	clientid := clientproxy.clientid

	gs.filterTreesLock.Lock()

	for key, val := range clientproxy.filterProps {
		ft, ok := gs.filterTrees[key]
		if !ok {
			continue
		}
		ft.Remove(clientid, val)
	}
	gs.filterTreesLock.Unlock()

	if consts.DEBUG_FILTER_PROP {
		gwlog.Debug("CLEAR CLIENT %s FILTER PROPS", clientproxy)
	}
}

func (gs *GateService) handleCallFilteredClientProxies(packet *netutil.Packet) {
	key := packet.ReadVarStr()
	val := packet.ReadVarStr()

	gs.filterTreesLock.Lock()
	gs.clientProxiesLock.RLock()

	ft := gs.filterTrees[key]
	if ft != nil {
		ft.Visit(val, func(clientid common.ClientID) {
			//// visit all clientids and
			clientproxy := gs.clientProxies[clientid]
			if clientproxy != nil {
				clientproxy.SendPacket(packet)
			}
		})
	}

	gs.clientProxiesLock.RUnlock()
	gs.filterTreesLock.Unlock()
}

func (gs *GateService) handlePacketRoutine() {
	for {
		item := gs.packetQueue.Pop().(packetQueueItem)
		op := opmon.StartOperation("GateServiceHandlePacket")
		gs.HandleDispatcherClientPacket(item.msgtype, item.packet)
		op.Finish(time.Millisecond * 100)
		item.packet.Release()
	}
}

func (gs *GateService) terminate() {
	gs.terminated.Store(true)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws/s3"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/fs"
	s3_kv "github.com/jacobsa/comeback/kv/s3"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"time"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func randUint64(randSrc *rand.Rand) uint64

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Validate the config file.
	if err := config.Validate(cfg); err != nil {
		fmt.Printf("Config file invalid: %v", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Open a connection to SimpleDB.
	db, err := sdb.NewSimpleDB(cfg.SdbRegion, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating SimpleDB: %v", err)
	}

	// Open a connection to S3.
	bucket, err := s3.OpenBucket(cfg.S3Bucket, cfg.S3Region, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating bucket: %v", err)
	}

	// Create the crypter.
	crypter, err := crypto.NewCrypter(cryptoKey)
	if err != nil {
		log.Fatalf("Creating crypter: %v", err)
	}

	// Create the backup registry.
	randSrc := rand.New(rand.NewSource(time.Now().UnixNano()))
	registry, err := backup.NewRegistry(db, cfg.SdbDomain, crypter, randSrc)
	if err != nil {
		log.Fatalf("Creating registry: %v", err)
	}

	// Create the kv store.
	kvStore, err := s3_kv.NewS3KvStore(bucket)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)
	blobStore = blob.NewCheckingStore(blobStore)
	blobStore = blob.NewEncryptingStore(crypter, blobStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Register the successful backup.
	completedJob := backup.CompletedJob{
		Id: randUint64(randSrc),
		Name: *g_jobName,
		StartTime: startTime,
		Score: score,
	}

	if err = registry.RecordBackup(completedJob); err != nil {
		log.Fatalf("Recoding to registry: %v", err)
	}

	fmt.Printf("Successfully backed up. ID: %16x\n", completedJob.Id)
}
<commit_msg>Fixed formatting.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/aws/s3"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/fs"
	s3_kv "github.com/jacobsa/comeback/kv/s3"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"time"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func randUint64(randSrc *rand.Rand) uint64

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Validate the config file.
	if err := config.Validate(cfg); err != nil {
		fmt.Printf("Config file invalid: %v", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Open a connection to SimpleDB.
	db, err := sdb.NewSimpleDB(cfg.SdbRegion, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating SimpleDB: %v", err)
	}

	// Open a connection to S3.
	bucket, err := s3.OpenBucket(cfg.S3Bucket, cfg.S3Region, cfg.AccessKey)
	if err != nil {
		log.Fatalf("Creating bucket: %v", err)
	}

	// Create the crypter.
	crypter, err := crypto.NewCrypter(cryptoKey)
	if err != nil {
		log.Fatalf("Creating crypter: %v", err)
	}

	// Create the backup registry.
	randSrc := rand.New(rand.NewSource(time.Now().UnixNano()))
	registry, err := backup.NewRegistry(db, cfg.SdbDomain, crypter, randSrc)
	if err != nil {
		log.Fatalf("Creating registry: %v", err)
	}

	// Create the kv store.
	kvStore, err := s3_kv.NewS3KvStore(bucket)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)
	blobStore = blob.NewCheckingStore(blobStore)
	blobStore = blob.NewEncryptingStore(crypter, blobStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Register the successful backup.
	completedJob := backup.CompletedJob{
		Id:        randUint64(randSrc),
		Name:      *g_jobName,
		StartTime: startTime,
		Score:     score,
	}

	if err = registry.RecordBackup(completedJob); err != nil {
		log.Fatalf("Recoding to registry: %v", err)
	}

	fmt.Printf("Successfully backed up. ID: %16x\n", completedJob.Id)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package httpvar provides a runtimevar implementation with variables
// backed by http endpoint. Use OpenVariable to construct a *runtimevar.Variable.
//
// URLs
//
// For runtimevar.OpenVariable, httpvar registers for the schemes "http" and
// "https". The default URL opener will use http.DefaultClient.
// To customize the URL opener, or for more details on the URL format,
// see URLOpener.
// See https://gocloud.dev/concepts/urls/ for background information.
//
// As
//
// httpvar exposes the following types for As:
//  - Snapshot: *http.Response
//  - Error: httpvar.RequestError, url.Error
package httpvar // import "gocloud.dev/runtimevar/httpvar"

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"gocloud.dev/gcerrors"
	"gocloud.dev/internal/gcerr"
	"gocloud.dev/runtimevar"
	"gocloud.dev/runtimevar/driver"
)

func init() {
	o := &URLOpener{Client: http.DefaultClient}
	for _, scheme := range Schemes {
		runtimevar.DefaultURLMux().RegisterVariable(scheme, o)
	}
}

// Schemes are the URL schemes httpvar registers its URLOpener under on runtimevar.DefaultMux.
var Schemes = []string{"http", "https"}

// URLOpener opens HTTP URLs like "http://myserver.com/foo.txt".
//
// The full URL, including scheme, is used as the endpoint, except that the
// the following URL parameters are removed if present:
//   - decoder: The decoder to use. Defaults to runtimevar.BytesDecoder.
//       See runtimevar.DecoderByName for supported values.
type URLOpener struct {
	// The Client to use; required.
	Client *http.Client

	// Decoder specifies the decoder to use if one is not specified in the URL.
	// Defaults to runtimevar.BytesDecoder.
	Decoder *runtimevar.Decoder

	// Options specifies the options to pass to OpenVariable.
	Options Options
}

// OpenVariableURL opens a httpvar Variable for u.
func (o *URLOpener) OpenVariableURL(ctx context.Context, u *url.URL) (*runtimevar.Variable, error) {
	// Clone u because we may strip some query parameters.
	u2 := *u
	q := u2.Query()

	decoderName := q.Get("decoder")
	q.Del("decoder")
	decoder, err := runtimevar.DecoderByName(ctx, decoderName, o.Decoder)
	if err != nil {
		return nil, fmt.Errorf("open variable %v: invalid decoder: %v", u, err)
	}
	// See if we changed the query parameters.
	if rawq := q.Encode(); rawq != u.Query().Encode() {
		u2.RawQuery = rawq
	}
	return OpenVariable(o.Client, u2.String(), decoder, &o.Options)
}

// Options sets options.
type Options struct {
	// WaitDuration controls the rate at which the HTTP endpoint is called to check for changes.
	// Defaults to 30 seconds.
	WaitDuration time.Duration
}

// RequestError represents an HTTP error that occurred during endpoint call.
type RequestError struct {
	Response *http.Response
}

func (e *RequestError) Error() string {
	return fmt.Sprintf("httpvar: received status code %d", e.Response.StatusCode)
}

func newRequestError(response *http.Response) *RequestError {
	return &RequestError{Response: response}
}

// OpenVariable constructs a *runtimevar.Variable that uses client
// to retrieve the variable contents from the URL urlStr.
func OpenVariable(client *http.Client, urlStr string, decoder *runtimevar.Decoder, opts *Options) (*runtimevar.Variable, error) {
	endpointURL, err := url.Parse(urlStr)
	if err != nil {
		return nil, fmt.Errorf("httpvar: failed to parse url %q: %v", urlStr, err)
	}

	return runtimevar.New(newWatcher(client, endpointURL, decoder, opts)), nil
}

type state struct {
	val        interface{}
	raw        *http.Response
	rawBytes   []byte
	updateTime time.Time
	err        error
}

// Value implements driver.State.Value.
func (s *state) Value() (interface{}, error) {
	return s.val, s.err
}

// UpdateTime implements driver.State.UpdateTime.
func (s *state) UpdateTime() time.Time {
	return s.updateTime
}

// As implements driver.State.As.
func (s *state) As(i interface{}) bool {
	if s.raw == nil {
		return false
	}
	p, ok := i.(**http.Response)
	if !ok {
		return false
	}
	*p = s.raw
	return true
}

// errorState returns a new State with err, unless prevS also represents
// the same error, in which case it returns nil.
func errorState(err error, prevS driver.State) driver.State {
	s := &state{err: err}
	if prevS == nil {
		return s
	}
	prev := prevS.(*state)
	if prev.err == nil {
		// New error.
		return s
	}
	if equivalentError(err, prev.err) {
		// Same error, return nil to indicate no change.
		return nil
	}
	return s
}

// equivalentError returns true if err1 and err2 represent an equivalent error;
// i.e., we don't want to return it to the user as a different error.
func equivalentError(err1, err2 error) bool {
	if err1 == err2 || err1.Error() == err2.Error() {
		return true
	}
	var code1, code2 int
	if e, ok := err1.(*RequestError); ok {
		code1 = e.Response.StatusCode
	}
	if e, ok := err2.(*RequestError); ok {
		code2 = e.Response.StatusCode
	}
	return code1 != 0 && code1 == code2
}

// watcher implements driver.Watcher for configurations provided by the Runtime Configurator
// service.
type watcher struct {
	client   *http.Client
	endpoint *url.URL
	decoder  *runtimevar.Decoder
	wait     time.Duration
}

// WatchVariable implements driver.WatchVariable.
func (w *watcher) WatchVariable(ctx context.Context, prev driver.State) (driver.State, time.Duration) {
	resp, err := w.client.Get(w.endpoint.String())
	if err != nil {
		return errorState(err, prev), w.wait
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err := newRequestError(resp)
		return errorState(err, prev), w.wait
	}

	respBodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errorState(err, prev), w.wait
	}

	// When endpoint returns the same response again, we return nil as state to not trigger variable update.
	if prev != nil && bytes.Equal(respBodyBytes, prev.(*state).rawBytes) {
		return nil, w.wait
	}

	val, err := w.decoder.Decode(ctx, respBodyBytes)
	if err != nil {
		return errorState(err, prev), w.wait
	}

	return &state{
		val:        val,
		raw:        resp,
		rawBytes:   respBodyBytes,
		updateTime: time.Now(),
	}, w.wait
}

// Close implements driver.Close.
func (w *watcher) Close() error {
	return nil
}

// ErrorAs implements driver.ErrorAs.
func (w *watcher) ErrorAs(err error, i interface{}) bool {
	switch v := err.(type) {
	case *url.Error:
		if p, ok := i.(*url.Error); ok {
			*p = *v
			return true
		}
	case *RequestError:
		if p, ok := i.(*RequestError); ok {
			*p = *v
			return true
		}
	}
	return false
}

// ErrorCode implements driver.ErrorCode.
func (*watcher) ErrorCode(err error) gcerrors.ErrorCode {
	if requestErr, ok := err.(*RequestError); ok {
		switch requestErr.Response.StatusCode {
		case http.StatusBadRequest:
			return gcerr.InvalidArgument
		case http.StatusNotFound:
			return gcerr.NotFound
		case http.StatusUnauthorized:
			return gcerr.PermissionDenied
		case http.StatusGatewayTimeout, http.StatusRequestTimeout:
			return gcerr.DeadlineExceeded
		case http.StatusInternalServerError, http.StatusServiceUnavailable, http.StatusBadGateway:
			return gcerr.Internal
		}
	}
	return gcerr.Unknown
}

func newWatcher(client *http.Client, endpoint *url.URL, decoder *runtimevar.Decoder, opts *Options) driver.Watcher {
	if opts == nil {
		opts = &Options{}
	}
	return &watcher{
		client:   client,
		endpoint: endpoint,
		decoder:  decoder,
		wait:     driver.WaitDuration(opts.WaitDuration),
	}
}
<commit_msg>runtimevar/httpvar: use context when watching values (#2834)<commit_after>// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package httpvar provides a runtimevar implementation with variables
// backed by http endpoint. Use OpenVariable to construct a *runtimevar.Variable.
//
// URLs
//
// For runtimevar.OpenVariable, httpvar registers for the schemes "http" and
// "https". The default URL opener will use http.DefaultClient.
// To customize the URL opener, or for more details on the URL format,
// see URLOpener.
// See https://gocloud.dev/concepts/urls/ for background information.
//
// As
//
// httpvar exposes the following types for As:
//  - Snapshot: *http.Response
//  - Error: httpvar.RequestError, url.Error
package httpvar // import "gocloud.dev/runtimevar/httpvar"

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"gocloud.dev/gcerrors"
	"gocloud.dev/internal/gcerr"
	"gocloud.dev/runtimevar"
	"gocloud.dev/runtimevar/driver"
)

func init() {
	o := &URLOpener{Client: http.DefaultClient}
	for _, scheme := range Schemes {
		runtimevar.DefaultURLMux().RegisterVariable(scheme, o)
	}
}

// Schemes are the URL schemes httpvar registers its URLOpener under on runtimevar.DefaultMux.
var Schemes = []string{"http", "https"}

// URLOpener opens HTTP URLs like "http://myserver.com/foo.txt".
//
// The full URL, including scheme, is used as the endpoint, except that the
// the following URL parameters are removed if present:
//   - decoder: The decoder to use. Defaults to runtimevar.BytesDecoder.
//       See runtimevar.DecoderByName for supported values.
type URLOpener struct {
	// The Client to use; required.
	Client *http.Client

	// Decoder specifies the decoder to use if one is not specified in the URL.
	// Defaults to runtimevar.BytesDecoder.
	Decoder *runtimevar.Decoder

	// Options specifies the options to pass to OpenVariable.
	Options Options
}

// OpenVariableURL opens a httpvar Variable for u.
func (o *URLOpener) OpenVariableURL(ctx context.Context, u *url.URL) (*runtimevar.Variable, error) {
	// Clone u because we may strip some query parameters.
	u2 := *u
	q := u2.Query()

	decoderName := q.Get("decoder")
	q.Del("decoder")
	decoder, err := runtimevar.DecoderByName(ctx, decoderName, o.Decoder)
	if err != nil {
		return nil, fmt.Errorf("open variable %v: invalid decoder: %v", u, err)
	}
	// See if we changed the query parameters.
	if rawq := q.Encode(); rawq != u.Query().Encode() {
		u2.RawQuery = rawq
	}
	return OpenVariable(o.Client, u2.String(), decoder, &o.Options)
}

// Options sets options.
type Options struct {
	// WaitDuration controls the rate at which the HTTP endpoint is called to check for changes.
	// Defaults to 30 seconds.
	WaitDuration time.Duration
}

// RequestError represents an HTTP error that occurred during endpoint call.
type RequestError struct {
	Response *http.Response
}

func (e *RequestError) Error() string {
	return fmt.Sprintf("httpvar: received status code %d", e.Response.StatusCode)
}

func newRequestError(response *http.Response) *RequestError {
	return &RequestError{Response: response}
}

// OpenVariable constructs a *runtimevar.Variable that uses client
// to retrieve the variable contents from the URL urlStr.
func OpenVariable(client *http.Client, urlStr string, decoder *runtimevar.Decoder, opts *Options) (*runtimevar.Variable, error) {
	endpointURL, err := url.Parse(urlStr)
	if err != nil {
		return nil, fmt.Errorf("httpvar: failed to parse url %q: %v", urlStr, err)
	}

	return runtimevar.New(newWatcher(client, endpointURL, decoder, opts)), nil
}

type state struct {
	val        interface{}
	raw        *http.Response
	rawBytes   []byte
	updateTime time.Time
	err        error
}

// Value implements driver.State.Value.
func (s *state) Value() (interface{}, error) {
	return s.val, s.err
}

// UpdateTime implements driver.State.UpdateTime.
func (s *state) UpdateTime() time.Time {
	return s.updateTime
}

// As implements driver.State.As.
func (s *state) As(i interface{}) bool {
	if s.raw == nil {
		return false
	}
	p, ok := i.(**http.Response)
	if !ok {
		return false
	}
	*p = s.raw
	return true
}

// errorState returns a new State with err, unless prevS also represents
// the same error, in which case it returns nil.
func errorState(err error, prevS driver.State) driver.State {
	s := &state{err: err}
	if prevS == nil {
		return s
	}
	prev := prevS.(*state)
	if prev.err == nil {
		// New error.
		return s
	}
	if equivalentError(err, prev.err) {
		// Same error, return nil to indicate no change.
		return nil
	}
	return s
}

// equivalentError returns true if err1 and err2 represent an equivalent error;
// i.e., we don't want to return it to the user as a different error.
func equivalentError(err1, err2 error) bool {
	if err1 == err2 || err1.Error() == err2.Error() {
		return true
	}
	var code1, code2 int
	if e, ok := err1.(*RequestError); ok {
		code1 = e.Response.StatusCode
	}
	if e, ok := err2.(*RequestError); ok {
		code2 = e.Response.StatusCode
	}
	return code1 != 0 && code1 == code2
}

// watcher implements driver.Watcher for configurations provided by the Runtime Configurator
// service.
type watcher struct {
	client   *http.Client
	endpoint *url.URL
	decoder  *runtimevar.Decoder
	wait     time.Duration
}

// WatchVariable implements driver.WatchVariable.
func (w *watcher) WatchVariable(ctx context.Context, prev driver.State) (driver.State, time.Duration) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, w.endpoint.String(), nil)
	if err != nil {
		return errorState(err, prev), w.wait
	}
	resp, err := w.client.Do(req)
	if err != nil {
		return errorState(err, prev), w.wait
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err := newRequestError(resp)
		return errorState(err, prev), w.wait
	}

	respBodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errorState(err, prev), w.wait
	}

	// When endpoint returns the same response again, we return nil as state to not trigger variable update.
	if prev != nil && bytes.Equal(respBodyBytes, prev.(*state).rawBytes) {
		return nil, w.wait
	}

	val, err := w.decoder.Decode(ctx, respBodyBytes)
	if err != nil {
		return errorState(err, prev), w.wait
	}

	return &state{
		val:        val,
		raw:        resp,
		rawBytes:   respBodyBytes,
		updateTime: time.Now(),
	}, w.wait
}

// Close implements driver.Close.
func (w *watcher) Close() error {
	return nil
}

// ErrorAs implements driver.ErrorAs.
func (w *watcher) ErrorAs(err error, i interface{}) bool {
	switch v := err.(type) {
	case *url.Error:
		if p, ok := i.(*url.Error); ok {
			*p = *v
			return true
		}
	case *RequestError:
		if p, ok := i.(*RequestError); ok {
			*p = *v
			return true
		}
	}
	return false
}

// ErrorCode implements driver.ErrorCode.
func (*watcher) ErrorCode(err error) gcerrors.ErrorCode {
	if requestErr, ok := err.(*RequestError); ok {
		switch requestErr.Response.StatusCode {
		case http.StatusBadRequest:
			return gcerr.InvalidArgument
		case http.StatusNotFound:
			return gcerr.NotFound
		case http.StatusUnauthorized:
			return gcerr.PermissionDenied
		case http.StatusGatewayTimeout, http.StatusRequestTimeout:
			return gcerr.DeadlineExceeded
		case http.StatusInternalServerError, http.StatusServiceUnavailable, http.StatusBadGateway:
			return gcerr.Internal
		}
	}
	return gcerr.Unknown
}

func newWatcher(client *http.Client, endpoint *url.URL, decoder *runtimevar.Decoder, opts *Options) driver.Watcher {
	if opts == nil {
		opts = &Options{}
	}
	return &watcher{
		client:   client,
		endpoint: endpoint,
		decoder:  decoder,
		wait:     driver.WaitDuration(opts.WaitDuration),
	}
}
<|endoftext|>