<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package race_test

import (
	"testing"
)

func TestRaceMapRW(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRW2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRWArray(t *testing.T) {
	// Check instrumentation of unaddressable arrays (issue 4578).
	m := make(map[int][2]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1][1]
		ch <- true
	}()
	m[2] = [2]int{1, 2}
	<-ch
}

func TestNoRaceMapRR(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	_ = m[1]
	<-ch
}

func TestRaceMapRange(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRange2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestNoRaceMapRangeRange(t *testing.T) {
	m := make(map[int]int)
	// now the map is not empty and range triggers an event
	// should work without this (as in other tests)
	// so it is suspicious if this test passes and others don't
	m[0] = 0
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	for range m {
	}
	<-ch
}

func TestRaceMapLen(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		_ = len(m)
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "")
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapLenDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "a")
		ch <- true
	}()
	_ = len(m)
	<-ch
}

func TestRaceMapVariable(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m = make(map[int]int)
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable2(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m[1] = 1
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable3(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

type Big struct {
	x [17]int32
}

func TestRaceMapLookupPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_ = m[*k]
	<-ch
}

func TestRaceMapLookupPartKey2(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_, _ = m[*k]
	<-ch
}
func TestRaceMapDeletePartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	delete(m, *k)
	<-ch
}

func TestRaceMapInsertPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	m[*k] = true
	<-ch
}

func TestRaceMapInsertPartVal(t *testing.T) {
	v := &Big{}
	m := make(map[int]Big)
	ch := make(chan bool, 1)
	go func() {
		v.x[8] = 1
		ch <- true
	}()
	m[1] = *v
	<-ch
}

// Test for issue 7561.
func TestRaceMapAssignMultipleReturn(t *testing.T) {
	connect := func() (int, error) { return 42, nil }
	conns := make(map[int][]int)
	conns[1] = []int{0}
	ch := make(chan bool, 1)
	var err error
	go func() {
		conns[1][0], err = connect()
		ch <- true
	}()
	x := conns[1][0]
	_ = x
	<-ch
}
<commit_msg>runtime/race: add tests for maps with big keys/vals With the recent GC changes large objects are handled somewhat differently.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package race_test

import (
	"testing"
)

func TestRaceMapRW(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRW2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRWArray(t *testing.T) {
	// Check instrumentation of unaddressable arrays (issue 4578).
	m := make(map[int][2]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1][1]
		ch <- true
	}()
	m[2] = [2]int{1, 2}
	<-ch
}

func TestNoRaceMapRR(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	_ = m[1]
	<-ch
}

func TestRaceMapRange(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRange2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestNoRaceMapRangeRange(t *testing.T) {
	m := make(map[int]int)
	// now the map is not empty and range triggers an event
	// should work without this (as in other tests)
	// so it is suspicious if this test passes and others don't
	m[0] = 0
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	for range m {
	}
	<-ch
}

func TestRaceMapLen(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		_ = len(m)
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "")
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapLenDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "a")
		ch <- true
	}()
	_ = len(m)
	<-ch
}

func TestRaceMapVariable(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m = make(map[int]int)
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable2(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m[1] = 1
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable3(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

type Big struct {
	x [17]int32
}

func TestRaceMapLookupPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_ = m[*k]
	<-ch
}

func TestRaceMapLookupPartKey2(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_, _ = m[*k]
	<-ch
}
func TestRaceMapDeletePartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	delete(m, *k)
	<-ch
}

func TestRaceMapInsertPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	m[*k] = true
	<-ch
}

func TestRaceMapInsertPartVal(t *testing.T) {
	v := &Big{}
	m := make(map[int]Big)
	ch := make(chan bool, 1)
	go func() {
		v.x[8] = 1
		ch <- true
	}()
	m[1] = *v
	<-ch
}

// Test for issue 7561.
func TestRaceMapAssignMultipleReturn(t *testing.T) {
	connect := func() (int, error) { return 42, nil }
	conns := make(map[int][]int)
	conns[1] = []int{0}
	ch := make(chan bool, 1)
	var err error
	go func() {
		conns[1][0], err = connect()
		ch <- true
	}()
	x := conns[1][0]
	_ = x
	<-ch
}

// BigKey and BigVal must be larger than 256 bytes,
// so that compiler sets KindGCProg for them.
type BigKey [1000]*int

type BigVal struct {
	x int
	y [1000]*int
}

func TestRaceMapBigKeyAccess1(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		_ = m[k]
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyAccess2(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[k]
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyInsert(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		m[k] = 1
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyDelete(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		delete(m, k)
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigValInsert(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		m[1] = v
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}

func TestRaceMapBigValAccess1(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		v = m[1]
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}

func TestRaceMapBigValAccess2(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		v, _ = m[1]
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}
<|endoftext|>


<commit_before>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*"

package gcrc32_test

import (
	"testing"

	"github.com/gogf/gf/g/crypto/gcrc32"
	"github.com/gogf/gf/g/test/gtest"
)

func TestEncryptString(t *testing.T) {
	gtest.Case(t, func() {
		s := "pibigstar"
		result := 693191136
		encrypt1 := gcrc32.EncryptString(s)
		encrypt2 := gcrc32.EncryptBytes([]byte(s))
		gtest.AssertEQ(int(encrypt1), result)
		gtest.AssertEQ(int(encrypt2), result)
	})
}

func TestEncrypt(t *testing.T) {
	gtest.Case(t, func() {
		s := "pibigstar"
		result := 693191136
		encrypt1 := gcrc32.Encrypt(s)
		encrypt2 := gcrc32.Encrypt([]byte(s))
		gtest.AssertEQ(int(encrypt1), result)
		gtest.AssertEQ(int(encrypt2), result)
	})
}
<commit_msg>add gcrc test2<commit_after>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*"

package gcrc32_test

import (
	"github.com/gogf/gf/g/crypto/gmd5"
	"testing"

	"github.com/gogf/gf/g/crypto/gcrc32"
	"github.com/gogf/gf/g/test/gtest"
)

func TestEncryptString(t *testing.T) {
	gtest.Case(t, func() {
		s := "pibigstar"
		result := 693191136
		encrypt1 := gcrc32.EncryptString(s)
		encrypt2 := gcrc32.EncryptBytes([]byte(s))
		gtest.AssertEQ(int(encrypt1), result)
		gtest.AssertEQ(int(encrypt2), result)
	})
}

func TestEncrypt(t *testing.T) {
	gtest.Case(t, func() {
		s := "pibigstar"
		result := 693191136
		encrypt1 := gcrc32.Encrypt(s)
		encrypt2 := gcrc32.Encrypt([]byte(s))
		gtest.AssertEQ(int(encrypt1), result)
		gtest.AssertEQ(int(encrypt2), result)

		strmd5 := gmd5.Encrypt(s)
		test1 := gcrc32.Encrypt(strmd5)
		test2 := gcrc32.Encrypt([]byte(strmd5))
		gtest.AssertEQ(test2, test1)
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// go test *.go -bench=".*"

package groutine_test

import (
    "testing"
    "gitee.com/johng/gf/g/os/groutine"
)

func test() {
    num := 0
    for i := 0; i < 1000000; i++ {
        num += i
    }
}

func BenchmarkGroutine(b *testing.B) {
    for i := 0; i < b.N; i++ {
        groutine.Add(test)
    }
}

func BenchmarkGoRoutine(b *testing.B) {
    for i := 0; i < b.N; i++ {
        go test()
    }
}<commit_msg>groutine性能测试用例<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// go test *.go -bench=".*"

package groutine_test

import (
    "testing"
    "gitee.com/johng/gf/g/os/groutine"
)

func increment() {
    for i := 0; i < 1000000; i++ {}
}

func BenchmarkGroutine(b *testing.B) {
    for i := 0; i < b.N; i++ {
        groutine.Add(increment)
    }
}

func BenchmarkGoRoutine(b *testing.B) {
    for i := 0; i < b.N; i++ {
        go increment()
    }
}<|endoftext|>


<commit_before>/*
 * Copyright 2015 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package web defines utility functions for exposing services over HTTP.
package web

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"kythe.io/kythe/go/util/httpencoding"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
)

const jsonBodyType = "application/json; charset=utf-8"

// JSONMarshaler is the marshaler used to encode all JSON web requests.
var JSONMarshaler = jsonpb.Marshaler{
	OrigName: true,
}

// RegisterQuitHandler adds a handler for /quitquitquit that call os.Exit(0).
func RegisterQuitHandler(mux *http.ServeMux) {
	mux.HandleFunc("/quitquitquit", func(w http.ResponseWriter, r *http.Request) {
		os.Exit(0)
	})
}

// Call sends req to the given server method as a JSON-encoded body and
// unmarshals the response body as JSON into reply.
func Call(server, method string, req, reply proto.Message) error {
	body := new(bytes.Buffer)
	if err := JSONMarshaler.Marshal(body, req); err != nil {
		return fmt.Errorf("error marshaling %T: %v", req, err)
	}
	resp, err := http.Post(strings.TrimSuffix(server, "/")+"/"+strings.Trim(method, "/"),
		jsonBodyType, body)
	if err != nil {
		return fmt.Errorf("http error: %v", err)
	}
	rec, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	} else if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("remote method error (code %d): %s", resp.StatusCode, string(rec))
	}
	if err := jsonpb.UnmarshalString(string(rec), reply); err != nil {
		return fmt.Errorf("error unmarshaling %T: %v", reply, err)
	}
	return nil
}

// ReadJSONBody reads the entire body of r and unmarshals it from JSON into msg.
// If the request body is empty, no error is returned and msg is unchanged.
func ReadJSONBody(r *http.Request, msg proto.Message) error {
	rec, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return fmt.Errorf("body read error: %v", err)
	}
	if len(rec) == 0 {
		return nil
	}
	return jsonpb.UnmarshalString(string(rec), msg)
}

// WriteResponse writes msg to w as a serialized protobuf if the "proto" query
// parameter is set; otherwise as JSON.
func WriteResponse(w http.ResponseWriter, r *http.Request, msg proto.Message) error {
	if Arg(r, "proto") != "" {
		return WriteProtoResponse(w, r, msg)
	}
	return WriteJSONResponse(w, r, msg)
}

// WriteJSONResponse encodes v as JSON and writes it to w.
func WriteJSONResponse(w http.ResponseWriter, r *http.Request, v interface{}) error {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	cw := httpencoding.CompressData(w, r)
	defer cw.Close()
	return json.NewEncoder(cw).Encode(v)
}

// WriteProtoResponse serializes msg to w.
func WriteProtoResponse(w http.ResponseWriter, r *http.Request, msg proto.Message) error {
	w.Header().Set("Content-Type", "application/x-protobuf")
	cw := httpencoding.CompressData(w, r)
	defer cw.Close()
	rec, err := proto.Marshal(msg)
	if err != nil {
		return fmt.Errorf("error marshaling proto: %v", err)
	}
	_, err = cw.Write(rec)
	return err
}

// Arg returns the first query value for the named parameter or "" if it was not
// set.
func Arg(r *http.Request, name string) string {
	args := r.URL.Query()[name]
	if len(args) == 0 {
		return ""
	}
	return args[0]
}

// ArgOrNil returns a pointer to first query value for the named parameter or
// nil if it was not set.
func ArgOrNil(r *http.Request, name string) *string {
	arg := Arg(r, name)
	if arg == "" {
		return nil
	}
	return &arg
}
<commit_msg>fix(web_api): properly marshal protos with jsonpb (#3424)<commit_after>/*
 * Copyright 2015 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package web defines utility functions for exposing services over HTTP.
package web

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"kythe.io/kythe/go/util/httpencoding"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
)

const jsonBodyType = "application/json; charset=utf-8"

// JSONMarshaler is the marshaler used to encode all JSON web requests.
var JSONMarshaler = jsonpb.Marshaler{
	OrigName: true,
}

// RegisterQuitHandler adds a handler for /quitquitquit that call os.Exit(0).
func RegisterQuitHandler(mux *http.ServeMux) {
	mux.HandleFunc("/quitquitquit", func(w http.ResponseWriter, r *http.Request) {
		os.Exit(0)
	})
}

// Call sends req to the given server method as a JSON-encoded body and
// unmarshals the response body as JSON into reply.
func Call(server, method string, req, reply proto.Message) error {
	body := new(bytes.Buffer)
	if err := JSONMarshaler.Marshal(body, req); err != nil {
		return fmt.Errorf("error marshaling %T: %v", req, err)
	}
	resp, err := http.Post(strings.TrimSuffix(server, "/")+"/"+strings.Trim(method, "/"),
		jsonBodyType, body)
	if err != nil {
		return fmt.Errorf("http error: %v", err)
	}
	rec, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	} else if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("remote method error (code %d): %s", resp.StatusCode, string(rec))
	}
	if err := jsonpb.UnmarshalString(string(rec), reply); err != nil {
		return fmt.Errorf("error unmarshaling %T: %v", reply, err)
	}
	return nil
}

// ReadJSONBody reads the entire body of r and unmarshals it from JSON into msg.
// If the request body is empty, no error is returned and msg is unchanged.
func ReadJSONBody(r *http.Request, msg proto.Message) error {
	rec, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return fmt.Errorf("body read error: %v", err)
	}
	if len(rec) == 0 {
		return nil
	}
	return jsonpb.UnmarshalString(string(rec), msg)
}

// WriteResponse writes msg to w as a serialized protobuf if the "proto" query
// parameter is set; otherwise as JSON.
func WriteResponse(w http.ResponseWriter, r *http.Request, msg proto.Message) error {
	if Arg(r, "proto") != "" {
		return WriteProtoResponse(w, r, msg)
	}
	return WriteJSONResponse(w, r, msg)
}

// WriteJSONResponse encodes v as JSON and writes it to w.
func WriteJSONResponse(w http.ResponseWriter, r *http.Request, v interface{}) error {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	cw := httpencoding.CompressData(w, r)
	defer cw.Close()
	if msg, ok := v.(proto.Message); ok {
		return JSONMarshaler.Marshal(cw, msg)
	}
	return json.NewEncoder(cw).Encode(v)
}

// WriteProtoResponse serializes msg to w.
func WriteProtoResponse(w http.ResponseWriter, r *http.Request, msg proto.Message) error {
	w.Header().Set("Content-Type", "application/x-protobuf")
	cw := httpencoding.CompressData(w, r)
	defer cw.Close()
	rec, err := proto.Marshal(msg)
	if err != nil {
		return fmt.Errorf("error marshaling proto: %v", err)
	}
	_, err = cw.Write(rec)
	return err
}

// Arg returns the first query value for the named parameter or "" if it was not
// set.
func Arg(r *http.Request, name string) string {
	args := r.URL.Query()[name]
	if len(args) == 0 {
		return ""
	}
	return args[0]
}

// ArgOrNil returns a pointer to first query value for the named parameter or
// nil if it was not set.
func ArgOrNil(r *http.Request, name string) *string {
	arg := Arg(r, name)
	if arg == "" {
		return nil
	}
	return &arg
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/gorilla/mux"

	"github.com/lcsontos/uwatch/webservice"
)

func init() {
	router := mux.NewRouter().StrictSlash(true)

	router.HandleFunc("/api/long_video_url/{videoType}/{videoId}", webservice.GetLongVideoUrl).Methods("GET")
	router.HandleFunc("/api/parse_video_url/{videoUrl}", webservice.GetParseVideoUrl).Methods("GET")

	http.Handle("/", router)
}
<commit_msg>Add panic handler<commit_after>package main

import (
	"log"
	"net/http"
	"runtime"

	"github.com/gorilla/mux"

	"github.com/lcsontos/uwatch/webservice"
)

type handlerFuncType func(http.ResponseWriter, *http.Request)

func handleSafely(handlerFunc handlerFuncType) handlerFuncType {
	return func(rw http.ResponseWriter, req *http.Request) {
		// Recover Panic
		defer func() {
			if err := recover(); err != nil {
				panicHandler(err, rw, req)
			}
		}()

		// Call original handler
		handlerFunc(rw, req)
	}
}

func init() {
	router := mux.NewRouter().StrictSlash(true)

	router.HandleFunc(
		"/api/long_video_url/{videoType}/{videoId}",
		handleSafely(webservice.GetLongVideoUrl)).Methods("GET")

	router.HandleFunc(
		"/api/parse_video_url/{videoUrl}",
		handleSafely(webservice.GetParseVideoUrl)).Methods("GET")

	http.Handle("/", router)
}

func panicHandler(err interface{}, rw http.ResponseWriter, req *http.Request) {
	var stack [4096]byte

	runtime.Stack(stack[:], false)

	log.Printf(
		"Handler for %s[url:%s, data:%s] has failed with %s\nStack trace:\n %s\n",
		req.Method, req.URL, req.Form, err, stack[:])

	http.Error(rw, "SYSTEM ERROR", http.StatusInternalServerError)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	pathpkg "path"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"

	"github.com/charlievieth/pkgs"
)

type mImportPaths struct {
	Fn            string
	Src           string
	Env           map[string]string
	InstallSuffix string
}

type mImportPathsDecl struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type mImportPathsDeclByName []mImportPathsDecl

func (m mImportPathsDeclByName) Len() int           { return len(m) }
func (m mImportPathsDeclByName) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m mImportPathsDeclByName) Less(i, j int) bool { return m[i].Name < m[j].Name }

type mImportPathsResponse struct {
	Imports []mImportPathsDecl `json:"imports"`
	Paths   []string           `json:"paths"`
}

func (m *mImportPaths) FileImports() ([]mImportPathsDecl, error) {
	fset := token.NewFileSet()
	af, err := parser.ParseFile(fset, m.Fn, m.Src, parser.ImportsOnly)
	if err != nil {
		return nil, err
	}

	imports := make([]mImportPathsDecl, 0, 8)
	for _, decl := range af.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, dspec := range d.Specs {
			spec, ok := dspec.(*ast.ImportSpec)
			if !ok {
				continue
			}
			quoted := spec.Path.Value
			path, err := strconv.Unquote(quoted)
			if err != nil {
				return nil, fmt.Errorf("%s: parser returned invalid quoted string: <%s>", m.Fn, quoted)
			}
			var name string
			if spec.Name != nil {
				name = spec.Name.String()
			} else {
				name = pathpkg.Base(path)
			}
			imports = append(imports, mImportPathsDecl{
				Path: path,
				Name: name,
			})
		}
	}

	return imports, nil
}

func (m *mImportPaths) Call() (interface{}, string) {
	imports, err := m.FileImports()
	if err != nil {
		return M{}, errStr(err)
	}

	names, err := importPaths(m.Env, m.InstallSuffix, filepath.Dir(m.Fn))
	if err != nil {
		return M{}, errStr(err)
	}

	sort.Strings(names)
	// dedupe since there may be duplicate vendored imports
	if len(names) > 0 {
		i := 0
		s := ""
		for _, x := range names {
			if x != s {
				names[i] = x
				s = x
				i++
			}
		}
		names = names[:i]
	}
	sort.Sort(mImportPathsDeclByName(imports))

	return &mImportPathsResponse{Imports: imports, Paths: names}, ""
}

func importPaths(environ map[string]string, installSuffix, importDir string) ([]string, error) {
	// TODO:
	// 	- Consider adding os.GOROOT and os.GOPATH to environ
	// 	- Check for duplicate paths
	var root string
	if s := environ["GOROOT"]; s != "" {
		root = s
	} else {
		root = runtime.GOROOT()
	}
	var path string
	if s := environ["GOPATH"]; s != "" {
		path = s
	} else {
		path = os.Getenv("GOPATH")
	}
	ctxt := build.Default
	if root != "" {
		ctxt.GOROOT = root
	}
	if path != "" {
		ctxt.GOPATH = path
	}
	if installSuffix != "" {
		ctxt.InstallSuffix = installSuffix
	}

	return pkgs.Walk(&ctxt, importDir)
}

func init() {
	registry.Register("import_paths", func(_ *Broker) Caller {
		return &mImportPaths{
			Env: map[string]string{},
		}
	})
}
<commit_msg>m_import_paths: fix caching logic<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	pathpkg "path"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/charlievieth/pkgs"
)

type mImportPaths struct {
	Fn            string
	Src           string
	Env           map[string]string
	InstallSuffix string
}

type mImportPathsDecl struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type mImportPathsDeclByName []mImportPathsDecl

func (m mImportPathsDeclByName) Len() int           { return len(m) }
func (m mImportPathsDeclByName) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m mImportPathsDeclByName) Less(i, j int) bool { return m[i].Name < m[j].Name }

type mImportPathsResponse struct {
	Imports []mImportPathsDecl `json:"imports"`
	Paths   []string           `json:"paths"`
}

func (m *mImportPaths) FileImports() ([]mImportPathsDecl, error) {
	fset := token.NewFileSet()
	af, err := parser.ParseFile(fset, m.Fn, m.Src, parser.ImportsOnly)
	if err != nil {
		return nil, err
	}

	imports := make([]mImportPathsDecl, 0, 8)
	for _, decl := range af.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, dspec := range d.Specs {
			spec, ok := dspec.(*ast.ImportSpec)
			if !ok {
				continue
			}
			quoted := spec.Path.Value
			path, err := strconv.Unquote(quoted)
			if err != nil {
				return nil, fmt.Errorf("%s: parser returned invalid quoted string: <%s>", m.Fn, quoted)
			}
			var name string
			if spec.Name != nil {
				name = spec.Name.String()
			} else {
				name = pathpkg.Base(path)
			}
			imports = append(imports, mImportPathsDecl{
				Path: path,
				Name: name,
			})
		}
	}

	return imports, nil
}

func (m *mImportPaths) Call() (interface{}, string) {
	imports, err := m.FileImports()
	if err != nil {
		return M{}, errStr(err)
	}

	names, err := importPaths(m.Env, m.InstallSuffix, filepath.Dir(m.Fn))
	if err != nil && len(names) == 0 {
		return M{}, errStr(err)
	}

	// dedupe since there may be duplicate vendored imports
	// names is sorted
	if len(names) > 0 {
		i := 0
		s := ""
		for _, x := range names {
			if x != s {
				names[i] = x
				s = x
				i++
			}
		}
		names = names[:i]
	}
	sort.Sort(mImportPathsDeclByName(imports))

	return &mImportPathsResponse{Imports: imports, Paths: names}, ""
}

type importsPathCacheEntry struct {
	Created time.Time
	Imports []string
}

// project root => *importsPathCacheEntry
var importsPathCache sync.Map

func init() {
	const TTL = time.Minute * 2

	go func() {
		for {
			time.Sleep(TTL / 4)

			importsPathCache.Range(func(key, value interface{}) bool {
				if e, ok := value.(*importsPathCacheEntry); ok {
					if time.Since(e.Created) > TTL {
						importsPathCache.Delete(key)
					}
				}
				return true
			})
		}
	}()
}

func isRoot(dir string) bool {
	// TODO: add ".svn" ".hg" ???
	for _, name := range []string{"vendor", "go.mod", ".git"} {
		if _, err := os.Lstat(dir + "/" + name); err == nil {
			return true
		}
	}
	return false
}

func projectRoot(dirname string) string {
	const sep = string(os.PathListSeparator)

	// special case for me
	for _, path := range strings.Split(build.Default.GOPATH, sep) {
		pfx := filepath.Join(path, "src", "repl")
		if strings.HasPrefix(dirname, pfx) {
			return pfx
		}
	}

	dir := filepath.ToSlash(dirname)
	for !isRoot(dir) {
		next := filepath.Dir(dir)
		if next == dir {
			break
		}
		dir = next
	}
	return dir
}

func importPaths(environ map[string]string, installSuffix, importDir string) ([]string, error) {
	cacheRoot := projectRoot(importDir)
	if v, ok := importsPathCache.Load(cacheRoot); ok {
		if e, _ := v.(*importsPathCacheEntry); e != nil {
			a := make([]string, len(e.Imports))
			copy(a, e.Imports)
			return a, nil
		}
	}

	// TODO:
	// 	- Consider adding os.GOROOT and os.GOPATH to environ
	// 	- Check for duplicate paths
	var root string
	if s := environ["GOROOT"]; s != "" {
		root = s
	} else {
		root = runtime.GOROOT()
	}
	var path string
	if s := environ["GOPATH"]; s != "" {
		path = s
	} else {
		path = os.Getenv("GOPATH")
	}
	ctxt := build.Default
	if root != "" {
		ctxt.GOROOT = root
	}
	if path != "" {
		ctxt.GOPATH = path
	}
	if installSuffix != "" {
		ctxt.InstallSuffix = installSuffix
	}

	paths, err := pkgs.Walk(&ctxt, importDir)
	if len(paths) != 0 {
		sort.Strings(paths)
	}
	if err != nil {
		return paths, err
	}

	importsPathCache.Store(cacheRoot, &importsPathCacheEntry{
		Created: time.Now(),
		Imports: append([]string(nil), paths...),
	})
	return paths, nil
}

func init() {
	registry.Register("import_paths", func(_ *Broker) Caller {
		return &mImportPaths{
			Env: map[string]string{},
		}
	})
}
<|endoftext|>


<commit_before>package collector

import (
	"autoscaler/collection"
	"autoscaler/db"
	"autoscaler/helpers"
	"autoscaler/models"
	"sync"
	"time"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
)

type MetricQueryFunc func(string, int64, int64, db.OrderType, map[string]string) ([]*models.AppInstanceMetric, bool)

type AppCollector interface {
	Start()
	Stop()
	Query(int64, int64, map[string]string) ([]collection.TSD, bool)
}

type Collector struct {
	refreshInterval    time.Duration
	collectInterval    time.Duration
	saveInterval       time.Duration
	nodeNum            int
	nodeIndex          int
	logger             lager.Logger
	policyDb           db.PolicyDB
	instancemetricsDb  db.InstanceMetricsDB
	cclock             clock.Clock
	createAppCollector func(string, chan *models.AppInstanceMetric) AppCollector
	doneChan           chan bool
	doneSaveChan       chan bool
	appCollectors      map[string]AppCollector
	ticker             clock.Ticker
	lock               *sync.RWMutex
	dataChan           chan *models.AppInstanceMetric
}

func NewCollector(refreshInterval time.Duration, collectInterval time.Duration, saveInterval time.Duration,
	nodeIndex, nodeNum int, logger lager.Logger, policyDb db.PolicyDB, instancemetricsDb db.InstanceMetricsDB,
	cclock clock.Clock, createAppCollector func(string, chan *models.AppInstanceMetric) AppCollector) *Collector {
	return &Collector{
		refreshInterval:    refreshInterval,
		collectInterval:    collectInterval,
		saveInterval:       saveInterval,
		nodeIndex:          nodeIndex,
		nodeNum:            nodeNum,
		logger:             logger,
		policyDb:           policyDb,
		instancemetricsDb:  instancemetricsDb,
		cclock:             cclock,
		createAppCollector: createAppCollector,
		doneChan:           make(chan bool),
		doneSaveChan:       make(chan bool),
		appCollectors:      make(map[string]AppCollector),
		lock:               &sync.RWMutex{},
		dataChan:           make(chan *models.AppInstanceMetric),
	}
}

func (c *Collector) Start() {
	c.ticker = c.cclock.NewTicker(c.refreshInterval)
	go c.startAppRefresh()
	go c.SaveMetricsInDB()
	c.logger.Info("collector-started")
}

func (c *Collector) startAppRefresh() {
	for {
		c.refreshApps()
		select {
		case <-c.doneChan:
			return
		case <-c.ticker.C():
		}
	}
}

func (c *Collector) refreshApps() {
	appIds, err := c.policyDb.GetAppIds()
	if err != nil {
		c.logger.Error("refresh-apps", err)
		return
	}

	appShard := map[string]bool{}
	if c.nodeNum == 1 {
		appShard = appIds
	} else {
		for id := range appIds {
			if helpers.FNVHash(id)%uint32(c.nodeNum) == uint32(c.nodeIndex) {
				appShard[id] = true
			}
		}
	}

	c.lock.Lock()
	for id, ap := range c.appCollectors {
		_, exist := appShard[id]
		if !exist {
			c.logger.Debug("refresh-apps-remove", lager.Data{"appId": id})
			ap.Stop()
			delete(c.appCollectors, id)
		}
	}

	for id := range appShard {
		_, exist := c.appCollectors[id]
		if !exist {
			c.logger.Debug("refresh-apps-add", lager.Data{"appId": id})
			ap := c.createAppCollector(id, c.dataChan)
			ap.Start()
			c.appCollectors[id] = ap
		}
	}
	c.lock.Unlock()
}

func (c *Collector) Stop() {
	if c.ticker != nil {
		c.ticker.Stop()
		close(c.doneChan)
		close(c.doneSaveChan)

		c.lock.RLock()
		for _, ap := range c.appCollectors {
			ap.Stop()
		}
		c.lock.RUnlock()
	}
	c.logger.Info("collector-stopped")
}

func (c *Collector) GetCollectorAppIds() []string {
	var appIds []string
	c.lock.RLock()
	for id := range c.appCollectors {
		appIds = append(appIds, id)
	}
	c.lock.RUnlock()
	return appIds
}

func (c *Collector) QueryMetrics(appID string, start, end int64, order db.OrderType, labels map[string]string) ([]*models.AppInstanceMetric, bool) {
	c.lock.RLock()
	appCollector, exist := c.appCollectors[appID]
	c.lock.RUnlock()
	if !exist {
		return nil, false
	}

	result, ok := appCollector.Query(start, end, labels)
	if !ok {
		return nil, false
	}
	metrics := make([]*models.AppInstanceMetric, len(result))
	if order == db.ASC {
		for index, tsd := range result {
			metrics[index] = tsd.(*models.AppInstanceMetric)
		}
	} else {
		for index, tsd := range result {
			metrics[len(result)-1-index] = tsd.(*models.AppInstanceMetric)
		}
	}
	return metrics, true
}

func (c *Collector) SaveMetricsInDB() {
	ticker := c.cclock.NewTicker(c.saveInterval)
	metrics := make([]*models.AppInstanceMetric, 0)
	for {
		select {
		case metric := <-c.dataChan:
			metrics = append(metrics, metric)
		case <-ticker.C():
			go func(instancemetricsDb db.InstanceMetricsDB, metrics []*models.AppInstanceMetric) {
				instancemetricsDb.SaveMetricsInBulk(metrics)
				metrics = nil
				return
			}(c.instancemetricsDb, metrics)
			metrics = nil
		case <-c.doneSaveChan:
			return
		}
	}
}
<commit_msg>fix the race condition in test (#386)<commit_after>package collector

import (
	"autoscaler/collection"
	"autoscaler/db"
	"autoscaler/helpers"
	"autoscaler/models"
	"sync"
	"time"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
)

type MetricQueryFunc func(string, int64, int64, db.OrderType, map[string]string) ([]*models.AppInstanceMetric, bool)

type AppCollector interface {
	Start()
	Stop()
	Query(int64, int64, map[string]string) ([]collection.TSD, bool)
}

type Collector struct {
	refreshInterval    time.Duration
	collectInterval    time.Duration
	saveInterval       time.Duration
	nodeNum            int
	nodeIndex          int
	logger             lager.Logger
	policyDb           db.PolicyDB
	instancemetricsDb  db.InstanceMetricsDB
	cclock             clock.Clock
	createAppCollector func(string, chan *models.AppInstanceMetric) AppCollector
	doneChan           chan bool
	doneSaveChan       chan bool
	appCollectors      map[string]AppCollector
	ticker             clock.Ticker
	lock               *sync.RWMutex
	dataChan           chan *models.AppInstanceMetric
}

func NewCollector(refreshInterval time.Duration, collectInterval time.Duration, saveInterval time.Duration,
	nodeIndex, nodeNum int, logger lager.Logger, policyDb db.PolicyDB, instancemetricsDb db.InstanceMetricsDB,
	cclock clock.Clock, createAppCollector func(string, chan *models.AppInstanceMetric) AppCollector) *Collector {
	return &Collector{
		refreshInterval:    refreshInterval,
		collectInterval:    collectInterval,
		saveInterval:       saveInterval,
		nodeIndex:          nodeIndex,
		nodeNum:            nodeNum,
		logger:             logger,
		policyDb:           policyDb,
		instancemetricsDb:  instancemetricsDb,
		cclock:             cclock,
		createAppCollector: createAppCollector,
		doneChan:           make(chan bool),
		doneSaveChan:       make(chan bool),
		appCollectors:      make(map[string]AppCollector),
		lock:               &sync.RWMutex{},
		dataChan:           make(chan *models.AppInstanceMetric),
	}
}

func (c *Collector) Start() {
	c.ticker = c.cclock.NewTicker(c.refreshInterval)
	go c.startAppRefresh()
	go c.SaveMetricsInDB()
	c.logger.Info("collector-started")
}

func (c *Collector) startAppRefresh() {
	for {
		c.refreshApps()
		select {
		case <-c.doneChan:
			return
		case <-c.ticker.C():
		}
	}
}

func (c *Collector) refreshApps() {
	appIds, err := c.policyDb.GetAppIds()
	if err != nil {
		c.logger.Error("refresh-apps", err)
		return
	}

	appShard := map[string]bool{}
	if c.nodeNum == 1 {
		appShard = appIds
	} else {
		for id := range appIds {
			if helpers.FNVHash(id)%uint32(c.nodeNum) == uint32(c.nodeIndex) {
				appShard[id] = true
			}
		}
	}

	c.lock.Lock()
	for id, ap := range c.appCollectors {
		_, exist := appShard[id]
		if !exist {
			c.logger.Debug("refresh-apps-remove", lager.Data{"appId": id})
			ap.Stop()
			delete(c.appCollectors, id)
		}
	}

	for id := range appShard {
		_, exist := c.appCollectors[id]
		if !exist {
			c.logger.Debug("refresh-apps-add", lager.Data{"appId": id})
			ap := c.createAppCollector(id, c.dataChan)
			ap.Start()
			c.appCollectors[id] = ap
		}
	}
	c.lock.Unlock()
}

func (c *Collector) Stop() {
	if c.ticker != nil {
		c.ticker.Stop()
		c.doneChan <- true
		c.doneSaveChan <- true

		c.lock.RLock()
		for _, ap := range c.appCollectors {
			ap.Stop()
		}
		c.lock.RUnlock()
	}
	c.logger.Info("collector-stopped")
}

func (c *Collector) GetCollectorAppIds() []string {
	var appIds []string
	c.lock.RLock()
	for id := range c.appCollectors {
		appIds = append(appIds, id)
	}
	c.lock.RUnlock()
	return appIds
}

func (c *Collector) QueryMetrics(appID string, start, end int64, order db.OrderType, labels map[string]string) ([]*models.AppInstanceMetric, bool) {
	c.lock.RLock()
	appCollector, exist := c.appCollectors[appID]
	c.lock.RUnlock()
	if !exist {
		return nil, false
	}

	result, ok := appCollector.Query(start, end, labels)
	if !ok {
		return nil, false
	}
	metrics := make([]*models.AppInstanceMetric, len(result))
	if order == db.ASC {
		for index, tsd := range result {
			metrics[index] = tsd.(*models.AppInstanceMetric)
		}
	} else {
		for index, tsd := range result {
			metrics[len(result)-1-index] = tsd.(*models.AppInstanceMetric)
		}
	}
	return metrics, true
}

func (c *Collector) SaveMetricsInDB() {
	ticker := c.cclock.NewTicker(c.saveInterval)
	metrics := make([]*models.AppInstanceMetric, 0)
	for {
		select {
		case metric := <-c.dataChan:
			metrics = append(metrics, metric)
		case <-ticker.C():
			go func(instancemetricsDb db.InstanceMetricsDB, metrics []*models.AppInstanceMetric) {
				instancemetricsDb.SaveMetricsInBulk(metrics)
				metrics = nil
				return
			}(c.instancemetricsDb, metrics)
			metrics = nil
		case <-c.doneSaveChan:
			return
		}
	}
}
<|endoftext|>


<commit_before>package fasthttputil

import (
	"fmt"
	"net"
	"sync"
)

// InmemoryListener provides in-memory dialer<->net.Listener implementation.
//
// It may be used either for fast in-process client<->server communications
// without network stack overhead or for client<->server tests.
type InmemoryListener struct {
	lock   sync.Mutex
	closed bool
	conns  chan net.Conn
}

// NewInmemoryListener returns new in-memory dialer<->net.Listener.
func NewInmemoryListener() *InmemoryListener {
	return &InmemoryListener{
		conns: make(chan net.Conn, 1024),
	}
}

// Accept implements net.Listener's Accept.
//
// It is safe calling Accept from concurrently running goroutines.
//
// Accept returns new connection per each Dial call.
func (ln *InmemoryListener) Accept() (net.Conn, error) {
	c, ok := <-ln.conns
	if !ok {
		return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection")
	}
	return c, nil
}

// Close implements net.Listener's Close.
func (ln *InmemoryListener) Close() error {
	var err error

	ln.lock.Lock()
	if !ln.closed {
		close(ln.conns)
		ln.closed = true
	} else {
		err = fmt.Errorf("InmemoryListener is already closed")
	}
	ln.lock.Unlock()
	return err
}

// Addr implements net.Listener's Addr.
func (ln *InmemoryListener) Addr() net.Addr {
	return &net.UnixAddr{
		Name: "InmemoryListener",
		Net:  "memory",
	}
}

// Dial creates new client<->server connection, enqueues server side
// of the connection to Accept and returns client side of the connection.
//
// It is safe calling Dial from concurrently running goroutines.
func (ln *InmemoryListener) Dial() (net.Conn, error) {
	pc := NewPipeConns()
	cConn := pc.Conn1()
	sConn := pc.Conn2()
	ln.lock.Lock()
	if !ln.closed {
		ln.conns <- sConn
	} else {
		sConn.Close()
		cConn.Close()
		cConn = nil
	}
	ln.lock.Unlock()

	if cConn == nil {
		return nil, fmt.Errorf("InmemoryListener is already closed")
	}
	return cConn, nil
}
<commit_msg>Make InmemoryListener.Dial return when the connection is accepted<commit_after>package fasthttputil

import (
	"fmt"
	"net"
	"sync"
)

// InmemoryListener provides in-memory dialer<->net.Listener implementation.
//
// It may be used either for fast in-process client<->server communications
// without network stack overhead or for client<->server tests.
type InmemoryListener struct {
	lock   sync.Mutex
	closed bool
	conns  chan acceptConn
}

type acceptConn struct {
	conn     net.Conn
	accepted chan struct{}
}

// NewInmemoryListener returns new in-memory dialer<->net.Listener.
func NewInmemoryListener() *InmemoryListener {
	return &InmemoryListener{
		conns: make(chan acceptConn, 1024),
	}
}

// Accept implements net.Listener's Accept.
//
// It is safe calling Accept from concurrently running goroutines.
//
// Accept returns new connection per each Dial call.
func (ln *InmemoryListener) Accept() (net.Conn, error) {
	c, ok := <-ln.conns
	if !ok {
		return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection")
	}
	close(c.accepted)
	return c.conn, nil
}

// Close implements net.Listener's Close.
func (ln *InmemoryListener) Close() error {
	var err error

	ln.lock.Lock()
	if !ln.closed {
		close(ln.conns)
		ln.closed = true
	} else {
		err = fmt.Errorf("InmemoryListener is already closed")
	}
	ln.lock.Unlock()
	return err
}

// Addr implements net.Listener's Addr.
func (ln *InmemoryListener) Addr() net.Addr {
	return &net.UnixAddr{
		Name: "InmemoryListener",
		Net:  "memory",
	}
}

// Dial creates new client<->server connection.
// Just like a real Dial it only returns once the server
// has accepted the connection.
//
// It is safe calling Dial from concurrently running goroutines.
func (ln *InmemoryListener) Dial() (net.Conn, error) {
	pc := NewPipeConns()
	cConn := pc.Conn1()
	sConn := pc.Conn2()
	ln.lock.Lock()
	accepted := make(chan struct{})
	if !ln.closed {
		ln.conns <- acceptConn{sConn, accepted}
		// Wait until the connection has been accepted.
		<-accepted
	} else {
		sConn.Close()
		cConn.Close()
		cConn = nil
	}
	ln.lock.Unlock()

	if cConn == nil {
		return nil, fmt.Errorf("InmemoryListener is already closed")
	}
	return cConn, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package disasm provides functions for disassembling WebAssembly bytecode.
package disasm

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math"

	"github.com/go-interpreter/wagon/internal/stack"
	"github.com/go-interpreter/wagon/wasm"
	"github.com/go-interpreter/wagon/wasm/leb128"
	ops "github.com/go-interpreter/wagon/wasm/operators"
)

// Instr describes an instruction, consisting of an operator, with its
// appropriate immediate value(s).
type Instr struct {
	Op ops.Op

	// Immediates are arguments to an operator in the bytecode stream itself.
	// Valid value types are:
	// - (u)(int/float)(32/64)
	// - wasm.BlockType
	Immediates []interface{}
	NewStack   *StackInfo // non-nil if the instruction requires the current stack to be unwound.
	Block      *BlockInfo // non-nil if the instruction starts a new block.
}

// StackInfo stores details about a new stack ended by an instruction.
type StackInfo struct {
	MaxDepth    int64 // The maximum depth reached
	PreserveTop bool  // Whether the value on the top of the stack should be preserved while unwinding
}

// BlockInfo stores details about a block created or ended by an instruction.
type BlockInfo struct {
	Start     bool           // If true, this instruction starts a block. Else this instruction ends it.
	Signature wasm.BlockType // The block signature
	// The index to the accompanying control operator.
	// For 'if', this is an index to the 'else' operator
	// For else/loop/block, the index is to the 'end' operator
	PairIndex int
}

// Disassembly is the result of disassembling a WebAssembly function.
type Disassembly struct {
	Code     []Instr
	MaxDepth int // The maximum stack depth that can be reached while executing this function
}

func (d *Disassembly) checkMaxDepth(depth int) {
	if depth > d.MaxDepth {
		d.MaxDepth = depth
	}
}

var ErrStackUnderflow = errors.New("disasm: stack underflow")

// Disassemble disassembles the given function. It also takes the function's
// parent module as an argument for locating any other functions referenced by
// fn.
func Disassemble(fn wasm.Function, module *wasm.Module) (*Disassembly, error) {
	code := fn.Body.Code
	reader := bytes.NewReader(code)
	disas := &Disassembly{}

	// a stack of current execution stack depth values, so that the depth for each
	// stack is maintained indepepdently for calculating discard values
	stackDepths := &stack.Stack{}
	stackDepths.Push(0)
	blockIndices := &stack.Stack{} // a stack of indices to operators which start new blocks
	curIndex := 0
	var lastOpReturn bool

	for {
		op, err := reader.ReadByte()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		logger.Printf("stack top is %d", stackDepths.Top())

		opStr, err := ops.New(op)
		if err != nil {
			return nil, err
		}
		instr := Instr{
			Op:         opStr,
			Immediates: [](interface{}){},
		}

		logger.Printf("Name is %s", opStr.Name)
		if !opStr.Polymorphic {
			top := int(stackDepths.Top())
			stackDepths.SetTop(uint64(top - len(opStr.Args)))
			if top < -1 {
				return nil, ErrStackUnderflow
			}
			if opStr.Returns != wasm.ValueType(wasm.BlockTypeEmpty) {
				stackDepths.SetTop(uint64(top) + 1)
			}
			if top > disas.MaxDepth {
				disas.MaxDepth = top
			}
		}

		switch op {
		case ops.Drop:
			stackDepths.SetTop(stackDepths.Top() - 1)
		case ops.Select:
			stackDepths.SetTop(stackDepths.Top() - 3)
		case ops.Return:
			stackDepths.SetTop(stackDepths.Top() - uint64(len(fn.Sig.ReturnTypes)))
			lastOpReturn = true
		case ops.End, ops.Else:
			// The max depth reached while execing the current block
			curDepth := stackDepths.Top()
			blockStartIndex := blockIndices.Pop()
			blockSig := disas.Code[blockStartIndex].Block.Signature
			instr.Block = &BlockInfo{
				Start:     false,
				Signature: blockSig,
				PairIndex: int(blockStartIndex),
			}

			// The max depth reached while execing the last block
			// If the signature of the current block is not empty,
			// this will be incremented
			prevDepthIndex := stackDepths.Len() - 2
			prevDepth := stackDepths.Get(prevDepthIndex)

			if blockSig != wasm.BlockTypeEmpty {
				stackDepths.Set(prevDepthIndex, prevDepth+1)
				disas.checkMaxDepth(int(stackDepths.Get(prevDepthIndex)))
			}

			if !lastOpReturn {
				elemsDiscard := int(curDepth) - int(prevDepth)
				if elemsDiscard < -1 {
					return nil, ErrStackUnderflow
				}
				instr.NewStack = &StackInfo{
					MaxDepth:    int64(elemsDiscard),
					PreserveTop: blockSig != wasm.BlockTypeEmpty,
				}
			} else {
				instr.NewStack = &StackInfo{}
			}

			stackDepths.Pop()
			if op == ops.Else {
				stackDepths.Push(0)
				blockIndices.Push(uint64(curIndex))
			}
		case ops.Block, ops.Loop, ops.If:
			sig, err := leb128.ReadVarint32(reader)
			if err != nil {
				return nil, err
			}
			logger.Printf("if, depth is %d", stackDepths.Top())
			stackDepths.Push(stackDepths.Top())
			instr.Block = &BlockInfo{
				Start:     true,
				Signature: wasm.BlockType(sig),
			}

			blockIndices.Push(uint64(curIndex))
			instr.Immediates = append(instr.Immediates, wasm.BlockType(sig))
		case ops.Br, ops.BrIf:
			depth, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, depth)

			curDepth := stackDepths.Top()
			elemsDiscard := int64(int(curDepth) - int(stackDepths.Get(int(depth))))
			if elemsDiscard < 0 {
				return nil, ErrStackUnderflow
			}

			// we use getBlockDepth because a getBlockIndex would
			// have the same code
			index := blockIndices.Get(int(depth))
			instr.NewStack = &StackInfo{
				MaxDepth:    elemsDiscard,
				PreserveTop: disas.Code[index].Block.Signature != wasm.BlockTypeEmpty,
			}

		case ops.BrTable:
			targetCount, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, targetCount)
			for i := uint32(0); i < targetCount; i++ {
				entry, err := leb128.ReadVarUint32(reader)
				if err != nil {
					return nil, err
				}
				instr.Immediates = append(instr.Immediates, entry)
			}

			defaultTarget, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, defaultTarget)
			stackDepths.SetTop(stackDepths.Top() - 1)
		case ops.Call, ops.CallIndirect:
			index, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, index)
			if op == ops.CallIndirect {
				reserved, err := leb128.ReadVarUint32(reader)
				if err != nil {
					return nil, err
				}
				instr.Immediates = append(instr.Immediates, reserved)
			}
			fn := module.GetFunction(int(index))
			top := int(stackDepths.Top())
			top -= len(fn.Sig.ParamTypes)
			top += len(fn.Sig.ReturnTypes)
			stackDepths.SetTop(uint64(top))
			disas.checkMaxDepth(top)
		case ops.GetLocal, ops.SetLocal, ops.TeeLocal, ops.GetGlobal, ops.SetGlobal:
			index, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, index)

			top := stackDepths.Top()
			switch op {
			case ops.GetLocal, ops.GetGlobal:
				stackDepths.SetTop(top + 1)
				disas.checkMaxDepth(int(stackDepths.Top()))
			case ops.SetLocal, ops.SetGlobal:
				stackDepths.SetTop(top - 1)
			case ops.TeeLocal:
				// stack remains unchanged for tee_local
			}
		case ops.I32Const:
			i, err := leb128.ReadVarint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, i)
		case ops.I64Const:
			i, err := leb128.ReadVarint64(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, i)
		case ops.F32Const:
			var i uint32
			// TODO(vibhavp): Switch to a reflect-free method in the future
			// for reading off the bytestream.
			err := binary.Read(reader, binary.LittleEndian, &i)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, math.Float32frombits(i))
		case ops.F64Const:
			var i uint64
			// TODO(vibhavp): Switch to a reflect-free method in the future
			// for reading off the bytestream.
			err := binary.Read(reader, binary.LittleEndian, &i)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, math.Float64frombits(i))
		case ops.I32Load, ops.I64Load, ops.F32Load, ops.F64Load, ops.I32Load8s, ops.I32Load8u, ops.I32Load16s, ops.I32Load16u, ops.I64Load8s, ops.I64Load8u, ops.I64Load16s, ops.I64Load16u, ops.I64Load32s, ops.I64Load32u, ops.I32Store, ops.I64Store, ops.F32Store, ops.F64Store, ops.I32Store8, ops.I32Store16, ops.I64Store8, ops.I64Store16, ops.I64Store32:
			// read memory_immediate
			flags, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, flags)

			offset, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, offset)
		case ops.CurrentMemory, ops.GrowMemory:
			res, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, res)

			curDepth := stackDepths.Top()
			switch op {
			case ops.CurrentMemory:
				stackDepths.SetTop(curDepth + 1)
				disas.checkMaxDepth(int(stackDepths.Top()))
			case ops.GrowMemory:
				stackDepths.SetTop(curDepth - 1)
			}
		}

		if op != ops.Return {
			lastOpReturn = false
		}

		disas.Code = append(disas.Code, instr)
		curIndex++
	}

	return disas, nil
}
<commit_msg>disasm: Rename StackInfo.MaxDepth to StackTopDiff<commit_after>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package disasm provides functions for disassembling WebAssembly bytecode.
package disasm

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"math"

	"github.com/go-interpreter/wagon/internal/stack"
	"github.com/go-interpreter/wagon/wasm"
	"github.com/go-interpreter/wagon/wasm/leb128"
	ops "github.com/go-interpreter/wagon/wasm/operators"
)

// Instr describes an instruction, consisting of an operator, with its
// appropriate immediate value(s).
type Instr struct {
	Op ops.Op

	// Immediates are arguments to an operator in the bytecode stream itself.
	// Valid value types are:
	// - (u)(int/float)(32/64)
	// - wasm.BlockType
	Immediates []interface{}
	NewStack   *StackInfo // non-nil if the instruction requires the current stack to be unwound.
	Block      *BlockInfo // non-nil if the instruction starts a new block.
}

// StackInfo stores details about a new stack ended by an instruction.
type StackInfo struct {
	StackTopDiff int64 // The difference between the stack depths at the end of the block
	PreserveTop  bool  // Whether the value on the top of the stack should be preserved while unwinding
}

// BlockInfo stores details about a block created or ended by an instruction.
type BlockInfo struct {
	Start     bool           // If true, this instruction starts a block. Else this instruction ends it.
	Signature wasm.BlockType // The block signature
	// The index to the accompanying control operator.
	// For 'if', this is an index to the 'else' operator
	// For else/loop/block, the index is to the 'end' operator
	PairIndex int
}

// Disassembly is the result of disassembling a WebAssembly function.
type Disassembly struct {
	Code     []Instr
	MaxDepth int // The maximum stack depth that can be reached while executing this function
}

func (d *Disassembly) checkMaxDepth(depth int) {
	if depth > d.MaxDepth {
		d.MaxDepth = depth
	}
}

var ErrStackUnderflow = errors.New("disasm: stack underflow")

// Disassemble disassembles the given function. It also takes the function's
// parent module as an argument for locating any other functions referenced by
// fn.
func Disassemble(fn wasm.Function, module *wasm.Module) (*Disassembly, error) {
	code := fn.Body.Code
	reader := bytes.NewReader(code)
	disas := &Disassembly{}

	// a stack of current execution stack depth values, so that the depth for each
	// stack is maintained indepepdently for calculating discard values
	stackDepths := &stack.Stack{}
	stackDepths.Push(0)
	blockIndices := &stack.Stack{} // a stack of indices to operators which start new blocks
	curIndex := 0
	var lastOpReturn bool

	for {
		op, err := reader.ReadByte()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		logger.Printf("stack top is %d", stackDepths.Top())

		opStr, err := ops.New(op)
		if err != nil {
			return nil, err
		}
		instr := Instr{
			Op:         opStr,
			Immediates: [](interface{}){},
		}

		logger.Printf("Name is %s", opStr.Name)
		if !opStr.Polymorphic {
			top := int(stackDepths.Top())
			stackDepths.SetTop(uint64(top - len(opStr.Args)))
			if top < -1 {
				return nil, ErrStackUnderflow
			}
			if opStr.Returns != wasm.ValueType(wasm.BlockTypeEmpty) {
				stackDepths.SetTop(uint64(top) + 1)
			}
			if top > disas.MaxDepth {
				disas.MaxDepth = top
			}
		}

		switch op {
		case ops.Drop:
			stackDepths.SetTop(stackDepths.Top() - 1)
		case ops.Select:
			stackDepths.SetTop(stackDepths.Top() - 3)
		case ops.Return:
			stackDepths.SetTop(stackDepths.Top() - uint64(len(fn.Sig.ReturnTypes)))
			lastOpReturn = true
		case ops.End, ops.Else:
			// The max depth reached while execing the current block
			curDepth := stackDepths.Top()
			blockStartIndex := blockIndices.Pop()
			blockSig := disas.Code[blockStartIndex].Block.Signature
			instr.Block = &BlockInfo{
				Start:     false,
				Signature: blockSig,
				PairIndex: int(blockStartIndex),
			}

			// The max depth reached while execing the last block
			// If the signature of the current block is not empty,
			// this will be incremented
			prevDepthIndex := stackDepths.Len() - 2
			prevDepth := stackDepths.Get(prevDepthIndex)

			if blockSig != wasm.BlockTypeEmpty {
				stackDepths.Set(prevDepthIndex, prevDepth+1)
				disas.checkMaxDepth(int(stackDepths.Get(prevDepthIndex)))
			}

			if !lastOpReturn {
				elemsDiscard := int(curDepth) - int(prevDepth)
				if elemsDiscard < -1 {
					return nil, ErrStackUnderflow
				}
				instr.NewStack = &StackInfo{
					StackTopDiff: int64(elemsDiscard),
					PreserveTop:  blockSig != wasm.BlockTypeEmpty,
				}
			} else {
				instr.NewStack = &StackInfo{}
			}

			stackDepths.Pop()
			if op == ops.Else {
				stackDepths.Push(0)
				blockIndices.Push(uint64(curIndex))
			}
		case ops.Block, ops.Loop, ops.If:
			sig, err := leb128.ReadVarint32(reader)
			if err != nil {
				return nil, err
			}
			logger.Printf("if, depth is %d", stackDepths.Top())
			stackDepths.Push(stackDepths.Top())
			instr.Block = &BlockInfo{
				Start:     true,
				Signature: wasm.BlockType(sig),
			}

			blockIndices.Push(uint64(curIndex))
			instr.Immediates = append(instr.Immediates, wasm.BlockType(sig))
		case ops.Br, ops.BrIf:
			depth, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, depth)

			curDepth := stackDepths.Top()
			elemsDiscard := int64(int(curDepth) - int(stackDepths.Get(int(depth))))
			if elemsDiscard < 0 {
				return nil, ErrStackUnderflow
			}

			// we use getBlockDepth because a getBlockIndex would
			// have the same code
			index := blockIndices.Get(int(depth))
			instr.NewStack = &StackInfo{
				StackTopDiff: elemsDiscard,
				PreserveTop:  disas.Code[index].Block.Signature != wasm.BlockTypeEmpty,
			}

		case ops.BrTable:
			targetCount, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, targetCount)
			for i := uint32(0); i < targetCount; i++ {
				entry, err := leb128.ReadVarUint32(reader)
				if err != nil {
					return nil, err
				}
				instr.Immediates = append(instr.Immediates, entry)
			}

			defaultTarget, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, defaultTarget)
			stackDepths.SetTop(stackDepths.Top() - 1)
		case ops.Call, ops.CallIndirect:
			index, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, index)
			if op == ops.CallIndirect {
				reserved, err := leb128.ReadVarUint32(reader)
				if err != nil {
					return nil, err
				}
				instr.Immediates = append(instr.Immediates, reserved)
			}
			fn := module.GetFunction(int(index))
			top := int(stackDepths.Top())
			top -= len(fn.Sig.ParamTypes)
			top += len(fn.Sig.ReturnTypes)
			stackDepths.SetTop(uint64(top))
			disas.checkMaxDepth(top)
		case ops.GetLocal, ops.SetLocal, ops.TeeLocal, ops.GetGlobal, ops.SetGlobal:
			index, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, index)

			top := stackDepths.Top()
			switch op {
			case ops.GetLocal, ops.GetGlobal:
				stackDepths.SetTop(top + 1)
				disas.checkMaxDepth(int(stackDepths.Top()))
			case ops.SetLocal, ops.SetGlobal:
				stackDepths.SetTop(top - 1)
			case ops.TeeLocal:
				// stack remains unchanged for tee_local
			}
		case ops.I32Const:
			i, err := leb128.ReadVarint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, i)
		case ops.I64Const:
			i, err := leb128.ReadVarint64(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, i)
		case ops.F32Const:
			var i uint32
			// TODO(vibhavp): Switch to a reflect-free method in the future
			// for reading off the bytestream.
			err := binary.Read(reader, binary.LittleEndian, &i)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, math.Float32frombits(i))
		case ops.F64Const:
			var i uint64
			// TODO(vibhavp): Switch to a reflect-free method in the future
			// for reading off the bytestream.
			err := binary.Read(reader, binary.LittleEndian, &i)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, math.Float64frombits(i))
		case ops.I32Load, ops.I64Load, ops.F32Load, ops.F64Load, ops.I32Load8s, ops.I32Load8u, ops.I32Load16s, ops.I32Load16u, ops.I64Load8s, ops.I64Load8u, ops.I64Load16s, ops.I64Load16u, ops.I64Load32s, ops.I64Load32u, ops.I32Store, ops.I64Store, ops.F32Store, ops.F64Store, ops.I32Store8, ops.I32Store16, ops.I64Store8, ops.I64Store16, ops.I64Store32:
			// read memory_immediate
			flags, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, flags)

			offset, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, offset)
		case ops.CurrentMemory, ops.GrowMemory:
			res, err := leb128.ReadVarUint32(reader)
			if err != nil {
				return nil, err
			}
			instr.Immediates = append(instr.Immediates, res)

			curDepth := stackDepths.Top()
			switch op {
			case ops.CurrentMemory:
				stackDepths.SetTop(curDepth + 1)
				disas.checkMaxDepth(int(stackDepths.Top()))
			case ops.GrowMemory:
				stackDepths.SetTop(curDepth - 1)
			}
		}

		if op != ops.Return {
			lastOpReturn = false
		}

		disas.Code = append(disas.Code, instr)
		curIndex++
	}

	return disas, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements accept for platforms that provide a fast path for
// setting SetNonblock and CloseOnExec.

//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris

package poll

import "syscall"

// Wrapper around the accept system call that marks the returned file
// descriptor as nonblocking and close-on-exec.
func accept(s int) (int, syscall.Sockaddr, string, error) {
	ns, sa, err := Accept4Func(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC)
	// On Linux the accept4 system call was introduced in 2.6.28
	// kernel and on FreeBSD it was introduced in 10 kernel. If we
	// get an ENOSYS error on both Linux and FreeBSD, or EINVAL
	// error on Linux, fall back to using accept.
	switch err {
	case nil:
		return ns, sa, "", nil
	default: // errors other than the ones listed
		return -1, sa, "accept4", err
	case syscall.ENOSYS: // syscall missing
	case syscall.EINVAL: // some Linux use this instead of ENOSYS
	case syscall.EACCES: // some Linux use this instead of ENOSYS
	case syscall.EFAULT: // some Linux use this instead of ENOSYS
	}

	// See ../syscall/exec_unix.go for description of ForkLock.
	// It is probably okay to hold the lock across syscall.Accept
	// because we have put fd.sysfd into non-blocking mode.
	// However, a call to the File method will put it back into
	// blocking mode. We can't take that risk, so no use of ForkLock here.
	ns, sa, err = AcceptFunc(s)
	if err == nil {
		syscall.CloseOnExec(ns)
	}
	if err != nil {
		return -1, nil, "accept", err
	}
	if err = syscall.SetNonblock(ns, true); err != nil {
		CloseFunc(ns)
		return -1, nil, "setnonblock", err
	}
	return ns, sa, "", nil
}
<commit_msg>internal/poll: remove fallback path in accept<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements accept for platforms that provide a fast path for
// setting SetNonblock and CloseOnExec.

//go:build dragonfly || freebsd || linux || netbsd || openbsd || solaris

package poll

import "syscall"

// Wrapper around the accept system call that marks the returned file
// descriptor as nonblocking and close-on-exec.
func accept(s int) (int, syscall.Sockaddr, string, error) {
	ns, sa, err := Accept4Func(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC)
	if err != nil {
		return -1, sa, "accept4", err
	}
	return ns, sa, "", nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/

package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/rds"
    "fmt"
    "os"
)

func main() {
    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create RDS service client
    svc := rds.New(sess)

    result, err := svc.DescribeDBSnapshots(nil)
    if err != nil {
        exitErrorf("Unable to list snapshots, %v", err)
    }

    for _, s := range result.DBSnapshots {
        fmt.Printf("* %s with status %s\n",
            aws.StringValue(s.DBSnapshotIdentifier), aws.StringValue(s.Status))
    }
}

func exitErrorf(msg string, args ...interface{}) {
    fmt.Fprintf(os.Stderr, msg+"\n", args...)
    os.Exit(1)
}
<commit_msg>Update rds_list_instance_snapshots.go<commit_after>// snippet-comment:[These are tags for the AWS doc team's sample catalog. Do not remove.]
// snippet-sourceauthor:[AWS]
// snippet-sourcedescription:[rds_list_instance_snapshots lists your RDS instance snapshots.]
// snippet-keyword:[Amazon Relational Database Service]
// snippet-keyword:[Amazon RDS]
// snippet-keyword:[DescribeDBSnapshots function]
// snippet-keyword:[Go]
// snippet-service:[s3]
// snippet-keyword:[Code Sample]
// snippet-sourcetype:[full-example]
// snippet-sourcedate:[2019-01-30]
/*
   Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
// snippet-start:[rds.go.describe_db_snapshots]
package main

import (
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/rds"
)

func main() {
	// Initialize a session in us-west-2 that the SDK will use to load
	// credentials from the shared credentials file ~/.aws/credentials.
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-west-2")},
	)

	// Create RDS service client
	svc := rds.New(sess)

	result, err := svc.DescribeDBSnapshots(nil)
	if err != nil {
		exitErrorf("Unable to list snapshots, %v", err)
	}

	for _, s := range result.DBSnapshots {
		fmt.Printf("* %s with status %s\n",
			aws.StringValue(s.DBSnapshotIdentifier), aws.StringValue(s.Status))
	}
}

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}
// snippet-end:[rds.go.describe_db_snapshots]
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"net/http"

	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/common/metrics"
	"strings"

	kmetrics "github.com/koding/metrics"

	tigertonic "github.com/rcrowley/go-tigertonic"

	gometrics "github.com/rcrowley/go-metrics"
)

const (
	PostRequest   = "POST"
	GetRequest    = "GET"
	DeleteRequest = "DELETE"
)

var (
	// TODO allowed origins must be configurable
	cors     = tigertonic.NewCORSBuilder().AddAllowedOrigins("*")
	conf     *config.Config
	trackers *metrics.Trackers
)

type Request struct {
	Handler        interface{}
	Endpoint       string
	Type           string
	Name           string
	CollectMetrics bool
	Metrics        *kmetrics.Metrics
	// used for external requests
	Params  map[string]string
	Cookie  string
	Cookies []*http.Cookie
}

// todo add prooper logging
func getAccount(r *http.Request) *models.Account {
	cookie, err := r.Cookie("clientId")
	if err != nil {
		return nil
	}

	// if cookie doenst exists return nil
	if cookie.Value == "" {
		return nil
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return nil
	}

	// if session doesnt have username, return nil
	if session.Username == "" {
		return nil
	}

	acc, err := models.Cache.Account.ByNick(session.Username)
	if err != nil {
		return nil
	}

	return acc
}

func Wrapper(r Request) http.Handler {
	handler := r.Handler

	var hHandler http.Handler

	// count the statuses of the requests
	hHandler = buildHandlerWithStatusCount(handler, r)

	hHandler = buildHandlerWithTimeTracking(hHandler, r)

	// create the final handler
	return cors.Build(hHandler)
}

// count the statuses of the requests
func buildHandlerWithStatusCount(handler interface{}, r Request) http.Handler {
	return CountedByStatus(
		tigertonic.Marshaled(handler), r.Name, r.CollectMetrics,
	)
}

// add request time tracking
func buildHandlerWithTimeTracking(handler http.Handler, r Request) http.Handler {
	var registry gometrics.Registry

	if r.Metrics != nil {
		registry = r.Metrics.Registry
	}

	return tigertonic.Timed(
		handler,
		r.Name,
		registry,
	)
}

func BuildHandlerWithContext(handler http.Handler) http.Handler {
	// add context
	return tigertonic.If(
		func(r *http.Request) (http.Header, error) {
			// this is an example
			// set group name to context
			//
			context := &models.Context{
				GroupName: "koding",
				Client: &models.Client{
					Account: getAccount(r),
				},
			}

			*(tigertonic.Context(r).(*models.Context)) = *context
			return nil, nil
		},
		handler,
	)
}

//----------------------------------------------------------
// CounterByStatus
//----------------------------------------------------------

type CounterByStatus struct {
	name           string
	handler        http.Handler
	collectMetrics bool
}

func (c *CounterByStatus) ServeHTTP(w0 http.ResponseWriter, r *http.Request) {
	w := tigertonic.NewTeeHeaderResponseWriter(w0)
	c.handler.ServeHTTP(w, r)

	if w.StatusCode <= 300 {
		c.track()
	}
}

func (c *CounterByStatus) track() {
	// don't log if analytics are disabled for that endpoint
	if !c.collectMetrics {
		return
	}

	// set `conf` and `trackers` if either is not set
	if conf == nil || trackers == nil {
		conf = config.MustGet()

		// don't log if analytics are disabled globally
		if !conf.Mixpanel.Enabled {
			return
		}

		trackers = metrics.InitTrackers(
			metrics.NewMixpanelTracker(conf.Mixpanel.Token),
		)
	}

	trackers.Track(c.name)
}

func CountedByStatus(handler http.Handler, name string, collectMetrics bool) *CounterByStatus {
	return &CounterByStatus{
		name:           name,
		handler:        handler,
		collectMetrics: collectMetrics,
	}
}

//
func MakeRequest(request *Request) (*http.Response, error) {
	if request.Cookie != "" {
		request.Cookies = parseCookies(request.Cookie)
	}

	request.Endpoint = prepareQueryString(request.Endpoint, request.Params)

	client := new(http.Client)
	req, err := http.NewRequest(request.Type, request.Endpoint, nil)
	if err != nil {
		return nil, err
	}

	// Add cookies
	for _, cookie := range request.Cookies {
		req.AddCookie(cookie)
	}

	return client.Do(req)
}

func prepareQueryString(endpoint string, params map[string]string) string {
	if len(params) == 0 {
		return endpoint
	}

	fullPath := fmt.Sprintf("%s?", endpoint)

	for key, value := range params {
		fullPath = fmt.Sprintf("%s%s=%s&", fullPath, key, value)
	}

	return fullPath[0 : len(fullPath)-1]
}

func parseCookies(cookie string) []*http.Cookie {
	pairs := strings.Split(cookie, "; ")
	cookies := make([]*http.Cookie, 0)

	if len(pairs) == 0 {
		return cookies
	}

	for _, val := range pairs {
		cp := strings.Split(val, "=")
		if len(cp) != 2 {
			continue
		}

		c := new(http.Cookie)
		c.Name = cp[0]
		c.Value = cp[1]

		cookies = append(cookies, c)
	}

	return cookies
}
<commit_msg>social: add header and cookie support for handler<commit_after>package handler

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"socialapi/config"
	"socialapi/models"
	"socialapi/workers/common/metrics"
	"strings"

	kmetrics "github.com/koding/metrics"

	tigertonic "github.com/rcrowley/go-tigertonic"

	gometrics "github.com/rcrowley/go-metrics"
)

const (
	PostRequest   = "POST"
	GetRequest    = "GET"
	DeleteRequest = "DELETE"
)

var (
	// TODO allowed origins must be configurable
	cors     = tigertonic.NewCORSBuilder().AddAllowedOrigins("*")
	conf     *config.Config
	trackers *metrics.Trackers
)

type Request struct {
	Handler        interface{}
	Endpoint       string
	Type           string
	Name           string
	CollectMetrics bool
	Metrics        *kmetrics.Metrics
	// used for external requests
	Params  map[string]string
	Cookie  string
	Cookies []*http.Cookie
	Body    interface{}
	Headers map[string]string
}

// todo add prooper logging
func getAccount(r *http.Request) *models.Account {
	cookie, err := r.Cookie("clientId")
	if err != nil {
		return nil
	}

	// if cookie doenst exists return nil
	if cookie.Value == "" {
		return nil
	}

	session, err := models.Cache.Session.ById(cookie.Value)
	if err != nil {
		return nil
	}

	// if session doesnt have username, return nil
	if session.Username == "" {
		return nil
	}

	acc, err := models.Cache.Account.ByNick(session.Username)
	if err != nil {
		return nil
	}

	return acc
}

func Wrapper(r Request) http.Handler {
	handler := r.Handler

	var hHandler http.Handler

	// count the statuses of the requests
	hHandler = buildHandlerWithStatusCount(handler, r)

	hHandler = buildHandlerWithTimeTracking(hHandler, r)

	// create the final handler
	return cors.Build(hHandler)
}

// count the statuses of the requests
func buildHandlerWithStatusCount(handler interface{}, r Request) http.Handler {
	return CountedByStatus(
		tigertonic.Marshaled(handler), r.Name, r.CollectMetrics,
	)
}

// add request time tracking
func buildHandlerWithTimeTracking(handler http.Handler, r Request) http.Handler {
	var registry gometrics.Registry

	if r.Metrics != nil {
		registry = r.Metrics.Registry
	}

	return tigertonic.Timed(
		handler,
		r.Name,
		registry,
	)
}

func BuildHandlerWithContext(handler http.Handler) http.Handler {
	// add context
	return tigertonic.If(
		func(r *http.Request) (http.Header, error) {
			// this is an example
			// set group name to context
			//
			context := &models.Context{
				GroupName: "koding",
				Client: &models.Client{
					Account: getAccount(r),
				},
			}

			*(tigertonic.Context(r).(*models.Context)) = *context
			return nil, nil
		},
		handler,
	)
}

//----------------------------------------------------------
// CounterByStatus
//----------------------------------------------------------

type CounterByStatus struct {
	name           string
	handler        http.Handler
	collectMetrics bool
}

func (c *CounterByStatus) ServeHTTP(w0 http.ResponseWriter, r *http.Request) {
	w := tigertonic.NewTeeHeaderResponseWriter(w0)
	c.handler.ServeHTTP(w, r)

	if w.StatusCode <= 300 {
		c.track()
	}
}

func (c *CounterByStatus) track() {
	// don't log if analytics are disabled for that endpoint
	if !c.collectMetrics {
		return
	}

	// set `conf` and `trackers` if either is not set
	if conf == nil || trackers == nil {
		conf = config.MustGet()

		// don't log if analytics are disabled globally
		if !conf.Mixpanel.Enabled {
			return
		}

		trackers = metrics.InitTrackers(
			metrics.NewMixpanelTracker(conf.Mixpanel.Token),
		)
	}

	trackers.Track(c.name)
}

func CountedByStatus(handler http.Handler, name string, collectMetrics bool) *CounterByStatus {
	return &CounterByStatus{
		name:           name,
		handler:        handler,
		collectMetrics: collectMetrics,
	}
}

//
func MakeRequest(request *Request) (*http.Response, error) {
	if request.Cookie != "" {
		request.Cookies = parseCookiesToArray(request.Cookie)
	}

	request.Endpoint = prepareQueryString(request.Endpoint, request.Params)

	client := new(http.Client)
	hostname := config.MustGet().CustomDomain.Public
	endpoint := fmt.Sprintf("%s/%s", hostname, request.Endpoint)

	var byteData io.Reader
	if request.Body != nil {
		body, err := json.Marshal(request.Body)
		if err != nil {
			return nil, err
		}
		byteData = bytes.NewReader(body)
	}

	req, err := http.NewRequest(request.Type, endpoint, byteData)
	req = prepareHeaders(req, request.Headers)

	if err != nil {
		return nil, err
	}

	// Add cookies
	for _, cookie := range request.Cookies {
		req.AddCookie(cookie)
	}

	return client.Do(req)
}

func prepareQueryString(endpoint string, params map[string]string) string {
	if len(params) == 0 {
		return endpoint
	}

	fullPath := fmt.Sprintf("%s?", endpoint)

	for key, value := range params {
		fullPath = fmt.Sprintf("%s%s=%s&", fullPath, key, value)
	}

	return fullPath[0 : len(fullPath)-1]
}

func prepareHeaders(req *http.Request, headers map[string]string) *http.Request {
	newReq := new(http.Request)
	*newReq = *req
	for k, v := range headers {
		newReq.Header.Set(k, v)
	}

	return newReq
}

func parseCookiesToArray(cookie string) []*http.Cookie {
	pairs := strings.Split(cookie, "; ")
	cookies := make([]*http.Cookie, 0)

	if len(pairs) == 0 {
		return cookies
	}

	for _, val := range pairs {
		cp := strings.Split(val, "=")
		if len(cp) != 2 {
			continue
		}

		c := new(http.Cookie)
		c.Name = cp[0]
		c.Value = cp[1]

		cookies = append(cookies, c)
	}

	return cookies
}

func ParseCookiesToMap(cookie string) map[string]*http.Cookie {
	pairs := strings.Split(cookie, "; ")
	cookies := make(map[string]*http.Cookie, 0)

	if len(pairs) == 0 {
		return cookies
	}

	for _, val := range pairs {
		cp := strings.Split(val, "=")
		if len(cp) != 2 {
			continue
		}

		c := new(http.Cookie)
		c.Name = cp[0]
		c.Value = cp[1]

		cookies[cp[0]] = c
	}

	return cookies
}
<|endoftext|>


<commit_before>/* Document collection. */
package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"loveoneanother.at/tiedot/file"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"
)

type IndexConf struct {
	FileName            string
	PerBucket, HashBits uint64
	IndexedPath         []string
}

type Config struct {
	Indexes []IndexConf
}

type Col struct {
	Data                                    *file.ColFile
	Config                                  *Config
	Dir, ConfigFileName, ConfBackupFileName string
	StrHT                                   map[string]*file.HashTable
	StrIC                                   map[string]*IndexConf
}

// Return string hash code.
func StrHash(thing interface{}) uint64 {
	// very similar to Java String.hashCode()
	// you must review (even rewrite) most collection test cases, if you change the hash algorithm
	str := fmt.Sprint(thing)
	length := len(str)
	hash := 0
	for i, c := range str {
		hash += int(c)*31 ^ (length - i)
	}
	return uint64(hash)
}

// Open a collection.
func OpenCol(dir string) (col *Col, err error) {
	if err = os.MkdirAll(dir, 0700); err != nil {
		return
	}
	col = &Col{ConfigFileName: path.Join(dir, "config"), ConfBackupFileName: path.Join(dir, "config.bak"), Dir: dir}
	// open data file
	if col.Data, err = file.OpenCol(path.Join(dir, "data")); err != nil {
		return
	}
	// make sure the config file exists
	tryOpen, err := os.OpenFile(col.ConfigFileName, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		return
	} else if err = tryOpen.Close(); err != nil {
		return
	}
	col.LoadConf()
	return
}

// Copy existing config file content to backup config file.
func (col *Col) BackupAndSaveConf() error {
	oldConfig, err := ioutil.ReadFile(col.ConfigFileName)
	if err != nil {
		return err
	}
	if err = ioutil.WriteFile(col.ConfBackupFileName, []byte(oldConfig), 0600); err != nil {
		return err
	}
	if col.Config != nil {
		newConfig, err := json.Marshal(col.Config)
		if err != nil {
			return err
		}
		if err = ioutil.WriteFile(col.ConfigFileName, newConfig, 0600); err != nil {
			return err
		}
	}
	return nil
}

// (Re)load configuration to collection.
func (col *Col) LoadConf() error {
	// read index config
	config, err := ioutil.ReadFile(col.ConfigFileName)
	if err != nil {
		return err
	}
	if string(config) == "" {
		col.Config = &Config{}
	} else if err = json.Unmarshal(config, &col.Config); err != nil {
		return err
	}
	// open each index file
	col.StrHT = make(map[string]*file.HashTable)
	col.StrIC = make(map[string]*IndexConf)
	for i, index := range col.Config.Indexes {
		ht, err := file.OpenHash(path.Join(col.Dir, index.FileName), index.HashBits, index.PerBucket)
		if err != nil {
			return err
		}
		col.StrHT[strings.Join(index.IndexedPath, ",")] = ht
		col.StrIC[strings.Join(index.IndexedPath, ",")] = &col.Config.Indexes[i]
	}
	return nil
}

// Get inside the data structure, along the given path.
func GetIn(doc interface{}, path []string) (ret []interface{}) {
	thing := doc
	for _, seg := range path {
		switch t := thing.(type) {
		case bool:
			return nil
		case float64:
			return nil
		case string:
			return nil
		case nil:
			return nil
		case interface{}:
			thing = t.(map[string]interface{})[seg]
		default:
			return nil
		}
	}
	switch thing.(type) {
	case []interface{}:
		return thing.([]interface{})
	default:
		return append(ret, thing)
	}
}

// Retrieve document data given its ID.
func (col *Col) Read(id uint64) (doc interface{}, size int) {
	data := col.Data.Read(id)
	size = len(data)
	if data == nil {
		return
	}
	if err := json.Unmarshal(col.Data.Read(id), &doc); err != nil {
		log.Printf("Cannot parse document %d in %s to JSON\n", id, col.Dir)
	}
	return
}

// Index the document on all indexes
func (col *Col) IndexDoc(id uint64, doc interface{}) {
	wg := new(sync.WaitGroup)
	for k, v := range col.StrIC {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for _, thing := range GetIn(doc, v.IndexedPath) {
				if thing != nil {
					col.StrHT[k].Put(StrHash(thing), id)
				}
			}
		}()
	}
	wg.Wait()
}

// Remove the document from all indexes
func (col *Col) UnindexDoc(id uint64, doc interface{}) {
	wg := new(sync.WaitGroup)
	for k, v := range col.StrIC {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for _, thing := range GetIn(doc, v.IndexedPath) {
				col.StrHT[k].Remove(StrHash(thing), 1, func(k, v uint64) bool {
					return v == id
				})
			}
		}()
	}
	wg.Wait()
}

// Insert a new document.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return
	}
	if id, err = col.Data.Insert(data); err != nil {
		return
	}
	col.IndexDoc(id, doc)
	return
}

// Update a document, return its new ID.
func (col *Col) Update(id uint64, doc interface{}) (newID uint64, err error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return
	}
	oldDoc, oldSize := col.Read(id)
	if oldDoc == nil {
		return id, nil
	}
	wg := new(sync.WaitGroup)
	if oldSize >= len(data) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, err = col.Data.Update(id, data)
		}()
	} else {
		newID, err = col.Data.Update(id, data)
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		col.UnindexDoc(id, oldDoc)
		col.IndexDoc(newID, doc)
	}()
	wg.Wait()
	return
}

// Delete a document.
func (col *Col) Delete(id uint64) {
	oldDoc, _ := col.Read(id)
	if oldDoc == nil {
		return
	}
	wg := new(sync.WaitGroup)
	wg.Add(2)
	go func() {
		defer wg.Done()
		col.Data.Delete(id)
	}()
	go func() {
		defer wg.Done()
		col.UnindexDoc(id, oldDoc)
	}()
	wg.Wait()
}

// Add an index.
func (col *Col) Index(path []string) error {
	joinedPath := strings.Join(path, ",")
	if _, found := col.StrHT[joinedPath]; found {
		return errors.New(fmt.Sprintf("Path %v is already indexed in collection %s", path, col.Dir))
	}
	newFileName := strings.Join(path, ",")
	if len(newFileName) > 100 {
		newFileName = newFileName[0:100]
	}
	// close all indexes
	for _, v := range col.StrHT {
		v.File.Close()
	}
	// save and reload config
	col.Config.Indexes = append(col.Config.Indexes, IndexConf{FileName: newFileName + strconv.Itoa(int(time.Now().UnixNano())), PerBucket: 100, HashBits: 14, IndexedPath: path})
	if err := col.BackupAndSaveConf(); err != nil {
		return err
	}
	if err := col.LoadConf(); err != nil {
		return err
	}
	// put all documents in the new index
	newIndex, ok := col.StrHT[strings.Join(path, ",")]
	if !ok {
		return errors.New(fmt.Sprintf("The new index %v in %s is gone??", path, col.Dir))
	}
	col.ForAll(func(id uint64, doc interface{}) bool {
		for _, thing := range GetIn(doc, path) {
			if thing != nil {
				newIndex.Put(StrHash(thing), id)
			}
		}
		return true
	})
	return nil
}

// Remove an index.
func (col *Col) Unindex(path []string) error {
	joinedPath := strings.Join(path, ",")
	if _, found := col.StrHT[joinedPath]; !found {
		return errors.New(fmt.Sprintf("Path %v was never indexed in collection %s", path, col.Dir))
	}
	// close all indexes
	for _, v := range col.StrHT {
		v.File.Close()
	}
	found := 0
	for i, index := range col.Config.Indexes {
		match := true
		for j, path := range path {
			if index.IndexedPath[j] != path {
				match = false
				break
			}
		}
		if match {
			found = i
			break
		}
	}
	// delete hash table file
	indexConf := col.Config.Indexes[found]
	indexHT := col.StrHT[strings.Join(indexConf.IndexedPath, ",")]
	indexHT.File.Close()
	if err := os.Remove(indexHT.File.Name); err != nil {
		return err
	}
	// remove it from config
	col.Config.Indexes = append(col.Config.Indexes[0:found], col.Config.Indexes[found+1:len(col.Config.Indexes)]...)
	if err := col.BackupAndSaveConf(); err != nil {
		return err
	}
	if err := col.LoadConf(); err != nil {
		return err
	}
	return nil
}

// Do fun for all documents.
func (col *Col) ForAll(fun func(id uint64, doc interface{}) bool) {
	col.Data.ForAll(func(id uint64, data []byte) bool {
		var parsed interface{}
		if err := json.Unmarshal(data, &parsed); err != nil {
			log.Printf("Cannot parse document '%v' in %s to JSON\n", data, col.Dir)
			return true
		} else {
			return fun(id, parsed)
		}
	})
}

// Close a collection.
func (col *Col) Close() {
	if err := col.Data.File.Close(); err != nil {
		log.Printf("Failed to close %s, reason: %v\n", col.Data.File.Name, err)
	}
	for _, ht := range col.StrHT {
		if err := ht.File.Close(); err != nil {
			log.Printf("Failed to close %s, reason: %v\n", ht.File.Name, err)
		}
	}
}
<commit_msg>fix for test cases<commit_after>/* Document collection. */
package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"loveoneanother.at/tiedot/file"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"
)

type IndexConf struct {
	FileName            string
	PerBucket, HashBits uint64
	IndexedPath         []string
}

type Config struct {
	Indexes []IndexConf
}

type Col struct {
	Data                                    *file.ColFile
	Config                                  *Config
	Dir, ConfigFileName, ConfBackupFileName string
	StrHT                                   map[string]*file.HashTable
	StrIC                                   map[string]*IndexConf
}

// Return string hash code.
func StrHash(thing interface{}) uint64 {
	// very similar to Java String.hashCode()
	// you must review (even rewrite) most collection test cases, if you change the hash algorithm
	str := fmt.Sprint(thing)
	length := len(str)
	hash := 0
	for i, c := range str {
		hash += int(c)*31 ^ (length - i)
	}
	return uint64(hash)
}

// Open a collection.
func OpenCol(dir string) (col *Col, err error) {
	if err = os.MkdirAll(dir, 0700); err != nil {
		return
	}
	col = &Col{ConfigFileName: path.Join(dir, "config"), ConfBackupFileName: path.Join(dir, "config.bak"), Dir: dir}
	// open data file
	if col.Data, err = file.OpenCol(path.Join(dir, "data")); err != nil {
		return
	}
	// make sure the config file exists
	tryOpen, err := os.OpenFile(col.ConfigFileName, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		return
	} else if err = tryOpen.Close(); err != nil {
		return
	}
	col.LoadConf()
	return
}

// Copy existing config file content to backup config file.
func (col *Col) BackupAndSaveConf() error {
	oldConfig, err := ioutil.ReadFile(col.ConfigFileName)
	if err != nil {
		return err
	}
	if err = ioutil.WriteFile(col.ConfBackupFileName, []byte(oldConfig), 0600); err != nil {
		return err
	}
	if col.Config != nil {
		newConfig, err := json.Marshal(col.Config)
		if err != nil {
			return err
		}
		if err = ioutil.WriteFile(col.ConfigFileName, newConfig, 0600); err != nil {
			return err
		}
	}
	return nil
}

// (Re)load configuration to collection.
func (col *Col) LoadConf() error {
	// read index config
	config, err := ioutil.ReadFile(col.ConfigFileName)
	if err != nil {
		return err
	}
	if string(config) == "" {
		col.Config = &Config{}
	} else if err = json.Unmarshal(config, &col.Config); err != nil {
		return err
	}
	// open each index file
	col.StrHT = make(map[string]*file.HashTable)
	col.StrIC = make(map[string]*IndexConf)
	for i, index := range col.Config.Indexes {
		ht, err := file.OpenHash(path.Join(col.Dir, index.FileName), index.HashBits, index.PerBucket)
		if err != nil {
			return err
		}
		col.StrHT[strings.Join(index.IndexedPath, ",")] = ht
		col.StrIC[strings.Join(index.IndexedPath, ",")] = &col.Config.Indexes[i]
	}
	return nil
}

// Get inside the data structure, along the given path.
func GetIn(doc interface{}, path []string) (ret []interface{}) {
	thing := doc
	for _, seg := range path {
		switch t := thing.(type) {
		case bool:
			return nil
		case float64:
			return nil
		case string:
			return nil
		case nil:
			return nil
		case interface{}:
			thing = t.(map[string]interface{})[seg]
		default:
			return nil
		}
	}
	switch thing.(type) {
	case []interface{}:
		return thing.([]interface{})
	default:
		return append(ret, thing)
	}
}

// Retrieve document data given its ID.
func (col *Col) Read(id uint64) (doc interface{}, size int) {
	data := col.Data.Read(id)
	size = len(data)
	if data == nil {
		return
	}
	if err := json.Unmarshal(col.Data.Read(id), &doc); err != nil {
		log.Printf("Cannot parse document %d in %s to JSON\n", id, col.Dir)
	}
	return
}

// Index the document on all indexes
func (col *Col) IndexDoc(id uint64, doc interface{}) {
	wg := new(sync.WaitGroup)
	wg.Add(len(col.StrIC))
	for k, v := range col.StrIC {
		go func(k string, v *IndexConf) {
			defer wg.Done()
			for _, thing := range GetIn(doc, v.IndexedPath) {
				if thing != nil {
					col.StrHT[k].Put(StrHash(thing), id)
				}
			}
		}(k, v)
	}
	wg.Wait()
}

// Remove the document from all indexes
func (col *Col) UnindexDoc(id uint64, doc interface{}) {
	wg := new(sync.WaitGroup)
	wg.Add(len(col.StrIC))
	for k, v := range col.StrIC {
		go func(k string, v *IndexConf) {
			defer wg.Done()
			for _, thing := range GetIn(doc, v.IndexedPath) {
				col.StrHT[k].Remove(StrHash(thing), 1, func(k, v uint64) bool {
					return v == id
				})
			}
		}(k, v)
	}
	wg.Wait()
}

// Insert a new document.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return
	}
	if id, err = col.Data.Insert(data); err != nil {
		return
	}
	col.IndexDoc(id, doc)
	return
}

// Update a document, return its new ID.
func (col *Col) Update(id uint64, doc interface{}) (newID uint64, err error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return
	}
	oldDoc, oldSize := col.Read(id)
	if oldDoc == nil {
		return id, nil
	}
	wg := new(sync.WaitGroup)
	if oldSize >= len(data) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, err = col.Data.Update(id, data)
		}()
	} else {
		newID, err = col.Data.Update(id, data)
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		col.UnindexDoc(id, oldDoc)
		col.IndexDoc(newID, doc)
	}()
	wg.Wait()
	return
}

// Delete a document.
func (col *Col) Delete(id uint64) {
	oldDoc, _ := col.Read(id)
	if oldDoc == nil {
		return
	}
	wg := new(sync.WaitGroup)
	wg.Add(2)
	go func() {
		defer wg.Done()
		col.Data.Delete(id)
	}()
	go func() {
		defer wg.Done()
		col.UnindexDoc(id, oldDoc)
	}()
	wg.Wait()
}

// Add an index.
func (col *Col) Index(path []string) error {
	joinedPath := strings.Join(path, ",")
	if _, found := col.StrHT[joinedPath]; found {
		return errors.New(fmt.Sprintf("Path %v is already indexed in collection %s", path, col.Dir))
	}
	newFileName := strings.Join(path, ",")
	if len(newFileName) > 100 {
		newFileName = newFileName[0:100]
	}
	// close all indexes
	for _, v := range col.StrHT {
		v.File.Close()
	}
	// save and reload config
	col.Config.Indexes = append(col.Config.Indexes, IndexConf{FileName: newFileName + strconv.Itoa(int(time.Now().UnixNano())), PerBucket: 100, HashBits: 14, IndexedPath: path})
	if err := col.BackupAndSaveConf(); err != nil {
		return err
	}
	if err := col.LoadConf(); err != nil {
		return err
	}
	// put all documents in the new index
	newIndex, ok := col.StrHT[strings.Join(path, ",")]
	if !ok {
		return errors.New(fmt.Sprintf("The new index %v in %s is gone??", path, col.Dir))
	}
	col.ForAll(func(id uint64, doc interface{}) bool {
		for _, thing := range GetIn(doc, path) {
			if thing != nil {
				newIndex.Put(StrHash(thing), id)
			}
		}
		return true
	})
	return nil
}

// Remove an index.
func (col *Col) Unindex(path []string) error {
	joinedPath := strings.Join(path, ",")
	if _, found := col.StrHT[joinedPath]; !found {
		return errors.New(fmt.Sprintf("Path %v was never indexed in collection %s", path, col.Dir))
	}
	// close all indexes
	for _, v := range col.StrHT {
		v.File.Close()
	}
	found := 0
	for i, index := range col.Config.Indexes {
		match := true
		for j, path := range path {
			if index.IndexedPath[j] != path {
				match = false
				break
			}
		}
		if match {
			found = i
			break
		}
	}
	// delete hash table file
	indexConf := col.Config.Indexes[found]
	indexHT := col.StrHT[strings.Join(indexConf.IndexedPath, ",")]
	indexHT.File.Close()
	if err := os.Remove(indexHT.File.Name); err != nil {
		return err
	}
	// remove it from config
	col.Config.Indexes = append(col.Config.Indexes[0:found], col.Config.Indexes[found+1:len(col.Config.Indexes)]...)
	if err := col.BackupAndSaveConf(); err != nil {
		return err
	}
	if err := col.LoadConf(); err != nil {
		return err
	}
	return nil
}

// Do fun for all documents.
func (col *Col) ForAll(fun func(id uint64, doc interface{}) bool) {
	col.Data.ForAll(func(id uint64, data []byte) bool {
		var parsed interface{}
		if err := json.Unmarshal(data, &parsed); err != nil {
			log.Printf("Cannot parse document '%v' in %s to JSON\n", data, col.Dir)
			return true
		} else {
			return fun(id, parsed)
		}
	})
}

// Close a collection.
func (col *Col) Close() {
	if err := col.Data.File.Close(); err != nil {
		log.Printf("Failed to close %s, reason: %v\n", col.Data.File.Name, err)
	}
	for _, ht := range col.StrHT {
		if err := ht.File.Close(); err != nil {
			log.Printf("Failed to close %s, reason: %v\n", ht.File.Name, err)
		}
	}
}
<|endoftext|>