<commit_before>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 Weaviate. All rights reserved.
 * LICENSE: https://github.com/weaviate/weaviate/blob/master/LICENSE
 * AUTHOR: Bob van Luijt (bob@weaviate.com)
 * See www.weaviate.com for details
 * Contact: @weaviate_iot / yourfriends@weaviate.com
 */
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	// "github.com/weaviate/weaviate/connectors"
	// "github.com/weaviate/weaviate/connectors/datastore"
	"github.com/go-openapi/strfmt"
	"github.com/weaviate/weaviate/models"
)

/*
 * Request function
 */
func doRequest(endpoint string, method string, accept string, body io.Reader, apiKey string) *http.Response {
	tr := &http.Transport{
		MaxIdleConns:       10,
		IdleConnTimeout:    30 * time.Second,
		DisableCompression: true,
	}
	client := &http.Client{Transport: tr}

	req, _ := http.NewRequest(method, "http://localhost:8080/weaviate/v1"+endpoint, body)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", accept)

	if apiKey != "" {
		req.Header.Set("X-API-KEY", apiKey)
	}

	response, err := client.Do(req)

	if err != nil {
		panic(err)
	}

	return response

}

func getResponseBody(response *http.Response) []byte {
	defer response.Body.Close()
	body, _ := ioutil.ReadAll(response.Body)

	return body
}

var apiKeyCmdLine string
var locationId string

func init() {
	flag.StringVar(&apiKeyCmdLine, "api-key", "", "API-KEY as used as haeder in the tests.")
	flag.Parse()
}

/******************
 * START TESTS
 ******************/

// weaviate.location.insert
func Test__weaviate_location_insert_JSON(t *testing.T) {
	// Create insert request
	jsonStr := bytes.NewBuffer([]byte(`{"address_components":[{"long_name":"TEST","short_name":"string","types":["UNDEFINED"]}],"formatted_address":"string","geometry":{"location":{},"location_type":"string","viewport":{"northeast":{},"southwest":{}}},"place_id":"string","types":["UNDEFINED"]} `))
	response := doRequest("/locations", "POST", "application/json", jsonStr, apiKeyCmdLine)

	// Check status code of insert
	if response.StatusCode != http.StatusAccepted {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusAccepted, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check whether generated UUID is added
	locationId = string(respObject.ID)

	if strfmt.IsUUID4(locationId) {
		t.Errorf("ID is not what expected. Got %s.\n", locationId)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)
}

// weaviate.location.list
func Test__weaviate_location_list_JSON(t *testing.T) {
	// Create list request
	response := doRequest("/locations", "GET", "application/json", nil, apiKeyCmdLine)

	// Check status code of list
	if response.StatusCode != http.StatusOK {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusOK, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationsListResponse{}
	json.Unmarshal(body, respObject)

	// Check most recent
	if string(respObject.Locations[0].ID) != locationId {
		t.Errorf("Expected ID %s. Got %s\n", locationId, respObject.Locations[0].ID)
	}
}

// weaviate.location.get
func Test__weaviate_location_get_JSON(t *testing.T) {
	// Create get request
	response := doRequest("/locations/"+locationId, "GET", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if response.StatusCode != http.StatusOK {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusOK, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check ID of object
	if string(respObject.ID) != locationId {
		t.Errorf("Expected ID %s. Got %s\n", locationId, respObject.ID)
	}

	// Create get request with non-existing location
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "GET", "application/json", nil, apiKeyCmdLine)

	// Check response of non-existing location
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.update
func Test__weaviate_location_update_JSON(t *testing.T) {
	// Create update request
	newLongName := "updated_name"
	jsonStr := bytes.NewBuffer([]byte(`{"address_components":[{"long_name":"` + newLongName + `","short_name":"string","types":["UNDEFINED"]}],"formatted_address":"string","geometry":{"location":{},"location_type":"string","viewport":{"northeast":{},"southwest":{}}},"place_id":"","types":["UNDEFINED"]} `))
	response := doRequest("/locations/"+locationId, "PUT", "application/json", jsonStr, apiKeyCmdLine)

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check location ID is same
	if string(respObject.ID) != locationId {
		t.Errorf("Expected ID %s. Got %s\n", locationId, respObject.ID)
	}

	// Check name after update
	if respObject.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected updated Long Name %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Check if update is also applied on object when using a new GET request on same object
	responseGet := doRequest("/locations/"+locationId, "GET", "application/json", nil, apiKeyCmdLine)

	bodyGet := getResponseBody(responseGet)

	respObjectGet := &models.Location{}
	json.Unmarshal(bodyGet, respObjectGet)

	// Check name after update and get
	if respObjectGet.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected updated Long Name after GET %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Check put on non-existing ID
	emptyJSON := bytes.NewBuffer([]byte(`{}`))
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "PUT", "application/json", emptyJSON, apiKeyCmdLine)
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code for not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.patch
func Test__weaviate_location_patch_JSON(t *testing.T) {
	// Create patch request
	newLongName := "patched_name"

	jsonStr := bytes.NewBuffer([]byte(`[{ "op": "replace", "path": "/address_components/0/long_name", "value": "` + newLongName + `"}]`))
	response := doRequest("/locations/"+locationId, "PATCH", "application/json", jsonStr, apiKeyCmdLine)

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check name is the same
	if string(respObject.ID) != locationId {
		t.Errorf("Expected ID %s. Got %s\n", locationId, respObject.ID)
	}

	// Check ID after patch
	if respObject.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected patched Long Name %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Check if patch is also applied on object when using a new GET request on same object
	responseGet := doRequest("/locations/"+locationId, "GET", "application/json", nil, apiKeyCmdLine)

	bodyGet := getResponseBody(responseGet)

	respObjectGet := &models.LocationGetResponse{}
	json.Unmarshal(bodyGet, respObjectGet)

	// Check name after patch and get
	if respObjectGet.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected patched Long Name after GET %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Check patch with incorrect contents
	jsonStrError := bytes.NewBuffer([]byte(`{ "op": "replace", "path": "/address_components/long_name", "value": "` + newLongName + `"}`))
	responseError := doRequest("/locations/"+locationId, "PATCH", "application/json", jsonStrError, apiKeyCmdLine)

	if responseError.StatusCode != http.StatusBadRequest {
		t.Errorf("Expected response code for wrong input %d. Got %d\n", http.StatusBadRequest, responseError.StatusCode)
	}

	// Check patch on non-existing ID
	emptyJSON := bytes.NewBuffer([]byte(`[{}]`))
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "PATCH", "application/json", emptyJSON, apiKeyCmdLine)

	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code for not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.delete
func Test__weaviate_location_delete_JSON(t *testing.T) {
	// Create delete request
	response := doRequest("/locations/"+locationId, "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if response.StatusCode != http.StatusNoContent {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusNoContent, response.StatusCode)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Create delete request
	responseAlreadyDeleted := doRequest("/locations/"+locationId, "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if responseAlreadyDeleted.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code already deleted %d. Got %d\n", http.StatusNotFound, responseAlreadyDeleted.StatusCode)
	}

	// Create get request with non-existing location
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check response of non-existing location
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.thing_template.create
func Test__weaviate_thing_templates_create_JSON(t *testing.T) {
	// Create create request
	jsonStr := bytes.NewBuffer([]byte(`{
		"commandsId": "11111111-1111-1111-1111-111111111111",
		"thingModelTemplate": {
			"modelName": "TEST name",
			"oemNumber": "TEST number",
			"oemAdditions": {
				"testAdd": 1,
				"testAdd2": "two",
				"testAddObj": {
					"testobj": "value"
				}
			},
			"oemName": "TEST oemname",
			"oemIcon": "TEST oemicon",
			"oemContact": "TEST oemcontact"
		},
		"name": "TEST THING TEMPLATE"
		}`))
	response := doRequest("/thingTemplates", "POST", "application/json", jsonStr, apiKeyCmdLine)

	// Check status code of create
	if response.StatusCode != http.StatusAccepted {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusAccepted, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.ThingGetResponse{}
	json.Unmarshal(body, respObject)

	// Check whether generated UUID is added
	thingTemplateID := string(respObject.ID)

	if strfmt.IsUUID4(thingTemplateID) {
		t.Errorf("ID is not what expected. Got %s.\n", thingTemplateID)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)
}
<commit_msg>gh-75: Fix tests and add List test.<commit_after>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 Weaviate. All rights reserved.
 * LICENSE: https://github.com/weaviate/weaviate/blob/master/LICENSE
 * AUTHOR: Bob van Luijt (bob@weaviate.com)
 * See www.weaviate.com for details
 * Contact: @weaviate_iot / yourfriends@weaviate.com
 */
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	// "github.com/weaviate/weaviate/connectors"
	// "github.com/weaviate/weaviate/connectors/datastore"
	"github.com/go-openapi/strfmt"
	"github.com/weaviate/weaviate/models"
)

/*
 * Request function
 */
func doRequest(endpoint string, method string, accept string, body io.Reader, apiKey string) *http.Response {
	tr := &http.Transport{
		MaxIdleConns:       10,
		IdleConnTimeout:    30 * time.Second,
		DisableCompression: true,
	}
	client := &http.Client{Transport: tr}

	req, _ := http.NewRequest(method, "http://localhost:8080/weaviate/v1"+endpoint, body)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", accept)

	if apiKey != "" {
		req.Header.Set("X-API-KEY", apiKey)
	}

	response, err := client.Do(req)

	if err != nil {
		panic(err)
	}

	return response

}

func getResponseBody(response *http.Response) []byte {
	defer response.Body.Close()
	body, _ := ioutil.ReadAll(response.Body)

	return body
}

var apiKeyCmdLine string
var locationID string
var thingTemplateID string

func init() {
	flag.StringVar(&apiKeyCmdLine, "api-key", "", "API-KEY as used as haeder in the tests.")
	flag.Parse()
}

/******************
 * START TESTS
 ******************/

// weaviate.location.insert
func Test__weaviate_location_insert_JSON(t *testing.T) {
	// Create insert request
	jsonStr := bytes.NewBuffer([]byte(`{"address_components":[{"long_name":"TEST","short_name":"string","types":["UNDEFINED"]}],"formatted_address":"string","geometry":{"location":{},"location_type":"string","viewport":{"northeast":{},"southwest":{}}},"place_id":"string","types":["UNDEFINED"]} `))
	response := doRequest("/locations", "POST", "application/json", jsonStr, apiKeyCmdLine)

	// Check status code of insert
	if response.StatusCode != http.StatusAccepted {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusAccepted, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check whether generated UUID is added
	locationID = string(respObject.ID)

	if !strfmt.IsUUID(locationID) {
		t.Errorf("ID is not what expected. Got %s.\n", locationID)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)
}

// weaviate.location.list
func Test__weaviate_location_list_JSON(t *testing.T) {
	// Create list request
	response := doRequest("/locations", "GET", "application/json", nil, apiKeyCmdLine)

	// Check status code of list
	if response.StatusCode != http.StatusOK {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusOK, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationsListResponse{}
	json.Unmarshal(body, respObject)

	// Check most recent
	if string(respObject.Locations[0].ID) != locationID {
		t.Errorf("Expected ID %s. Got %s\n", locationID, respObject.Locations[0].ID)
	}
}

// weaviate.location.get
func Test__weaviate_location_get_JSON(t *testing.T) {
	// Create get request
	response := doRequest("/locations/"+locationID, "GET", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if response.StatusCode != http.StatusOK {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusOK, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check ID of object
	if string(respObject.ID) != locationID {
		t.Errorf("Expected ID %s. Got %s\n", locationID, respObject.ID)
	}

	// Create get request with non-existing location
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "GET", "application/json", nil, apiKeyCmdLine)

	// Check response of non-existing location
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.update
func Test__weaviate_location_update_JSON(t *testing.T) {
	// Create update request
	newLongName := "updated_name"
	jsonStr := bytes.NewBuffer([]byte(`{"address_components":[{"long_name":"` + newLongName + `","short_name":"string","types":["UNDEFINED"]}],"formatted_address":"string","geometry":{"location":{},"location_type":"string","viewport":{"northeast":{},"southwest":{}}},"place_id":"","types":["UNDEFINED"]} `))
	response := doRequest("/locations/"+locationID, "PUT", "application/json", jsonStr, apiKeyCmdLine)

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check location ID is same
	if string(respObject.ID) != locationID {
		t.Errorf("Expected ID %s. Got %s\n", locationID, respObject.ID)
	}

	// Check name after update
	if respObject.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected updated Long Name %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Check if update is also applied on object when using a new GET request on same object
	responseGet := doRequest("/locations/"+locationID, "GET", "application/json", nil, apiKeyCmdLine)

	bodyGet := getResponseBody(responseGet)

	respObjectGet := &models.Location{}
	json.Unmarshal(bodyGet, respObjectGet)

	// Check name after update and get
	if respObjectGet.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected updated Long Name after GET %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Check put on non-existing ID
	emptyJSON := bytes.NewBuffer([]byte(`{}`))
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "PUT", "application/json", emptyJSON, apiKeyCmdLine)
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code for not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.patch
func Test__weaviate_location_patch_JSON(t *testing.T) {
	// Create patch request
	newLongName := "patched_name"

	jsonStr := bytes.NewBuffer([]byte(`[{ "op": "replace", "path": "/address_components/0/long_name", "value": "` + newLongName + `"}]`))
	response := doRequest("/locations/"+locationID, "PATCH", "application/json", jsonStr, apiKeyCmdLine)

	body := getResponseBody(response)

	respObject := &models.LocationGetResponse{}
	json.Unmarshal(body, respObject)

	// Check name is the same
	if string(respObject.ID) != locationID {
		t.Errorf("Expected ID %s. Got %s\n", locationID, respObject.ID)
	}

	// Check ID after patch
	if respObject.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected patched Long Name %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Check if patch is also applied on object when using a new GET request on same object
	responseGet := doRequest("/locations/"+locationID, "GET", "application/json", nil, apiKeyCmdLine)

	bodyGet := getResponseBody(responseGet)

	respObjectGet := &models.LocationGetResponse{}
	json.Unmarshal(bodyGet, respObjectGet)

	// Check name after patch and get
	if respObjectGet.AddressComponents[0].LongName != newLongName {
		t.Errorf("Expected patched Long Name after GET %s. Got %s\n", newLongName, respObject.AddressComponents[0].LongName)
	}

	// Check patch with incorrect contents
	jsonStrError := bytes.NewBuffer([]byte(`{ "op": "replace", "path": "/address_components/long_name", "value": "` + newLongName + `"}`))
	responseError := doRequest("/locations/"+locationID, "PATCH", "application/json", jsonStrError, apiKeyCmdLine)

	if responseError.StatusCode != http.StatusBadRequest {
		t.Errorf("Expected response code for wrong input %d. Got %d\n", http.StatusBadRequest, responseError.StatusCode)
	}

	// Check patch on non-existing ID
	emptyJSON := bytes.NewBuffer([]byte(`[{}]`))
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "PATCH", "application/json", emptyJSON, apiKeyCmdLine)

	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code for not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.location.delete
func Test__weaviate_location_delete_JSON(t *testing.T) {
	// Create delete request
	response := doRequest("/locations/"+locationID, "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if response.StatusCode != http.StatusNoContent {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusNoContent, response.StatusCode)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)

	// Create delete request
	responseAlreadyDeleted := doRequest("/locations/"+locationID, "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check status code get request
	if responseAlreadyDeleted.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code already deleted %d. Got %d\n", http.StatusNotFound, responseAlreadyDeleted.StatusCode)
	}

	// Create get request with non-existing location
	responseNotFound := doRequest("/locations/11111111-1111-1111-1111-111111111111", "DELETE", "application/json", nil, apiKeyCmdLine)

	// Check response of non-existing location
	if responseNotFound.StatusCode != http.StatusNotFound {
		t.Errorf("Expected response code not found %d. Got %d\n", http.StatusNotFound, responseNotFound.StatusCode)
	}
}

// weaviate.thing_template.create
func Test__weaviate_thing_templates_create_JSON(t *testing.T) {
	// Create create request
	jsonStr := bytes.NewBuffer([]byte(`{
		"commandsId": "11111111-1111-1111-1111-111111111111",
		"thingModelTemplate": {
			"modelName": "TEST name",
			"oemNumber": "TEST number",
			"oemAdditions": {
				"testAdd": 1,
				"testAdd2": "two",
				"testAddObj": {
					"testobj": "value"
				}
			},
			"oemName": "TEST oemname",
			"oemIcon": "TEST oemicon",
			"oemContact": "TEST oemcontact"
		},
		"name": "TEST THING TEMPLATE"
		}`))
	response := doRequest("/thingTemplates", "POST", "application/json", jsonStr, apiKeyCmdLine)

	// Check status code of create
	if response.StatusCode != http.StatusAccepted {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusAccepted, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.ThingGetResponse{}
	json.Unmarshal(body, respObject)

	// Check whether generated UUID is added
	thingTemplateID = string(respObject.ID)

	if !strfmt.IsUUID(thingTemplateID) {
		t.Errorf("ID is not what expected. Got %s.\n", thingTemplateID)
	}

	// Test is faster than adding to DB.
	time.Sleep(1 * time.Second)
}

// weaviate.thing_templates.list
func Test__weaviate_thing_templates_list_JSON(t *testing.T) {
	// Create list request
	response := doRequest("/thingTemplates", "GET", "application/json", nil, apiKeyCmdLine)

	// Check status code of list
	if response.StatusCode != http.StatusOK {
		t.Errorf("Expected response code %d. Got %d\n", http.StatusOK, response.StatusCode)
	}

	body := getResponseBody(response)

	respObject := &models.ThingTemplatesListResponse{}
	json.Unmarshal(body, respObject)

	// Check most recent
	if string(respObject.ThingTemplates[0].ID) != thingTemplateID {
		t.Errorf("Expected ID %s. Got %s\n", thingTemplateID, respObject.ThingTemplates[0].ID)
	}
}
<|endoftext|>


<commit_before><commit_msg>Fix issue<commit_after><|endoftext|>


<commit_before>package main

// Print the MX records of a domain
// (c) Miek Gieben - 2011
import (
	"dns"
        "os"
        "fmt"
)

func main() {
	r := new(dns.Resolver)
        r.FromFile("/etc/resolv.conf")
        if len(os.Args) != 2 {
                fmt.Printf("%s DOMAIN\n", os.Args[0])
                os.Exit(1)
        }
        m := new(dns.Msg)
        m.MsgHdr.RecursionDesired = true //only set this bit
        m.Question = make([]dns.Question, 1)
        m.Question[0] = dns.Question{os.Args[1], dns.TypeDNSKEY, dns.ClassINET}

        in, err := r.Query(m)
        if in != nil {
                if in.Rcode != dns.RcodeSuccess {
                        fmt.Printf(" *** invalid answer name %s after DNSKEY query for %s\n", os.Args[1], os.Args[1])
                        os.Exit(1)
                }
                // Stuff must be in the answer section
                for _, k := range in.Answer {
                        // Foreach key would need to provide a DS records, both sha1 and sha256
                        if key, ok := k.(*dns.RR_DNSKEY); ok {
                                ds := key.ToDS(dns.HashSHA1)
                                fmt.Printf("%v\n", ds)
                                ds = key.ToDS(dns.HashSHA256)
                                fmt.Printf("%v\n", ds)
                        }
                }
        } else {
                fmt.Printf("*** error: %s\n", err.String())
        }
}
<commit_msg>tweak comment<commit_after>package main

// Print the DNSKEY records of a domain as DS records
// (c) Miek Gieben - 2011
import (
	"dns"
        "os"
        "fmt"
)

func main() {
	r := new(dns.Resolver)
        r.FromFile("/etc/resolv.conf")
        if len(os.Args) != 2 {
                fmt.Printf("%s DOMAIN\n", os.Args[0])
                os.Exit(1)
        }
        m := new(dns.Msg)
        m.MsgHdr.RecursionDesired = true //only set this bit
        m.Question = make([]dns.Question, 1)
        m.Question[0] = dns.Question{os.Args[1], dns.TypeDNSKEY, dns.ClassINET}

        in, err := r.Query(m)
        if in != nil {
                if in.Rcode != dns.RcodeSuccess {
                        fmt.Printf(" *** invalid answer name %s after DNSKEY query for %s\n", os.Args[1], os.Args[1])
                        os.Exit(1)
                }
                // Stuff must be in the answer section
                for _, k := range in.Answer {
                        // Foreach key would need to provide a DS records, both sha1 and sha256
                        if key, ok := k.(*dns.RR_DNSKEY); ok {
                                ds := key.ToDS(dns.HashSHA1)
                                fmt.Printf("%v\n", ds)
                                ds = key.ToDS(dns.HashSHA256)
                                fmt.Printf("%v\n", ds)
                        }
                }
        } else {
                fmt.Printf("*** error: %s\n", err.String())
        }
}
<|endoftext|>


<commit_before><commit_msg>Add support for generating labels to annotations_prep (#1277)<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/Shopify/sarama"
	"gopkg.in/mcuadros/go-syslog.v2"
	"log"
	"os"
	"strings"
)

var (
	addr    = flag.String("addr", "0.0.0.0:514", "The address to bind to")
	brokers = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The Kafka brokers to connect to, as a comma separated list")
	verbose = flag.Bool("verbose", false, "Turn on Sarama logging")
)

func main() {
	flag.Parse()

	if *verbose {
		sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
	}

	if *brokers == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	brokerList := strings.Split(*brokers, ",")
	log.Printf("Kafka brokers: %s", strings.Join(brokerList, ", "))

	channel := make(syslog.LogPartsChannel)
	handler := syslog.NewChannelHandler(channel)

	server := syslog.NewServer()
	//server.SetFormat(syslog.RFC5424)
	server.SetFormat(syslog.RFC3164) //BSD format
	server.SetHandler(handler)
	server.ListenUDP(*addr)
	server.Boot()
	dataCollector := newDataCollector(brokerList)

	go func(channel syslog.LogPartsChannel) {
		for logParts := range channel {
			var encoded, err = json.Marshal(logParts)
			fmt.Println(string(encoded))
			if err != nil {
				fmt.Println("Failed to Marshall the syslog data:, %s", err)
			}
			// We are not setting a message key, which means that all messages will
			// be distributed randomly over the different partitions.
			partition, offset, err := dataCollector.SendMessage(&sarama.ProducerMessage{
				Topic: "syslog-passeplat-test",
				Value: sarama.StringEncoder(encoded),
			})

			if err != nil {
				fmt.Println("Failed to store your data:, %s", err)
			} else {
				// The tuple (topic, partition, offset) can be used as a unique identifier
				// for a message in a Kafka cluster.
				fmt.Println("Your data is stored with unique identifier syslog-passeplat-test/%d/%d", partition, offset)
			}
		}
	}(channel)

	server.Wait()
}

func newDataCollector(brokerList []string) sarama.SyncProducer {

	// For the data collector, we are looking for strong consistency semantics.
	// Because we don't change the flush settings, sarama will try to produce messages
	// as fast as possible to keep latency low.
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message
	config.Producer.Retry.Max = 10                   // Retry up to 10 times to produce the message
	config.Producer.Return.Successes = true
	// On the broker side, you may want to change the following settings to get
	// stronger consistency guarantees:
	// - For your broker, set `unclean.leader.election.enable` to false
	// - For the topic, you could increase `min.insync.replicas`.

	producer, err := sarama.NewSyncProducer(brokerList, config)
	if err != nil {
		log.Fatalln("Failed to start Sarama producer:", err)
	}

	return producer
}
<commit_msg>add tcp listener<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/Shopify/sarama"
	"gopkg.in/mcuadros/go-syslog.v2"
	"log"
	"os"
	"strings"
)

var (
	addr    = flag.String("addr", "0.0.0.0:514", "The address to bind to")
	brokers = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The Kafka brokers to connect to, as a comma separated list")
	verbose = flag.Bool("verbose", false, "Turn on Sarama logging")
)

func main() {
	flag.Parse()

	if *verbose {
		sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
	}

	if *brokers == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	brokerList := strings.Split(*brokers, ",")
	log.Printf("Kafka brokers: %s", strings.Join(brokerList, ", "))

	channel := make(syslog.LogPartsChannel)
	handler := syslog.NewChannelHandler(channel)

	server := syslog.NewServer()
	//server.SetFormat(syslog.RFC5424)
	server.SetFormat(syslog.RFC3164) //BSD format
	server.SetHandler(handler)
	server.ListenUDP(*addr)
	server.ListenTCP(*addr)
	server.Boot()
	dataCollector := newDataCollector(brokerList)

	go func(channel syslog.LogPartsChannel) {
		for logParts := range channel {
			var encoded, err = json.Marshal(logParts)
			fmt.Println(string(encoded))
			if err != nil {
				fmt.Println("Failed to Marshall the syslog data:, %s", err)
			}
			// We are not setting a message key, which means that all messages will
			// be distributed randomly over the different partitions.
			partition, offset, err := dataCollector.SendMessage(&sarama.ProducerMessage{
				Topic: "syslog-passeplat-test",
				Value: sarama.StringEncoder(encoded),
			})

			if err != nil {
				fmt.Println("Failed to store your data:, %s", err)
			} else {
				// The tuple (topic, partition, offset) can be used as a unique identifier
				// for a message in a Kafka cluster.
				fmt.Println("Your data is stored with unique identifier syslog-passeplat-test/%d/%d", partition, offset)
			}
		}
	}(channel)

	server.Wait()
}

func newDataCollector(brokerList []string) sarama.SyncProducer {

	// For the data collector, we are looking for strong consistency semantics.
	// Because we don't change the flush settings, sarama will try to produce messages
	// as fast as possible to keep latency low.
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message
	config.Producer.Retry.Max = 10                   // Retry up to 10 times to produce the message
	config.Producer.Return.Successes = true
	// On the broker side, you may want to change the following settings to get
	// stronger consistency guarantees:
	// - For your broker, set `unclean.leader.election.enable` to false
	// - For the topic, you could increase `min.insync.replicas`.

	producer, err := sarama.NewSyncProducer(brokerList, config)
	if err != nil {
		log.Fatalln("Failed to start Sarama producer:", err)
	}

	return producer
}
<|endoftext|>


<commit_before>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

var a = []int { 1, 2, }
var b = []int { }
var c = []int { 1 }

func main() {
	if len(a) != 2 { panicln("len a", len(a)) }
	if len(b) != 5 { panicln("len b", len(b)) }
	if len(c) != 1 { panicln("len a", len(c)) }
}
<commit_msg>fixed and added more tests<commit_after>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

var a = []int { 1,2, }
var b = [5]int { 1,2,3 }
var c = []int { 1 }
var d = [...]int { 1,2,3 }

func main() {
	if len(a) != 2 { panicln("len a", len(a)) }
	if len(b) != 5 { panicln("len b", len(b)) }
	if len(c) != 1 { panicln("len d", len(c)) }
	if len(d) != 3 { panicln("len c", len(d)) }

	if a[0] != 1 { panicln("a[0]", a[0]) }
	if a[1] != 2 { panicln("a[1]", a[1]) }

	if b[0] != 1 { panicln("b[0]", b[0]) }
	if b[1] != 2 { panicln("b[1]", b[1]) }
	if b[2] != 3 { panicln("b[2]", b[2]) }
	if b[3] != 0 { panicln("b[3]", b[3]) }
	if b[4] != 0 { panicln("b[4]", b[4]) }

	if c[0] != 1 { panicln("c[0]", c[0]) }

	if d[0] != 1 { panicln("d[0]", d[0]) }
	if d[1] != 2 { panicln("d[1]", d[1]) }
	if d[2] != 3 { panicln("d[2]", d[2]) }
}
<|endoftext|>


<commit_before><commit_msg>style(rename get.go to get_pipelines.go): rename cmd help description vars<commit_after><|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbconfigs

// This file contains logic for a plugable credentials system.
// The default implementation is file based.
// The flags are global, but only programs that need to access the database
// link with this library, so we should be safe.

import (
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"sync"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/log"
)

var (
	// generic flags
	dbCredentialsServer = flag.String("db-credentials-server", "file", "db credentials server type (use 'file' for the file implementation)")

	// 'file' implementation flags
	dbCredentialsFile = flag.String("db-credentials-file", "", "db credentials file")

	// ErrUnknownUser is returned by credential server when the
	// user doesn't exist
	ErrUnknownUser = errors.New("unknown user")
)

// CredentialsServer is the interface for a credential server
type CredentialsServer interface {
	// GetUserAndPassword returns the user / password to use for a given
	// user. May return ErrUnknownUser. The user might be altered
	// to support versioned users.
	// Note this call needs to be thread safe, as we may call this from
	// multiple go routines.
	GetUserAndPassword(user string) (string, string, error)
}

// AllCredentialsServers contains all the known CredentialsServer
// implementations.  Note we will only access this after flags have
// been parsed.
var AllCredentialsServers = make(map[string]CredentialsServer)

// GetCredentialsServer returns the current CredentialsServer. Only valid
// after flag.Init was called.
func GetCredentialsServer() CredentialsServer {
	cs, ok := AllCredentialsServers[*dbCredentialsServer]
	if !ok {
		log.Exitf("Invalid credential server: %v", *dbCredentialsServer)
	}
	return cs
}

// FileCredentialsServer is a simple implementation of CredentialsServer using
// a json file. Protected by mu.
type FileCredentialsServer struct {
	mu            sync.Mutex
	dbCredentials map[string][]string
}

// GetUserAndPassword is part of the CredentialsServer interface
func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	fcs.mu.Lock()
	defer fcs.mu.Unlock()

	if *dbCredentialsFile == "" {
		return "", "", ErrUnknownUser
	}

	// read the json file only once
	if fcs.dbCredentials == nil {
		fcs.dbCredentials = make(map[string][]string)

		data, err := ioutil.ReadFile(*dbCredentialsFile)
		if err != nil {
			log.Warningf("Failed to read dbCredentials file: %v", *dbCredentialsFile)
			return "", "", err
		}

		if err = json.Unmarshal(data, &fcs.dbCredentials); err != nil {
			log.Warningf("Failed to parse dbCredentials file: %v", *dbCredentialsFile)
			return "", "", err
		}
	}

	passwd, ok := fcs.dbCredentials[user]
	if !ok {
		return "", "", ErrUnknownUser
	}
	return user, passwd[0], nil
}

// WithCredentials returns a copy of the provided ConnParams that we can use
// to connect, after going through the CredentialsServer.
func WithCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) {
	result := *cp
	user, passwd, err := GetCredentialsServer().GetUserAndPassword(cp.Uname)
	switch err {
	case nil:
		result.Uname = user
		result.Pass = passwd
	case ErrUnknownUser:
		// we just use what we have, and will fail later anyway
		err = nil
	}
	return &result, err
}

func init() {
	AllCredentialsServers["file"] = &FileCredentialsServer{}
}
<commit_msg>reload db-credentials-file upon SIGHUP<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbconfigs

// This file contains logic for a plugable credentials system.
// The default implementation is file based.
// The flags are global, but only programs that need to access the database
// link with this library, so we should be safe.

import (
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/log"
)

var (
	// generic flags
	dbCredentialsServer = flag.String("db-credentials-server", "file", "db credentials server type (use 'file' for the file implementation)")

	// 'file' implementation flags
	dbCredentialsFile = flag.String("db-credentials-file", "", "db credentials file")

	// ErrUnknownUser is returned by credential server when the
	// user doesn't exist
	ErrUnknownUser = errors.New("unknown user")
)

// CredentialsServer is the interface for a credential server
type CredentialsServer interface {
	// GetUserAndPassword returns the user / password to use for a given
	// user. May return ErrUnknownUser. The user might be altered
	// to support versioned users.
	// Note this call needs to be thread safe, as we may call this from
	// multiple go routines.
	GetUserAndPassword(user string) (string, string, error)
}

// AllCredentialsServers contains all the known CredentialsServer
// implementations.  Note we will only access this after flags have
// been parsed.
var AllCredentialsServers = make(map[string]CredentialsServer)

// GetCredentialsServer returns the current CredentialsServer. Only valid
// after flag.Init was called.
func GetCredentialsServer() CredentialsServer {
	cs, ok := AllCredentialsServers[*dbCredentialsServer]
	if !ok {
		log.Exitf("Invalid credential server: %v", *dbCredentialsServer)
	}
	return cs
}

// FileCredentialsServer is a simple implementation of CredentialsServer using
// a json file. Protected by mu.
type FileCredentialsServer struct {
	mu            sync.Mutex
	dbCredentials map[string][]string
}

// GetUserAndPassword is part of the CredentialsServer interface
func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	fcs.mu.Lock()
	defer fcs.mu.Unlock()

	if *dbCredentialsFile == "" {
		return "", "", ErrUnknownUser
	}

	// read the json file only once
	if fcs.dbCredentials == nil {
		fcs.dbCredentials = make(map[string][]string)

		data, err := ioutil.ReadFile(*dbCredentialsFile)
		if err != nil {
			log.Warningf("Failed to read dbCredentials file: %v", *dbCredentialsFile)
			return "", "", err
		}

		if err = json.Unmarshal(data, &fcs.dbCredentials); err != nil {
			log.Warningf("Failed to parse dbCredentials file: %v", *dbCredentialsFile)
			return "", "", err
		}
	}

	passwd, ok := fcs.dbCredentials[user]
	if !ok {
		return "", "", ErrUnknownUser
	}
	return user, passwd[0], nil
}

// WithCredentials returns a copy of the provided ConnParams that we can use
// to connect, after going through the CredentialsServer.
func WithCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) {
	result := *cp
	user, passwd, err := GetCredentialsServer().GetUserAndPassword(cp.Uname)
	switch err {
	case nil:
		result.Uname = user
		result.Pass = passwd
	case ErrUnknownUser:
		// we just use what we have, and will fail later anyway
		err = nil
	}
	return &result, err
}

func init() {
	AllCredentialsServers["file"] = &FileCredentialsServer{}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGHUP)
	go func() {
		for {
			<-sigChan
			fcs, ok := AllCredentialsServers["file"].(*FileCredentialsServer)
			if ok {
				fcs.dbCredentials = nil
			}
		}
	}()
}
<|endoftext|>


<commit_before>package sqlbuilder

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

func TestLiteralConvert(t *testing.T) {
	a := assert.New(t)
	type testcase struct {
		in  literal
		out interface{}
		err bool
	}
	var cases = []testcase{
		{toLiteral(int(10)), int64(10), false},
		{toLiteral(int64(10)), int64(10), false},
		{toLiteral(uint(10)), uint64(10), false},
		{toLiteral(uint64(10)), uint64(10), false},
		{toLiteral(float32(10)), float64(10), false},
		{toLiteral(float64(10)), float64(10), false},
		{toLiteral(bool(true)), bool(true), false},
		{toLiteral([]byte{0x11}), []byte{0x11}, false},
		{toLiteral(string("makise-kurisu")), string("makise-kurisu"), false},
		{toLiteral(time.Unix(0, 0)), time.Unix(0, 0), false},
		{toLiteral(complex(0, 0)), nil, true},
	}

	for _, c := range cases {
		val, err := c.in.(*literalImpl).converted()
		a.Equal(c.out, val)
		if c.err {
			a.Error(err)
		} else {
			a.NoError(err)
		}
	}
}

func TestLiteralString(t *testing.T) {
	a := assert.New(t)
	type testcase struct {
		in  literal
		out string
		err bool
	}
	var cases = []testcase{
		{toLiteral(int(10)), "10", false},
		{toLiteral(int64(10)), "10", false},
		{toLiteral(uint(10)), "10", false},
		{toLiteral(uint64(10)), "10", false},
		{toLiteral(float32(10)), "10.0000000000", false},
		{toLiteral(float64(10)), "10.0000000000", false},
		{toLiteral(bool(true)), "true", false},
		{toLiteral([]byte{0x11}), string([]byte{0x11}), false},
		{toLiteral(string("shibuya-rin")), "shibuya-rin", false},
		{toLiteral(time.Unix(0, 0)), "1970-01-01 09:00:00", false},
		{toLiteral(complex(0, 0)), "", true},
	}

	for _, c := range cases {
		val := c.in.(*literalImpl).string()
		a.Equal(c.out, val)
	}
}
<commit_msg>Fixed using UTC in testcase<commit_after>package sqlbuilder

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

func TestLiteralConvert(t *testing.T) {
	a := assert.New(t)
	type testcase struct {
		in  literal
		out interface{}
		err bool
	}
	var cases = []testcase{
		{toLiteral(int(10)), int64(10), false},
		{toLiteral(int64(10)), int64(10), false},
		{toLiteral(uint(10)), uint64(10), false},
		{toLiteral(uint64(10)), uint64(10), false},
		{toLiteral(float32(10)), float64(10), false},
		{toLiteral(float64(10)), float64(10), false},
		{toLiteral(bool(true)), bool(true), false},
		{toLiteral([]byte{0x11}), []byte{0x11}, false},
		{toLiteral(string("makise-kurisu")), string("makise-kurisu"), false},
		{toLiteral(time.Unix(0, 0)), time.Unix(0, 0), false},
		{toLiteral(complex(0, 0)), nil, true},
	}

	for _, c := range cases {
		val, err := c.in.(*literalImpl).converted()
		a.Equal(c.out, val)
		if c.err {
			a.Error(err)
		} else {
			a.NoError(err)
		}
	}
}

func TestLiteralString(t *testing.T) {
	a := assert.New(t)
	type testcase struct {
		in  literal
		out string
		err bool
	}
	var cases = []testcase{
		{toLiteral(int(10)), "10", false},
		{toLiteral(int64(10)), "10", false},
		{toLiteral(uint(10)), "10", false},
		{toLiteral(uint64(10)), "10", false},
		{toLiteral(float32(10)), "10.0000000000", false},
		{toLiteral(float64(10)), "10.0000000000", false},
		{toLiteral(bool(true)), "true", false},
		{toLiteral([]byte{0x11}), string([]byte{0x11}), false},
		{toLiteral(string("shibuya-rin")), "shibuya-rin", false},
		{toLiteral(time.Unix(0, 0).UTC()), "1970-01-01 00:00:00", false},
		{toLiteral(complex(0, 0)), "", true},
	}

	for _, c := range cases {
		val := c.in.(*literalImpl).string()
		a.Equal(c.out, val)
	}
}
<|endoftext|>


<commit_before>package godevicedetect

import (
	"encoding/json"
	"fmt"
	"hash/crc32"
	"log"
	"net/http"

	"github.com/Shaked/godevicedetect/platform"
	"github.com/Shaked/user-agents/bin/go/gouseragents"
	"github.com/gorilla/context"
)

const (
	UnknownOs      = "UnknownOs"
	UnknownVersion = "UnknownVersion"
)

type PreCompiledHandler struct{}

type Compiled struct {
	hashFunc   string
	userAgents map[string]platform.Device
}

//func() platform.Device
func (p *PreCompiledHandler) compile() *Compiled {
	v := struct {
		Meta       map[string]string                 `json:meta`
		UserAgents map[string]map[string]interface{} `json:userAgents`
	}{}

	log.Println(gouseragents.CompiledUserAgents)
	err := json.Unmarshal([]byte(gouseragents.CompiledUserAgents), &v)
	if nil != err {
		panic(err)
	}

	c := &Compiled{}

	hash := v.Meta["hash"]
	c.hashFunc = hash

	userAgents := map[string]platform.Device{}
	var device platform.Device
	for key, deviceInfo := range v.UserAgents {
		switch deviceInfo["type"].(string) {
		case "desktop":
			device = platform.NewDesktop(deviceInfo["name"].(string))
			break
		case "tablet":
			device = platform.NewTablet(deviceInfo["name"].(string))
			break
		default:
			panic("WTF?")
		}
		userAgents[key] = device
	}
	c.userAgents = userAgents

	return c
}

type DeviceDetect struct {
	r    *http.Request
	meta *Compiled
}

func NewDeviceDetect(r *http.Request, p *PreCompiledHandler) *DeviceDetect {
	meta := p.compile()
	return &DeviceDetect{r: r, meta: meta}
}

func (d *DeviceDetect) FindByUserAgent(userAgent string) platform.Device {
	key := UserAgentToKey(userAgent)
	f, ok := d.meta.userAgents[fmt.Sprint(key)]
	if ok {
		return f
	}
	return platform.NewUnknown(userAgent)
}

func (d *DeviceDetect) PlatformType() platform.Device {
	return d.FindByUserAgent(d.r.Header.Get("User-Agent"))
}

// Vars returns the route variables for the current request, if any.
func Platform(r *http.Request) platform.Device {
	rv := context.Get(r, "Platform")
	device := rv.(platform.Device)
	return device
}

type PlatformHandler interface {
	Mobile(w http.ResponseWriter, r *http.Request, d *platform.DeviceMobile)
	Tablet(w http.ResponseWriter, r *http.Request, d *platform.DeviceTablet)
	Desktop(w http.ResponseWriter, r *http.Request, d *platform.DeviceDesktop)
	Tv(w http.ResponseWriter, r *http.Request, d *platform.DeviceTv)
	Watch(w http.ResponseWriter, r *http.Request, d *platform.DeviceWatch)
	Bot(w http.ResponseWriter, r *http.Request, d *platform.DeviceBot)
	Unknown(w http.ResponseWriter, r *http.Request, d *platform.DeviceUnknown)
}

func Handler(h PlatformHandler, p *PreCompiledHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := NewDeviceDetect(r, p)
		d := m.FindByUserAgent(r.Header.Get("User-Agent"))
		switch m.PlatformType().(type) {
		case *platform.DeviceTablet:
			h.Tablet(w, r, d.(*platform.DeviceTablet))
			break
		case *platform.DeviceMobile:
			h.Mobile(w, r, d.(*platform.DeviceMobile))
			break
		case *platform.DeviceTv:
			h.Tv(w, r, d.(*platform.DeviceTv))
			break
		case *platform.DeviceWatch:
			h.Watch(w, r, d.(*platform.DeviceWatch))
			break
		case *platform.DeviceDesktop:
			h.Desktop(w, r, d.(*platform.DeviceDesktop))
			break
		case *platform.DeviceBot:
			h.Bot(w, r, d.(*platform.DeviceBot))
			break
		}

	})
}

func HandlerMux(s *http.ServeMux, p *PreCompiledHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := NewDeviceDetect(r, p)
		context.Set(r, "Platform", m.PlatformType())
		s.ServeHTTP(w, r)
	})
}

func UserAgentToKey(userAgent string) uint32 {
	return crc32.ChecksumIEEE([]byte(userAgent))
}
<commit_msg>update device support<commit_after>package godevicedetect

import (
	"encoding/json"
	"fmt"
	"hash/crc32"
	"log"
	"net/http"

	"github.com/Shaked/godevicedetect/platform"
	"github.com/Shaked/user-agents/packages/go/gouseragents"
	"github.com/gorilla/context"
)

const (
	UnknownOs      = "UnknownOs"
	UnknownVersion = "UnknownVersion"
)

type PreCompiledHandler struct{}

type Compiled struct {
	hashFunc   string
	userAgents map[string]platform.Device
}

//func() platform.Device
func (p *PreCompiledHandler) compile() *Compiled {
	v := struct {
		Meta       map[string]string                 `json:meta`
		UserAgents map[string]map[string]interface{} `json:userAgents`
	}{}

	log.Println(gouseragents.CompiledUserAgents)
	err := json.Unmarshal([]byte(gouseragents.CompiledUserAgents), &v)
	if nil != err {
		panic(err)
	}

	c := &Compiled{}

	hash := v.Meta["hash"]
	c.hashFunc = hash

	userAgents := map[string]platform.Device{}
	var device platform.Device
	for key, deviceInfo := range v.UserAgents {
		switch deviceInfo["type"].(string) {
		case "desktop":
			device = platform.NewDesktop(deviceInfo["name"].(string))
			break
		case "tablet":
			device = platform.NewTablet(deviceInfo["name"].(string))
			break
		case "app":
		case "mobile":
			device = platform.NewMobile(deviceInfo["name"].(string))
		case "bot":
			device = platform.NewBot(deviceInfo["name"].(string))
			break
		default:
			panic("WTF :" + deviceInfo["type"].(string))
		}
		userAgents[key] = device
	}
	c.userAgents = userAgents

	return c
}

type DeviceDetect struct {
	r    *http.Request
	meta *Compiled
}

func NewDeviceDetect(r *http.Request, p *PreCompiledHandler) *DeviceDetect {
	meta := p.compile()
	return &DeviceDetect{r: r, meta: meta}
}

func (d *DeviceDetect) FindByUserAgent(userAgent string) platform.Device {
	key := UserAgentToKey(userAgent)
	f, ok := d.meta.userAgents[fmt.Sprint(key)]
	if ok {
		return f
	}
	return platform.NewUnknown(userAgent)
}

func (d *DeviceDetect) PlatformType() platform.Device {
	return d.FindByUserAgent(d.r.Header.Get("User-Agent"))
}

// Vars returns the route variables for the current request, if any.
func Platform(r *http.Request) platform.Device {
	rv := context.Get(r, "Platform")
	device := rv.(platform.Device)
	return device
}

type PlatformHandler interface {
	Mobile(w http.ResponseWriter, r *http.Request, d *platform.DeviceMobile)
	Tablet(w http.ResponseWriter, r *http.Request, d *platform.DeviceTablet)
	Desktop(w http.ResponseWriter, r *http.Request, d *platform.DeviceDesktop)
	Tv(w http.ResponseWriter, r *http.Request, d *platform.DeviceTv)
	Watch(w http.ResponseWriter, r *http.Request, d *platform.DeviceWatch)
	Bot(w http.ResponseWriter, r *http.Request, d *platform.DeviceBot)
	Unknown(w http.ResponseWriter, r *http.Request, d *platform.DeviceUnknown)
}

func Handler(h PlatformHandler, p *PreCompiledHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := NewDeviceDetect(r, p)
		d := m.FindByUserAgent(r.Header.Get("User-Agent"))
		switch m.PlatformType().(type) {
		case *platform.DeviceTablet:
			h.Tablet(w, r, d.(*platform.DeviceTablet))
			break
		case *platform.DeviceMobile:
			h.Mobile(w, r, d.(*platform.DeviceMobile))
			break
		case *platform.DeviceTv:
			h.Tv(w, r, d.(*platform.DeviceTv))
			break
		case *platform.DeviceWatch:
			h.Watch(w, r, d.(*platform.DeviceWatch))
			break
		case *platform.DeviceDesktop:
			h.Desktop(w, r, d.(*platform.DeviceDesktop))
			break
		case *platform.DeviceBot:
			h.Bot(w, r, d.(*platform.DeviceBot))
			break
		}

	})
}

func HandlerMux(s *http.ServeMux, p *PreCompiledHandler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := NewDeviceDetect(r, p)
		context.Set(r, "Platform", m.PlatformType())
		s.ServeHTTP(w, r)
	})
}

func UserAgentToKey(userAgent string) uint32 {
	return crc32.ChecksumIEEE([]byte(userAgent))
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chromium

// TODO: remove when Realms are implemented.
const Realm = "chromium"
<commit_msg>[resultdb] Use realm "chromium/public"<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chromium

// TODO: remove when Realms are implemented.
const Realm = "chromium/public"
<|endoftext|>


<commit_before>//
// Written by Maxim Khitrov (November 2012)
//

// Package flowcontrol provides the tools for monitoring and limiting the
// transfer rate of an arbitrary data stream.
package flowcontrol

import (
	"math"
	"sync"
	"time"
)

// Monitor monitors and limits the transfer rate of a data stream.
type Monitor struct {
	mu      sync.Mutex    // Mutex guarding access to all internal fields
	active  bool          // Flag indicating an active transfer
	start   time.Duration // Transfer start time (clock() value)
	bytes   int64         // Total number of bytes transferred
	samples int64         // Total number of samples taken

	rSample float64 // Most recent transfer rate sample (bytes per second)
	rEMA    float64 // Exponential moving average of rSample
	rPeak   float64 // Peak transfer rate (max of all rSamples)
	rWindow float64 // rEMA window (seconds)

	sBytes int64         // Number of bytes transferred since sLast
	sLast  time.Duration // Most recent sample time (stop time when inactive)
	sRate  time.Duration // Sampling rate

	tBytes int64         // Number of bytes expected in the current transfer
	tLast  time.Duration // Time of the most recent transfer of at least 1 byte
}

// New creates a new flow control monitor. Instantaneous transfer rate is
// measured and updated for each sampleRate interval. windowSize determines the
// weight of each sample in the exponential moving average (EMA) calculation.
// The exact formulas are:
//
// 	sampleTime = currentTime - prevSampleTime
// 	sampleRate = byteCount / sampleTime
// 	weight     = 1 - exp(-sampleTime/windowSize)
// 	newRate    = weight*sampleRate + (1-weight)*oldRate
//
// The default values for sampleRate and windowSize (if <= 0) are 100ms and 1s,
// respectively.
func New(sampleRate, windowSize time.Duration) *Monitor {
	if sampleRate = clockRound(sampleRate); sampleRate <= 0 {
		sampleRate = 5 * clockRate
	}
	if windowSize <= 0 {
		windowSize = 1 * time.Second
	}
	now := clock()
	return &Monitor{
		active:  true,
		start:   now,
		rWindow: windowSize.Seconds(),
		sLast:   now,
		sRate:   sampleRate,
		tLast:   now,
	}
}

// Update records the transfer of n bytes and returns n. It should be called
// after each Read/Write operation, even if n is 0.
func (m *Monitor) Update(n int) int {
	m.mu.Lock()
	m.update(n)
	m.mu.Unlock()
	return n
}

// IO is a convenience method intended to wrap io.Reader and io.Writer method
// execution. It calls m.Update(n) and then returns (n, err) unmodified.
func (m *Monitor) IO(n int, err error) (int, error) {
	return m.Update(n), err
}

// Done marks the transfer as finished and prevents any further updates or
// limiting. Instantaneous and current transfer rates drop to 0. Update, IO, and
// Limit methods become NOOPs. It returns the total number of bytes transferred.
func (m *Monitor) Done() int64 {
	m.mu.Lock()
	if now := m.update(0); m.sBytes > 0 {
		m.reset(now)
	}
	m.active = false
	m.tLast = 0
	n := m.bytes
	m.mu.Unlock()
	return n
}

// Status represents the current Monitor status. All transfer rates are in bytes
// per second rounded to the nearest byte.
type Status struct {
	Active   bool          // Flag indicating an active transfer
	Start    time.Time     // Transfer start time
	Duration time.Duration // Time period covered by the statistics
	Idle     time.Duration // Time since the last transfer of at least 1 byte
	Bytes    int64         // Total number of bytes transferred
	Samples  int64         // Total number of samples taken
	InstRate int64         // Instantaneous transfer rate
	CurRate  int64         // Current transfer rate (EMA of InstRate)
	AvgRate  int64         // Average transfer rate (Bytes / Duration)
	PeakRate int64         // Maximum instantaneous transfer rate
	BytesRem int64         // Number of bytes remaining in the transfer
	TimeRem  time.Duration // Estimated time to completion
	Progress Percent       // Overall transfer progress
}

// Status returns current transfer status information. The returned value
// becomes static after a call to Done.
func (m *Monitor) Status() Status {
	m.mu.Lock()
	now := m.update(0)
	s := Status{
		Active:   m.active,
		Start:    clockToTime(m.start),
		Duration: m.sLast - m.start,
		Idle:     now - m.tLast,
		Bytes:    m.bytes,
		Samples:  m.samples,
		PeakRate: round(m.rPeak),
		BytesRem: m.tBytes - m.bytes,
		Progress: percentOf(float64(m.bytes), float64(m.tBytes)),
	}
	if s.BytesRem < 0 {
		s.BytesRem = 0
	}
	if s.Duration > 0 {
		rAvg := float64(s.Bytes) / s.Duration.Seconds()
		s.AvgRate = round(rAvg)
		if s.Active {
			s.InstRate = round(m.rSample)
			s.CurRate = round(m.rEMA)
			if s.BytesRem > 0 {
				if tRate := 0.8*m.rEMA + 0.2*rAvg; tRate > 0 {
					seconds := float64(s.BytesRem) / tRate
					s.TimeRem = clockRound(time.Duration(seconds * 1e9))
				}
			}
		}
	}
	m.mu.Unlock()
	return s
}

// Limit restricts the instantaneous (per-sample) data flow to rate bytes per
// second. It returns the maximum number of bytes (0 <= n <= want) that may be
// transferred immediately without exceeding the limit. If block == true, the
// call blocks until n > 0. want is returned unmodified if want < 1, rate < 1,
// or the transfer is inactive (after a call to Done).
//
// At least one byte is always allowed to be transferred in any given sampling
// period. Thus, if the sampling rate is 100ms, the lowest achievable flow rate
// is 10 bytes per second.
//
// For usage examples, see the implementation of Reader and Writer in io.go.
func (m *Monitor) Limit(want int, rate int64, block bool) (n int) {
	if want < 1 || rate < 1 {
		return want
	}
	m.mu.Lock()

	// Determine the maximum number of bytes that can be sent in one sample
	limit := round(float64(rate) * m.sRate.Seconds())
	if limit <= 0 {
		limit = 1
	}

	// If block == true, wait until m.sBytes < limit
	if now := m.update(0); block {
		for m.sBytes >= limit && m.active {
			now = m.waitNextSample(now)
		}
	}

	// Make limit <= want (unlimited if the transfer is no longer active)
	if limit -= m.sBytes; limit > int64(want) || !m.active {
		limit = int64(want)
	}
	m.mu.Unlock()

	if limit < 0 {
		limit = 0
	}
	return int(limit)
}

// SetTransferSize specifies the total size of the data transfer, which allows
// the Monitor to calculate the overall progress and time to completion.
func (m *Monitor) SetTransferSize(bytes int64) {
	if bytes < 0 {
		bytes = 0
	}
	m.mu.Lock()
	m.tBytes = bytes
	m.mu.Unlock()
}

// update accumulates the transferred byte count for the current sample until
// clock() - m.sLast >= m.sRate. The monitor status is updated once the current
// sample is done.
func (m *Monitor) update(n int) (now time.Duration) {
	if !m.active {
		return
	}
	if now = clock(); n > 0 {
		m.tLast = now
	}
	m.sBytes += int64(n)
	if sTime := now - m.sLast; sTime >= m.sRate {
		t := sTime.Seconds()
		if m.rSample = float64(m.sBytes) / t; m.rSample > m.rPeak {
			m.rPeak = m.rSample
		}

		// Exponential moving average using a method similar to *nix load
		// average calculation. Longer sampling periods carry greater weight.
		if m.samples > 0 {
			w := math.Exp(-t / m.rWindow)
			m.rEMA = m.rSample + w*(m.rEMA-m.rSample)
		} else {
			m.rEMA = m.rSample
		}
		m.reset(now)
	}
	return
}

// reset clears the current sample state in preparation for the next sample.
func (m *Monitor) reset(sampleTime time.Duration) {
	m.bytes += m.sBytes
	m.samples++
	m.sBytes = 0
	m.sLast = sampleTime
}

// waitNextSample sleeps for the remainder of the current sample. The lock is
// released and reacquired during the actual sleep period, so it's possible for
// the transfer to be inactive when this method returns.
func (m *Monitor) waitNextSample(now time.Duration) time.Duration {
	const minWait = 5 * time.Millisecond
	current := m.sLast

	// sleep until the last sample time changes (ideally, just one iteration)
	for m.sLast == current && m.active {
		d := current + m.sRate - now
		m.mu.Unlock()
		if d < minWait {
			d = minWait
		}
		time.Sleep(d)
		m.mu.Lock()
		now = m.update(0)
	}
	return now
}
<commit_msg>flowcontrol: Fixed an overflow bug for Status.TimeRem.<commit_after>//
// Written by Maxim Khitrov (November 2012)
//

// Package flowcontrol provides the tools for monitoring and limiting the
// transfer rate of an arbitrary data stream.
package flowcontrol

import (
	"math"
	"sync"
	"time"
)

// Monitor monitors and limits the transfer rate of a data stream.
type Monitor struct {
	mu      sync.Mutex    // Mutex guarding access to all internal fields
	active  bool          // Flag indicating an active transfer
	start   time.Duration // Transfer start time (clock() value)
	bytes   int64         // Total number of bytes transferred
	samples int64         // Total number of samples taken

	rSample float64 // Most recent transfer rate sample (bytes per second)
	rEMA    float64 // Exponential moving average of rSample
	rPeak   float64 // Peak transfer rate (max of all rSamples)
	rWindow float64 // rEMA window (seconds)

	sBytes int64         // Number of bytes transferred since sLast
	sLast  time.Duration // Most recent sample time (stop time when inactive)
	sRate  time.Duration // Sampling rate

	tBytes int64         // Number of bytes expected in the current transfer
	tLast  time.Duration // Time of the most recent transfer of at least 1 byte
}

// New creates a new flow control monitor. Instantaneous transfer rate is
// measured and updated for each sampleRate interval. windowSize determines the
// weight of each sample in the exponential moving average (EMA) calculation.
// The exact formulas are:
//
// 	sampleTime = currentTime - prevSampleTime
// 	sampleRate = byteCount / sampleTime
// 	weight     = 1 - exp(-sampleTime/windowSize)
// 	newRate    = weight*sampleRate + (1-weight)*oldRate
//
// The default values for sampleRate and windowSize (if <= 0) are 100ms and 1s,
// respectively.
func New(sampleRate, windowSize time.Duration) *Monitor {
	if sampleRate = clockRound(sampleRate); sampleRate <= 0 {
		sampleRate = 5 * clockRate
	}
	if windowSize <= 0 {
		windowSize = 1 * time.Second
	}
	now := clock()
	return &Monitor{
		active:  true,
		start:   now,
		rWindow: windowSize.Seconds(),
		sLast:   now,
		sRate:   sampleRate,
		tLast:   now,
	}
}

// Update records the transfer of n bytes and returns n. It should be called
// after each Read/Write operation, even if n is 0.
func (m *Monitor) Update(n int) int {
	m.mu.Lock()
	m.update(n)
	m.mu.Unlock()
	return n
}

// IO is a convenience method intended to wrap io.Reader and io.Writer method
// execution. It calls m.Update(n) and then returns (n, err) unmodified.
func (m *Monitor) IO(n int, err error) (int, error) {
	return m.Update(n), err
}

// Done marks the transfer as finished and prevents any further updates or
// limiting. Instantaneous and current transfer rates drop to 0. Update, IO, and
// Limit methods become NOOPs. It returns the total number of bytes transferred.
func (m *Monitor) Done() int64 {
	m.mu.Lock()
	if now := m.update(0); m.sBytes > 0 {
		m.reset(now)
	}
	m.active = false
	m.tLast = 0
	n := m.bytes
	m.mu.Unlock()
	return n
}

// timeRemLimit is the maximum Status.TimeRem value.
const timeRemLimit = 999*time.Hour + 59*time.Minute + 59*time.Second

// Status represents the current Monitor status. All transfer rates are in bytes
// per second rounded to the nearest byte.
type Status struct {
	Active   bool          // Flag indicating an active transfer
	Start    time.Time     // Transfer start time
	Duration time.Duration // Time period covered by the statistics
	Idle     time.Duration // Time since the last transfer of at least 1 byte
	Bytes    int64         // Total number of bytes transferred
	Samples  int64         // Total number of samples taken
	InstRate int64         // Instantaneous transfer rate
	CurRate  int64         // Current transfer rate (EMA of InstRate)
	AvgRate  int64         // Average transfer rate (Bytes / Duration)
	PeakRate int64         // Maximum instantaneous transfer rate
	BytesRem int64         // Number of bytes remaining in the transfer
	TimeRem  time.Duration // Estimated time to completion
	Progress Percent       // Overall transfer progress
}

// Status returns current transfer status information. The returned value
// becomes static after a call to Done.
func (m *Monitor) Status() Status {
	m.mu.Lock()
	now := m.update(0)
	s := Status{
		Active:   m.active,
		Start:    clockToTime(m.start),
		Duration: m.sLast - m.start,
		Idle:     now - m.tLast,
		Bytes:    m.bytes,
		Samples:  m.samples,
		PeakRate: round(m.rPeak),
		BytesRem: m.tBytes - m.bytes,
		Progress: percentOf(float64(m.bytes), float64(m.tBytes)),
	}
	if s.BytesRem < 0 {
		s.BytesRem = 0
	}
	if s.Duration > 0 {
		rAvg := float64(s.Bytes) / s.Duration.Seconds()
		s.AvgRate = round(rAvg)
		if s.Active {
			s.InstRate = round(m.rSample)
			s.CurRate = round(m.rEMA)
			if s.BytesRem > 0 {
				if tRate := 0.8*m.rEMA + 0.2*rAvg; tRate > 0 {
					ns := float64(s.BytesRem) / tRate * 1e9
					if ns > float64(timeRemLimit) {
						ns = float64(timeRemLimit)
					}
					s.TimeRem = clockRound(time.Duration(ns))
				}
			}
		}
	}
	m.mu.Unlock()
	return s
}

// Limit restricts the instantaneous (per-sample) data flow to rate bytes per
// second. It returns the maximum number of bytes (0 <= n <= want) that may be
// transferred immediately without exceeding the limit. If block == true, the
// call blocks until n > 0. want is returned unmodified if want < 1, rate < 1,
// or the transfer is inactive (after a call to Done).
//
// At least one byte is always allowed to be transferred in any given sampling
// period. Thus, if the sampling rate is 100ms, the lowest achievable flow rate
// is 10 bytes per second.
//
// For usage examples, see the implementation of Reader and Writer in io.go.
func (m *Monitor) Limit(want int, rate int64, block bool) (n int) {
	if want < 1 || rate < 1 {
		return want
	}
	m.mu.Lock()

	// Determine the maximum number of bytes that can be sent in one sample
	limit := round(float64(rate) * m.sRate.Seconds())
	if limit <= 0 {
		limit = 1
	}

	// If block == true, wait until m.sBytes < limit
	if now := m.update(0); block {
		for m.sBytes >= limit && m.active {
			now = m.waitNextSample(now)
		}
	}

	// Make limit <= want (unlimited if the transfer is no longer active)
	if limit -= m.sBytes; limit > int64(want) || !m.active {
		limit = int64(want)
	}
	m.mu.Unlock()

	if limit < 0 {
		limit = 0
	}
	return int(limit)
}

// SetTransferSize specifies the total size of the data transfer, which allows
// the Monitor to calculate the overall progress and time to completion.
func (m *Monitor) SetTransferSize(bytes int64) {
	if bytes < 0 {
		bytes = 0
	}
	m.mu.Lock()
	m.tBytes = bytes
	m.mu.Unlock()
}

// update accumulates the transferred byte count for the current sample until
// clock() - m.sLast >= m.sRate. The monitor status is updated once the current
// sample is done.
func (m *Monitor) update(n int) (now time.Duration) {
	if !m.active {
		return
	}
	if now = clock(); n > 0 {
		m.tLast = now
	}
	m.sBytes += int64(n)
	if sTime := now - m.sLast; sTime >= m.sRate {
		t := sTime.Seconds()
		if m.rSample = float64(m.sBytes) / t; m.rSample > m.rPeak {
			m.rPeak = m.rSample
		}

		// Exponential moving average using a method similar to *nix load
		// average calculation. Longer sampling periods carry greater weight.
		if m.samples > 0 {
			w := math.Exp(-t / m.rWindow)
			m.rEMA = m.rSample + w*(m.rEMA-m.rSample)
		} else {
			m.rEMA = m.rSample
		}
		m.reset(now)
	}
	return
}

// reset clears the current sample state in preparation for the next sample.
func (m *Monitor) reset(sampleTime time.Duration) {
	m.bytes += m.sBytes
	m.samples++
	m.sBytes = 0
	m.sLast = sampleTime
}

// waitNextSample sleeps for the remainder of the current sample. The lock is
// released and reacquired during the actual sleep period, so it's possible for
// the transfer to be inactive when this method returns.
func (m *Monitor) waitNextSample(now time.Duration) time.Duration {
	const minWait = 5 * time.Millisecond
	current := m.sLast

	// sleep until the last sample time changes (ideally, just one iteration)
	for m.sLast == current && m.active {
		d := current + m.sRate - now
		m.mu.Unlock()
		if d < minWait {
			d = minWait
		}
		time.Sleep(d)
		m.mu.Lock()
		now = m.update(0)
	}
	return now
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html"
	"net/http"
	"net/url"
	"strconv"
	"time"
	"unicode/utf8"

	"github.com/gin-gonic/gin"
	"gopkg.in/flosch/pongo2.v3"

	grn "github.com/hnakamur/cgoroonga"
)

var ctx *grn.Ctx
var dbFilename string
var listenAddress string

func init() {
	flag.StringVar(&dbFilename, "d", "wikipedia_ja.db", "database filename")
	flag.StringVar(&listenAddress, "l", ":8080", "listen address (address:port)")
}

func formIntValue(c *gin.Context, key string, defaultValue int) (int, error) {
	strValue := c.Request.FormValue(key)
	intValue := defaultValue
	if strValue != "" {
		var err error
		intValue, err = strconv.Atoi(strValue)
		if err != nil {
			return defaultValue,
				fmt.Errorf("int parameter expected, but got \"%s\" for parameter \"%s\"",
					strValue, key)
		}
	}
	return intValue, nil
}

func formDateValue(c *gin.Context, key string, defaultValue time.Time) (time.Time, error) {
	strValue := c.Request.FormValue(key)
	timeValue := defaultValue
	if strValue != "" {
		var err error
		timeValue, err = time.Parse("2006-1-2", strValue)
		if err != nil {
			return defaultValue,
				fmt.Errorf("date parameter expected, but got \"%s\" for parameter \"%s\"",
					strValue, key)
		}
	}
	return timeValue, nil
}

type Result struct {
	URL       string
	Title     string
	Content   string
	UpdatedAt string
}

func titleToURL(title string) string {
	return fmt.Sprintf("https://ja.wikipedia.org/wiki/%s",
		url.QueryEscape(title))
}

func calcStartDate(timespan string) time.Time {
	var duration time.Duration
	switch timespan {
	case "week":
		duration = 7 * 24 * time.Hour
	case "month":
		duration = 31 * 24 * time.Hour
	case "year":
		duration = 365 * 24 * time.Hour
	default:
		return time.Unix(0, 0)
	}
	return time.Now().Add(-duration)
}

const viewablePageCount = 9

func getIndex(c *gin.Context) {
	c.Request.ParseForm()
	var query string
	queries, hasQueries := c.Request.Form["q"]
	timespan := c.Request.Form.Get("timespan")
	var limitCount int = 10
	var page int = 1
	var numPages int
	var matchedCount uint
	var err error
	viewablePages := []int{}
	results := []Result{}
	var cond *grn.Obj
	if hasQueries {
		query = queries[0]
		q := fmt.Sprintf("_key:@%s OR text:@%s", query, query)

		page, err = formIntValue(c, "page", 0)
		if err != nil {
			c.String(http.StatusBadRequest, err.Error())
			return
		}

		limitCount, err = formIntValue(c, "limit", 10)
		if err != nil {
			c.String(http.StatusBadRequest, err.Error())
			return
		}
		offset := (page - 1) * limitCount

		startDate := calcStartDate(timespan)

		table := ctx.Get("Articles")
		defer ctx.ObjUnlinkDefer(&err, table)

		var res *grn.Obj
		if q != "" || !startDate.IsZero() {
			var v *grn.Obj
			cond, v, err = ctx.ExprCreateForQuery(table)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			defer ctx.ObjUnlinkDefer(&err, cond)
			defer ctx.ObjUnlinkDefer(&err, v)

			if q != "" {
				flags := grn.EXPR_SYNTAX_QUERY | grn.EXPR_ALLOW_PRAGMA | grn.EXPR_ALLOW_COLUMN
				err = ctx.ExprParse(cond, q, nil, grn.OP_MATCH, grn.OP_AND, flags)
				if err != nil {
					c.String(http.StatusInternalServerError, err.Error())
					return
				}
			}
			if !startDate.IsZero() {
				usecStartTime := startDate.UnixNano() / 1000
				filter := "updated_at >= " + strconv.FormatInt(usecStartTime, 10)
				err = ctx.ExprParse(cond, filter, nil, grn.OP_MATCH, grn.OP_AND,
					grn.EXPR_SYNTAX_SCRIPT)
				if err != nil {
					c.String(http.StatusInternalServerError, err.Error())
					return
				}

				if q != "" {
					err = ctx.ExprAppendOp(cond, grn.OP_AND, 2)
					if err != nil {
						c.String(http.StatusInternalServerError, err.Error())
						return
					}
				}
			}

			res, err = ctx.TableSelect(table, cond, nil, grn.OP_OR)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			defer ctx.ObjUnlinkDefer(&err, res)
		} else {
			res = table
		}

		matchedCount, err = ctx.TableSize(res)
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		numPages = ((int(matchedCount) - 1) / limitCount) + 1
		if page > numPages {
			page = numPages
		}
		startPage := page - (viewablePageCount / 2)
		if startPage < 1 {
			startPage = 1
		}
		endPage := startPage + viewablePageCount
		if endPage > numPages+1 {
			endPage = numPages + 1
			if endPage-startPage < viewablePageCount {
				startPage = endPage - viewablePageCount
				if startPage < 1 {
					startPage = 1
				}
			}
		}
		for i := startPage; i < endPage; i++ {
			viewablePages = append(viewablePages, i)
		}

		var keyColumn *grn.Obj
		keyColumn, err = ctx.ObjColumn(res, "_key")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, keyColumn)

		var textColumn *grn.Obj
		textColumn, err = ctx.ObjColumn(res, "text")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, textColumn)

		var updatedAtColumn *grn.Obj
		updatedAtColumn, err = ctx.ObjColumn(res, "updated_at")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, updatedAtColumn)

		tc, err := ctx.TableCursorOpen(res, "", "", offset, limitCount, grn.CURSOR_ASCENDING)
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.TableCursorClose(tc)

		var buf grn.Obj
		defer ctx.ObjUnlinkDefer(&err, &buf)
		for {
			id := ctx.TableCursorNext(tc)
			if id == grn.ID_NIL {
				break
			}
			grn.TextInit(&buf, 0)
			grn.BulkRewind(&buf)
			ctx.ObjGetValue(keyColumn, id, &buf)
			title := grn.BulkHead(&buf)

			grn.TextInit(&buf, 0)
			grn.BulkRewind(&buf)
			ctx.ObjGetValue(textColumn, id, &buf)
			text := grn.BulkHead(&buf)

			flags := grn.SNIP_COPY_TAG | grn.SNIP_SKIP_LEADING_SPACES
			textManLen := 200
			snippet := ctx.ExprSnippet(cond, flags, textManLen, 1, true,
				[][]string{
					[]string{"", ""},
				})
			snipResults, err := ctx.SnipExec(snippet, text)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			if len(snipResults) > 0 {
				text = "..." + snipResults[0] + "..."

			} else {
				if utf8.RuneCountInString(text) >= textManLen {
					r := []rune(text)
					text = string(r[:textManLen]) + "..."
				}
				text = html.EscapeString(text)
			}

			grn.TimeInit(&buf, 0)
			ctx.ObjGetValue(updatedAtColumn, id, &buf)
			updatedAt := grn.TimeValue(&buf)

			url := titleToURL(title)

			results = append(results, Result{
				URL:       url,
				Title:     title,
				Content:   text,
				UpdatedAt: updatedAt.Format("2006/01/02"),
			})
		}
	}

	url_ := fmt.Sprintf("%s?q=%s&timespan=%s&limit=%d",
		c.Request.URL.Path, url.QueryEscape(query), timespan, limitCount)
	tpl, err := pongo2.FromFile("templates/index.html")
	if err != nil {
		c.String(500, "Internal Server Error")
	}
	err = tpl.ExecuteWriter(pongo2.Context{
		"url":           url_,
		"query":         query,
		"timespan":      timespan,
		"hasQueries":    hasQueries,
		"matchedCount":  matchedCount,
		"results":       results,
		"page":          page,
		"numPages":      numPages,
		"viewablePages": viewablePages,
	}, c.Writer)
	if err != nil {
		c.String(500, "Internal Server Error")
	}
}

func main() {
	flag.Parse()

	var err error
	err = grn.Init()
	if err != nil {
		panic(err)
	}
	defer grn.FinDefer(&err)

	ctx, err = grn.CtxOpen(0)
	if err != nil {
		panic(err)
	}
	defer ctx.CloseDefer(&err)

	var db *grn.Obj
	db, err = ctx.DBOpenOrCreate(dbFilename, nil)
	if err != nil {
		panic(err)
	}
	defer ctx.ObjUnlinkDefer(&err, db)

	r := gin.Default()
	r.Static("/static", "./static")
	r.GET("/", getIndex)
	r.Run(listenAddress)
}
<commit_msg>Set page to 1 when it is less than 1<commit_after>package main

import (
	"flag"
	"fmt"
	"html"
	"net/http"
	"net/url"
	"strconv"
	"time"
	"unicode/utf8"

	"github.com/gin-gonic/gin"
	"gopkg.in/flosch/pongo2.v3"

	grn "github.com/hnakamur/cgoroonga"
)

var ctx *grn.Ctx
var dbFilename string
var listenAddress string

func init() {
	flag.StringVar(&dbFilename, "d", "wikipedia_ja.db", "database filename")
	flag.StringVar(&listenAddress, "l", ":8080", "listen address (address:port)")
}

func formIntValue(c *gin.Context, key string, defaultValue int) (int, error) {
	strValue := c.Request.FormValue(key)
	intValue := defaultValue
	if strValue != "" {
		var err error
		intValue, err = strconv.Atoi(strValue)
		if err != nil {
			return defaultValue,
				fmt.Errorf("int parameter expected, but got \"%s\" for parameter \"%s\"",
					strValue, key)
		}
	}
	return intValue, nil
}

func formDateValue(c *gin.Context, key string, defaultValue time.Time) (time.Time, error) {
	strValue := c.Request.FormValue(key)
	timeValue := defaultValue
	if strValue != "" {
		var err error
		timeValue, err = time.Parse("2006-1-2", strValue)
		if err != nil {
			return defaultValue,
				fmt.Errorf("date parameter expected, but got \"%s\" for parameter \"%s\"",
					strValue, key)
		}
	}
	return timeValue, nil
}

type Result struct {
	URL       string
	Title     string
	Content   string
	UpdatedAt string
}

func titleToURL(title string) string {
	return fmt.Sprintf("https://ja.wikipedia.org/wiki/%s",
		url.QueryEscape(title))
}

func calcStartDate(timespan string) time.Time {
	var duration time.Duration
	switch timespan {
	case "week":
		duration = 7 * 24 * time.Hour
	case "month":
		duration = 31 * 24 * time.Hour
	case "year":
		duration = 365 * 24 * time.Hour
	default:
		return time.Unix(0, 0)
	}
	return time.Now().Add(-duration)
}

const viewablePageCount = 9

func getIndex(c *gin.Context) {
	c.Request.ParseForm()
	var query string
	queries, hasQueries := c.Request.Form["q"]
	timespan := c.Request.Form.Get("timespan")
	var limitCount int = 10
	var page int = 1
	var numPages int
	var matchedCount uint
	var err error
	viewablePages := []int{}
	results := []Result{}
	var cond *grn.Obj
	if hasQueries {
		query = queries[0]
		q := fmt.Sprintf("_key:@%s OR text:@%s", query, query)

		page, err = formIntValue(c, "page", 1)
		if err != nil {
			c.String(http.StatusBadRequest, err.Error())
			return
		}
		if page < 1 {
			page = 1
		}

		limitCount, err = formIntValue(c, "limit", 10)
		if err != nil {
			c.String(http.StatusBadRequest, err.Error())
			return
		}
		offset := (page - 1) * limitCount

		startDate := calcStartDate(timespan)

		table := ctx.Get("Articles")
		defer ctx.ObjUnlinkDefer(&err, table)

		var res *grn.Obj
		if q != "" || !startDate.IsZero() {
			var v *grn.Obj
			cond, v, err = ctx.ExprCreateForQuery(table)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			defer ctx.ObjUnlinkDefer(&err, cond)
			defer ctx.ObjUnlinkDefer(&err, v)

			if q != "" {
				flags := grn.EXPR_SYNTAX_QUERY | grn.EXPR_ALLOW_PRAGMA | grn.EXPR_ALLOW_COLUMN
				err = ctx.ExprParse(cond, q, nil, grn.OP_MATCH, grn.OP_AND, flags)
				if err != nil {
					c.String(http.StatusInternalServerError, err.Error())
					return
				}
			}
			if !startDate.IsZero() {
				usecStartTime := startDate.UnixNano() / 1000
				filter := "updated_at >= " + strconv.FormatInt(usecStartTime, 10)
				err = ctx.ExprParse(cond, filter, nil, grn.OP_MATCH, grn.OP_AND,
					grn.EXPR_SYNTAX_SCRIPT)
				if err != nil {
					c.String(http.StatusInternalServerError, err.Error())
					return
				}

				if q != "" {
					err = ctx.ExprAppendOp(cond, grn.OP_AND, 2)
					if err != nil {
						c.String(http.StatusInternalServerError, err.Error())
						return
					}
				}
			}

			res, err = ctx.TableSelect(table, cond, nil, grn.OP_OR)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			defer ctx.ObjUnlinkDefer(&err, res)
		} else {
			res = table
		}

		matchedCount, err = ctx.TableSize(res)
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		numPages = ((int(matchedCount) - 1) / limitCount) + 1
		if page > numPages {
			page = numPages
		}
		startPage := page - (viewablePageCount / 2)
		if startPage < 1 {
			startPage = 1
		}
		endPage := startPage + viewablePageCount
		if endPage > numPages+1 {
			endPage = numPages + 1
			if endPage-startPage < viewablePageCount {
				startPage = endPage - viewablePageCount
				if startPage < 1 {
					startPage = 1
				}
			}
		}
		for i := startPage; i < endPage; i++ {
			viewablePages = append(viewablePages, i)
		}

		var keyColumn *grn.Obj
		keyColumn, err = ctx.ObjColumn(res, "_key")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, keyColumn)

		var textColumn *grn.Obj
		textColumn, err = ctx.ObjColumn(res, "text")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, textColumn)

		var updatedAtColumn *grn.Obj
		updatedAtColumn, err = ctx.ObjColumn(res, "updated_at")
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.ObjUnlinkDefer(&err, updatedAtColumn)

		tc, err := ctx.TableCursorOpen(res, "", "", offset, limitCount, grn.CURSOR_ASCENDING)
		if err != nil {
			c.String(http.StatusInternalServerError, err.Error())
			return
		}
		defer ctx.TableCursorClose(tc)

		var buf grn.Obj
		defer ctx.ObjUnlinkDefer(&err, &buf)
		for {
			id := ctx.TableCursorNext(tc)
			if id == grn.ID_NIL {
				break
			}
			grn.TextInit(&buf, 0)
			grn.BulkRewind(&buf)
			ctx.ObjGetValue(keyColumn, id, &buf)
			title := grn.BulkHead(&buf)

			grn.TextInit(&buf, 0)
			grn.BulkRewind(&buf)
			ctx.ObjGetValue(textColumn, id, &buf)
			text := grn.BulkHead(&buf)

			flags := grn.SNIP_COPY_TAG | grn.SNIP_SKIP_LEADING_SPACES
			textManLen := 200
			snippet := ctx.ExprSnippet(cond, flags, textManLen, 1, true,
				[][]string{
					[]string{"", ""},
				})
			snipResults, err := ctx.SnipExec(snippet, text)
			if err != nil {
				c.String(http.StatusInternalServerError, err.Error())
				return
			}
			if len(snipResults) > 0 {
				text = "..." + snipResults[0] + "..."

			} else {
				if utf8.RuneCountInString(text) >= textManLen {
					r := []rune(text)
					text = string(r[:textManLen]) + "..."
				}
				text = html.EscapeString(text)
			}

			grn.TimeInit(&buf, 0)
			ctx.ObjGetValue(updatedAtColumn, id, &buf)
			updatedAt := grn.TimeValue(&buf)

			url := titleToURL(title)

			results = append(results, Result{
				URL:       url,
				Title:     title,
				Content:   text,
				UpdatedAt: updatedAt.Format("2006/01/02"),
			})
		}
	}

	url_ := fmt.Sprintf("%s?q=%s&timespan=%s&limit=%d",
		c.Request.URL.Path, url.QueryEscape(query), timespan, limitCount)
	tpl, err := pongo2.FromFile("templates/index.html")
	if err != nil {
		c.String(500, "Internal Server Error")
	}
	err = tpl.ExecuteWriter(pongo2.Context{
		"url":           url_,
		"query":         query,
		"timespan":      timespan,
		"hasQueries":    hasQueries,
		"matchedCount":  matchedCount,
		"results":       results,
		"page":          page,
		"numPages":      numPages,
		"viewablePages": viewablePages,
	}, c.Writer)
	if err != nil {
		c.String(500, "Internal Server Error")
	}
}

func main() {
	flag.Parse()

	var err error
	err = grn.Init()
	if err != nil {
		panic(err)
	}
	defer grn.FinDefer(&err)

	ctx, err = grn.CtxOpen(0)
	if err != nil {
		panic(err)
	}
	defer ctx.CloseDefer(&err)

	var db *grn.Obj
	db, err = ctx.DBOpenOrCreate(dbFilename, nil)
	if err != nil {
		panic(err)
	}
	defer ctx.ObjUnlinkDefer(&err, db)

	r := gin.Default()
	r.Static("/static", "./static")
	r.GET("/", getIndex)
	r.Run(listenAddress)
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	//"github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	. "github.com/RyanCi/hdc/hive"
	// "reflect"
	//"log"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type students struct {
	name    string
	age     int
	phone   string
	address string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
}

/* Populate will exec query and immidiately return the value into object
Populate is suitable for short type query that return limited data,
Exec is suitable for long type query that return massive amount of data and require time to produce it

Ideally Populate should call Exec as well but already have predefined function on it receiving process
*/
// func TestHivePopulate(t *testing.T) {
// 	q := "select * from sample_07 limit 5;"

// 	var result []toolkit.M

// 	h.Conn.Open()

// 	e := h.Populate(q, &result)
// 	fatalCheck(t, "Populate", e)

// 	if len(result) != 5 {
// 		log.Printf("Error want %d got %d", 5, len(result))
// 	}

// 	log.Printf("Result: \n%s", toolkit.JsonString(result))

// 	h.Conn.Close()
// }

<<<<<<< HEAD
// func TestLoad(t *testing.T) {
// 	h.Conn.Open()

// 	var student students

// 	retVal, err := h.Load("students", "|", &student)

// 	if err != nil {
// 		fmt.Println(err)
// 	}
// 	h.Conn.Close()
// 	fmt.Println(retVal)
// }

func TestLoadFile(t *testing.T) {
	h.Conn.Open()

	var student students

	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "txt", &student)

	if err != nil {
		fmt.Println(err)
	}
	h.Conn.Close()
	fmt.Println(retVal)
}

// func TestExecLine(t *testing.T) {
// 	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
// 	//=======
// 	//func TestHiveExec(t *testing.T) {
// 	//>>>>>>> ccf5cbc4dbf71b6702fa10127569b327c853df4a
// 	q := "select * from sample_07 limit 5;"
// 	// x := "select * from sample_07 limit 10;"
// 	DoSomething := func(res HiveResult) (e error) {
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 5: %v", res.ResultObj)
// 		return
// 	}

// 	DoElse := func(res HiveResult) (e error) {
// 		tmp := toolkit.M{}
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 10: %v", tmp)
// 		return
// 	}

// 	// h.Conn.SetFn(DoSomething)
// 	h.Conn.FnReceive = DoSomething
=======
func TestHiveExec(t *testing.T) {
	q := "select * from sample_07 limit 1;"
	// x := "select * from sample_07 limit 3;"

	DoSomething := func(res HiveResult) (e error) {
		toolkit.Serde(res, &res.ResultObj, "json")
		log.Printf("limit 1: \n%v\n", res.ResultObj)
		return
	}

	/*DoElse := func(res HiveResult) (e error) {
		tmp := toolkit.M{}
		toolkit.Serde(res, &res.ResultObj, "json")
		log.Printf("limit 3: \n%v\n", tmp)
		return
	}*/

	h.Conn.Open()

	h.Conn.FnReceive = DoSomething
	h.Exec(q)
	/*h.Conn.Wait()

	h.Conn.FnReceive = DoElse
	h.Exec(x)
	h.Conn.Wait()
>>>>>>> 523f6ec530b5f735623c4bfa5fee50e46b10dc21

// 	h.Conn.Open()
// 	h.Exec(q)

<<<<<<< HEAD
// 	/*h.Conn.FnReceive = DoElse
// 	h.Exec(x)*/

// 	h.Conn.Close()

// 	/*h.Conn.Exec = true
// 	h.Conn.Open()
// 	h.Conn.FnReceive = DoSomething
// 	h.Exec(q)

// 	h.Conn.FnReceive = DoElse
// 	h.Exec(x)

// 	h.Conn.Exec = false

// 	var res []toolkit.M

// 	e := h.Populate(q, &res)
// 	log.Printf("res: %v\n", res)
// 	log.Printf("e: %v\n", e)

// 	h.Conn.Close()*/
// }
=======
	e := h.Populate(q, &res)
	log.Printf("populate res: \n%v\n", res)
	log.Printf("populate e: \n%v\n", e)*/

	h.Conn.Close()
}

func TestHiveExecMulti(t *testing.T) {
	var ms1 []HiveResult
	q := "select * from sample_07 limit 5;"

	DoSomething := func(res HiveResult) (e error) {
		ms1 = append(ms1, res)
		return
	}

	h.Conn.Open()

	h.Conn.FnReceive = DoSomething
	h.Exec(q)
	h.Exec(q)

	for _, v1 := range ms1 {
		log.Println(v1)
	}
}

/*func TestHiveClose(t *testing.T) {
	if h != nil {
		h.Conn.Close()
	}
}*/
>>>>>>> 523f6ec530b5f735623c4bfa5fee50e46b10dc21
<commit_msg>update<commit_after>package test

import (
	"fmt"
	//"github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	. "github.com/RyanCi/hdc/hive"
	// "reflect"
	//"log"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type students struct {
	name    string
	age     int
	phone   string
	address string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
}

/* Populate will exec query and immidiately return the value into object
Populate is suitable for short type query that return limited data,
Exec is suitable for long type query that return massive amount of data and require time to produce it

Ideally Populate should call Exec as well but already have predefined function on it receiving process
*/
// func TestHivePopulate(t *testing.T) {
// 	q := "select * from sample_07 limit 5;"

// 	var result []toolkit.M

// 	h.Conn.Open()

// 	e := h.Populate(q, &result)
// 	fatalCheck(t, "Populate", e)

// 	if len(result) != 5 {
// 		log.Printf("Error want %d got %d", 5, len(result))
// 	}

// 	log.Printf("Result: \n%s", toolkit.JsonString(result))

// 	h.Conn.Close()
// }

// func TestLoad(t *testing.T) {
// 	h.Conn.Open()

// 	var student students

// 	retVal, err := h.Load("students", "|", &student)

// 	if err != nil {
// 		fmt.Println(err)
// 	}
// 	h.Conn.Close()
// 	fmt.Println(retVal)
// }

func TestLoadFile(t *testing.T) {
	h.Conn.Open()

	var student students

	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "txt", &student)

	if err != nil {
		fmt.Println(err)
	}
	h.Conn.Close()
	fmt.Println(retVal)
}

// func TestExecLine(t *testing.T) {
// 	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")

// 	//func TestHiveExec(t *testing.T) {

// 	q := "select * from sample_07 limit 5;"
// 	// x := "select * from sample_07 limit 10;"
// 	DoSomething := func(res HiveResult) (e error) {
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 5: %v", res.ResultObj)
// 		return
// 	}

// 	DoElse := func(res HiveResult) (e error) {
// 		tmp := toolkit.M{}
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 10: %v", tmp)
// 		return
// 	}

// 	// h.Conn.SetFn(DoSomething)
// 	h.Conn.FnReceive = DoSomething

// func TestHiveExec(t *testing.T) {
// 	q := "select * from sample_07 limit 1;"
// 	// x := "select * from sample_07 limit 3;"

// 	DoSomething := func(res HiveResult) (e error) {
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 1: \n%v\n", res.ResultObj)
// 		return
// 	}

// 	/*DoElse := func(res HiveResult) (e error) {
// 		tmp := toolkit.M{}
// 		toolkit.Serde(res, &res.ResultObj, "json")
// 		log.Printf("limit 3: \n%v\n", tmp)
// 		return
// 	}*/

// 	h.Conn.Open()

// 	h.Conn.FnReceive = DoSomething
// 	h.Exec(q)
// 	/*h.Conn.Wait()

// 	h.Conn.FnReceive = DoElse
// 	h.Exec(x)
// 	h.Conn.Wait()

// // 	h.Conn.Open()
// // 	h.Exec(q)

// // 	/*h.Conn.FnReceive = DoElse
// // 	h.Exec(x)*/

// // 	h.Conn.Close()

// // 	/*h.Conn.Exec = true
// // 	h.Conn.Open()
// // 	h.Conn.FnReceive = DoSomething
// // 	h.Exec(q)

// // 	h.Conn.FnReceive = DoElse
// // 	h.Exec(x)

// // 	h.Conn.Exec = false

// // 	var res []toolkit.M

// // 	e := h.Populate(q, &res)
// // 	log.Printf("res: %v\n", res)
// // 	log.Printf("e: %v\n", e)

// // 	h.Conn.Close()*/
// // }

// 	e := h.Populate(q, &res)
// 	log.Printf("populate res: \n%v\n", res)
// 	log.Printf("populate e: \n%v\n", e)*/

// 	h.Conn.Close()
// }

// func TestHiveExecMulti(t *testing.T) {
// 	var ms1 []HiveResult
// 	q := "select * from sample_07 limit 5;"

// 	DoSomething := func(res HiveResult) (e error) {
// 		ms1 = append(ms1, res)
// 		return
// 	}

// 	h.Conn.Open()

// 	h.Conn.FnReceive = DoSomething
// 	h.Exec(q)
// 	h.Exec(q)

// 	for _, v1 := range ms1 {
// 		log.Println(v1)
// 	}
// }

/*func TestHiveClose(t *testing.T) {
	if h != nil {
		h.Conn.Close()
	}
}*/
<|endoftext|>


<commit_before>/*
Copyright (c) 2020 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"context"
	"errors"
	"flag"

	"github.com/vmware/govmomi/cns"
	"github.com/vmware/govmomi/cns/types"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/soap"
)

type rm struct {
	*flags.ClientFlag
}

func init() {
	cli.Register("volume.rm", &rm{})
}

func (cmd *rm) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.ClientFlag, ctx = flags.NewClientFlag(ctx)
	cmd.ClientFlag.Register(ctx, f)
}

func (cmd *rm) Usage() string {
	return "ID"
}

func (cmd *rm) Description() string {
	return `Remove CNS volume.

Examples:
  govc volume.rm f75989dc-95b9-4db7-af96-8583f24bc59d
  govc volume.rm $(govc volume.ls -i pvc-de368f19-a997-4d5d-9eae-4496f10f429a)`
}

func (cmd *rm) Run(ctx context.Context, f *flag.FlagSet) error {
	vc, err := cmd.Client()
	if err != nil {
		return err
	}
	_ = vc.UseServiceVersion("vsan")

	c, err := cns.NewClient(ctx, vc)
	if err != nil {
		return err
	}

	ids := []types.CnsVolumeId{{Id: f.Arg(0)}}

	task, err := c.DeleteVolume(ctx, ids, true)
	if err != nil {
		return err
	}

	info, err := task.WaitForResult(ctx, nil)
	if err != nil {
		return err
	}

	if res, ok := info.Result.(types.CnsVolumeOperationBatchResult); ok {
		for _, r := range res.VolumeResults {
			fault := r.GetCnsVolumeOperationResult().Fault

			if fault != nil {
				if fault.Fault != nil {
					return soap.WrapVimFault(fault.Fault)
				}
				return errors.New(fault.LocalizedMessage)
			}
		}
	}

	return nil
}
<commit_msg>govc: note 'disk.ls -R' in volume.rm help<commit_after>/*
Copyright (c) 2020 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"context"
	"errors"
	"flag"

	"github.com/vmware/govmomi/cns"
	"github.com/vmware/govmomi/cns/types"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/soap"
)

type rm struct {
	*flags.ClientFlag
}

func init() {
	cli.Register("volume.rm", &rm{})
}

func (cmd *rm) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.ClientFlag, ctx = flags.NewClientFlag(ctx)
	cmd.ClientFlag.Register(ctx, f)
}

func (cmd *rm) Usage() string {
	return "ID"
}

func (cmd *rm) Description() string {
	return `Remove CNS volume.

Note: if volume.rm returns not found errors,
consider using 'govc disk.ls -R' to reconcile the datastore inventory.

Examples:
  govc volume.rm f75989dc-95b9-4db7-af96-8583f24bc59d`
}

func (cmd *rm) Run(ctx context.Context, f *flag.FlagSet) error {
	vc, err := cmd.Client()
	if err != nil {
		return err
	}
	_ = vc.UseServiceVersion("vsan")

	c, err := cns.NewClient(ctx, vc)
	if err != nil {
		return err
	}

	ids := []types.CnsVolumeId{{Id: f.Arg(0)}}

	task, err := c.DeleteVolume(ctx, ids, true)
	if err != nil {
		return err
	}

	info, err := task.WaitForResult(ctx, nil)
	if err != nil {
		return err
	}

	if res, ok := info.Result.(types.CnsVolumeOperationBatchResult); ok {
		for _, r := range res.VolumeResults {
			fault := r.GetCnsVolumeOperationResult().Fault

			if fault != nil {
				if fault.Fault != nil {
					return soap.WrapVimFault(fault.Fault)
				}
				return errors.New(fault.LocalizedMessage)
			}
		}
	}

	return nil
}
<|endoftext|>