<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"github.com/aliyun/alibaba-cloud-sdk-go/services/alidns"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/pvtz"
	"github.com/kubernetes-incubator/external-dns/endpoint"
	"testing"

	"github.com/kubernetes-incubator/external-dns/plan"
)

type mockAlibabaCloudDNSAPI struct {
	records []alidns.Record
}

func NewMockAlibabaCloudDNSAPI() *mockAlibabaCloudDNSAPI {
	api := mockAlibabaCloudDNSAPI{}
	api.records = []alidns.Record{
		alidns.Record{
			RecordId:   "1",
			DomainName: "container-service.top",
			Type:       "A",
			TTL:        300,
			RR:         "abc",
			Value:      "1.2.3.4",
		},
		alidns.Record{
			RecordId:   "2",
			DomainName: "container-service.top",
			Type:       "TXT",
			TTL:        300,
			RR:         "abc",
			Value:      "heritage=external-dns;external-dns/owner=default",
		},
	}
	return &api
}

func (m *mockAlibabaCloudDNSAPI) AddDomainRecord(request *alidns.AddDomainRecordRequest) (response *alidns.AddDomainRecordResponse, err error) {
	ttl, _ := request.TTL.GetValue()
	m.records = append(m.records, alidns.Record{
		RecordId:   "3",
		DomainName: request.DomainName,
		Type:       request.Type,
		TTL:        ttl,
		RR:         request.RR,
		Value:      request.Value,
	})
	response = alidns.CreateAddDomainRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) DeleteDomainRecord(request *alidns.DeleteDomainRecordRequest) (response *alidns.DeleteDomainRecordResponse, err error) {
	var result []alidns.Record
	for _, record := range m.records {
		if record.RecordId != request.RecordId {
			result = append(result, record)
		}
	}
	m.records = result
	response = alidns.CreateDeleteDomainRecordResponse()
	response.RecordId = request.RecordId
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) UpdateDomainRecord(request *alidns.UpdateDomainRecordRequest) (response *alidns.UpdateDomainRecordResponse, err error) {
	ttl, _ := request.TTL.GetValue()
	for i, _ := range m.records {
		if m.records[i].RecordId == request.RecordId {
			m.records[i].TTL = ttl
		}
	}
	response = alidns.CreateUpdateDomainRecordResponse()
	response.RecordId = request.RecordId
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) DescribeDomainRecords(request *alidns.DescribeDomainRecordsRequest) (response *alidns.DescribeDomainRecordsResponse, err error) {
	var result []alidns.Record
	for _, record := range m.records {
		if record.DomainName == request.DomainName {
			result = append(result, record)
		}
	}
	response = alidns.CreateDescribeDomainRecordsResponse()
	response.DomainRecords.Record = result
	return response, nil
}

type mockAlibabaCloudPrivateZoneAPI struct {
	zone    pvtz.Zone
	records []pvtz.Record
}

func NewMockAlibabaCloudPrivateZoneAPI() *mockAlibabaCloudPrivateZoneAPI {
	api := mockAlibabaCloudPrivateZoneAPI{}
	api.zone = pvtz.Zone{
		ZoneId:   "test-zone",
		ZoneName: "container-service.top",
	}
	api.records = []pvtz.Record{
		pvtz.Record{
			RecordId: 1,
			Type:     "A",
			Ttl:      300,
			Rr:       "abc",
			Value:    "1.2.3.4",
		},
		pvtz.Record{
			RecordId: 2,
			Type:     "TXT",
			Ttl:      300,
			Rr:       "abc",
			Value:    "heritage=external-dns;external-dns/owner=default",
		},
	}
	return &api
}

func (m *mockAlibabaCloudPrivateZoneAPI) AddZoneRecord(request *pvtz.AddZoneRecordRequest) (response *pvtz.AddZoneRecordResponse, err error) {
	ttl, _ := request.Ttl.GetValue()
	m.records = append(m.records, pvtz.Record{
		RecordId: 3,
		Type:     request.Type,
		Ttl:      ttl,
		Rr:       request.Rr,
		Value:    request.Value,
	})
	response = pvtz.CreateAddZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DeleteZoneRecord(request *pvtz.DeleteZoneRecordRequest) (response *pvtz.DeleteZoneRecordResponse, err error) {
	recordId, _ := request.RecordId.GetValue()

	var result []pvtz.Record
	for _, record := range m.records {
		if record.RecordId != recordId {
			result = append(result, record)
		}
	}
	response = pvtz.CreateDeleteZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) UpdateZoneRecord(request *pvtz.UpdateZoneRecordRequest) (response *pvtz.UpdateZoneRecordResponse, err error) {
	recordId, _ := request.RecordId.GetValue()
	ttl, _ := request.Ttl.GetValue()
	for i, _ := range m.records {
		if m.records[i].RecordId == recordId {
			m.records[i].Ttl = ttl
		}
	}
	response = pvtz.CreateUpdateZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZoneRecords(request *pvtz.DescribeZoneRecordsRequest) (response *pvtz.DescribeZoneRecordsResponse, err error) {
	response = pvtz.CreateDescribeZoneRecordsResponse()
	response.Records.Record = append(response.Records.Record, m.records...)
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZones(request *pvtz.DescribeZonesRequest) (response *pvtz.DescribeZonesResponse, err error) {
	response = pvtz.CreateDescribeZonesResponse()
	response.Zones.Zone = append(response.Zones.Zone, m.zone)
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZoneInfo(request *pvtz.DescribeZoneInfoRequest) (response *pvtz.DescribeZoneInfoResponse, err error) {
	response = pvtz.CreateDescribeZoneInfoResponse()
	response.ZoneId = m.zone.ZoneId
	response.ZoneName = m.zone.ZoneName
	return response, nil
}

func newTestAlibabaCloudProvider(private bool) *AlibabaCloudProvider {
	cfg := alibabaCloudConfig{
		RegionID:        "cn-beijing",
		AccessKeyID:     "xxxxxx",
		AccessKeySecret: "xxxxxx",
		VPCID:           "vpc-xxxxxx",
	}
	//
	//dnsClient, _ := alidns.NewClientWithAccessKey(
	//	cfg.RegionID,
	//	cfg.AccessKeyID,
	//	cfg.AccessKeySecret,
	//)
	//
	//pvtzClient, _ := pvtz.NewClientWithAccessKey(
	//	"cn-hangzhou",
	//	cfg.AccessKeyID,
	//	cfg.AccessKeySecret,
	//)
	domainFilterTest := NewDomainFilter([]string{"container-service.top.", "example.org"})

	return &AlibabaCloudProvider{
		domainFilter: domainFilterTest,
		vpcID:        cfg.VPCID,
		dryRun:       false,
		dnsClient:    NewMockAlibabaCloudDNSAPI(),
		pvtzClient:   &mockAlibabaCloudPrivateZoneAPI{},
		privateZone:  false,
	}
}

func TestAlibabaCloudProvider_Records(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_ApplyChanges(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	changes := plan.Changes{
		Create: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "xyz.container-service.top",
				RecordType: "A",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("4.3.2.1"),
			},
		},
		UpdateNew: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "A",
				RecordTTL:  500,
				Targets:    endpoint.NewTargets("1.2.3.4", "5.6.7.8"),
			},
		},
		Delete: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "TXT",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("\"heritage=external-dns,external-dns/owner=default\""),
			},
		},
	}
	p.ApplyChanges(&changes)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_Records_PrivateZone(t *testing.T) {
	p := newTestAlibabaCloudProvider(true)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_ApplyChanges_PrivateZone(t *testing.T) {
	p := newTestAlibabaCloudProvider(true)
	changes := plan.Changes{
		Create: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "xyz.container-service.top",
				RecordType: "A",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("4.3.2.1"),
			},
		},
		UpdateNew: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "A",
				RecordTTL:  500,
				Targets:    endpoint.NewTargets("1.2.3.4", "5.6.7.8"),
			},
		},
		Delete: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "TXT",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("\"heritage=external-dns,external-dns/owner=default\""),
			},
		},
	}
	p.ApplyChanges(&changes)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_splitDNSName(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	endpoint := &endpoint.Endpoint{}
	endpoint.DNSName = "www.example.org"
	rr, domain := p.splitDNSName(endpoint)
	if rr != "www" || domain != "example.org" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = ".example.org"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "example.org" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "www"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "www" || domain != "" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = ""
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "_30000._tcp.container-service.top"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "_30000._tcp" || domain != "container-service.top" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "container-service.top"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "container-service.top" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
}

func TestAlibabaCloudProvider_TXTEndpoint(t *testing.T) {

	p := newTestAlibabaCloudProvider(false)
	const recordValue = "heritage=external-dns;external-dns/owner=default"
	const endpointTarget = "\"heritage=external-dns,external-dns/owner=default\""

	if p.normalizedTXTRecordValue(endpointTarget) != recordValue {
		t.Errorf("Failed to normalizedTXTRecordValue: %s", p.normalizedTXTRecordValue(endpointTarget))
	}
	if p.unescapeTXTRecordValue(recordValue) != endpointTarget {
		t.Errorf("Failed to unescapeTXTRecordValue: %s", p.unescapeTXTRecordValue(recordValue))
	}
}
<commit_msg>Fix test case<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"github.com/aliyun/alibaba-cloud-sdk-go/services/alidns"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/pvtz"
	"github.com/kubernetes-incubator/external-dns/endpoint"
	"testing"

	"github.com/kubernetes-incubator/external-dns/plan"
)

type mockAlibabaCloudDNSAPI struct {
	records []alidns.Record
}

func NewMockAlibabaCloudDNSAPI() *mockAlibabaCloudDNSAPI {
	api := mockAlibabaCloudDNSAPI{}
	api.records = []alidns.Record{
		alidns.Record{
			RecordId:   "1",
			DomainName: "container-service.top",
			Type:       "A",
			TTL:        300,
			RR:         "abc",
			Value:      "1.2.3.4",
		},
		alidns.Record{
			RecordId:   "2",
			DomainName: "container-service.top",
			Type:       "TXT",
			TTL:        300,
			RR:         "abc",
			Value:      "heritage=external-dns;external-dns/owner=default",
		},
	}
	return &api
}

func (m *mockAlibabaCloudDNSAPI) AddDomainRecord(request *alidns.AddDomainRecordRequest) (response *alidns.AddDomainRecordResponse, err error) {
	ttl, _ := request.TTL.GetValue()
	m.records = append(m.records, alidns.Record{
		RecordId:   "3",
		DomainName: request.DomainName,
		Type:       request.Type,
		TTL:        ttl,
		RR:         request.RR,
		Value:      request.Value,
	})
	response = alidns.CreateAddDomainRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) DeleteDomainRecord(request *alidns.DeleteDomainRecordRequest) (response *alidns.DeleteDomainRecordResponse, err error) {
	var result []alidns.Record
	for _, record := range m.records {
		if record.RecordId != request.RecordId {
			result = append(result, record)
		}
	}
	m.records = result
	response = alidns.CreateDeleteDomainRecordResponse()
	response.RecordId = request.RecordId
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) UpdateDomainRecord(request *alidns.UpdateDomainRecordRequest) (response *alidns.UpdateDomainRecordResponse, err error) {
	ttl, _ := request.TTL.GetValue()
	for i, _ := range m.records {
		if m.records[i].RecordId == request.RecordId {
			m.records[i].TTL = ttl
		}
	}
	response = alidns.CreateUpdateDomainRecordResponse()
	response.RecordId = request.RecordId
	return response, nil
}

func (m *mockAlibabaCloudDNSAPI) DescribeDomainRecords(request *alidns.DescribeDomainRecordsRequest) (response *alidns.DescribeDomainRecordsResponse, err error) {
	var result []alidns.Record
	for _, record := range m.records {
		if record.DomainName == request.DomainName {
			result = append(result, record)
		}
	}
	response = alidns.CreateDescribeDomainRecordsResponse()
	response.DomainRecords.Record = result
	return response, nil
}

type mockAlibabaCloudPrivateZoneAPI struct {
	zone    pvtz.Zone
	records []pvtz.Record
}

func NewMockAlibabaCloudPrivateZoneAPI() *mockAlibabaCloudPrivateZoneAPI {
	api := mockAlibabaCloudPrivateZoneAPI{}
	api.zone = pvtz.Zone{
		ZoneId:   "test-zone",
		ZoneName: "container-service.top",
	}
	api.records = []pvtz.Record{
		pvtz.Record{
			RecordId: 1,
			Type:     "A",
			Ttl:      300,
			Rr:       "abc",
			Value:    "1.2.3.4",
		},
		pvtz.Record{
			RecordId: 2,
			Type:     "TXT",
			Ttl:      300,
			Rr:       "abc",
			Value:    "heritage=external-dns;external-dns/owner=default",
		},
	}
	return &api
}

func (m *mockAlibabaCloudPrivateZoneAPI) AddZoneRecord(request *pvtz.AddZoneRecordRequest) (response *pvtz.AddZoneRecordResponse, err error) {
	ttl, _ := request.Ttl.GetValue()
	m.records = append(m.records, pvtz.Record{
		RecordId: 3,
		Type:     request.Type,
		Ttl:      ttl,
		Rr:       request.Rr,
		Value:    request.Value,
	})
	response = pvtz.CreateAddZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DeleteZoneRecord(request *pvtz.DeleteZoneRecordRequest) (response *pvtz.DeleteZoneRecordResponse, err error) {
	recordId, _ := request.RecordId.GetValue()

	var result []pvtz.Record
	for _, record := range m.records {
		if record.RecordId != recordId {
			result = append(result, record)
		}
	}
	response = pvtz.CreateDeleteZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) UpdateZoneRecord(request *pvtz.UpdateZoneRecordRequest) (response *pvtz.UpdateZoneRecordResponse, err error) {
	recordId, _ := request.RecordId.GetValue()
	ttl, _ := request.Ttl.GetValue()
	for i, _ := range m.records {
		if m.records[i].RecordId == recordId {
			m.records[i].Ttl = ttl
		}
	}
	response = pvtz.CreateUpdateZoneRecordResponse()
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZoneRecords(request *pvtz.DescribeZoneRecordsRequest) (response *pvtz.DescribeZoneRecordsResponse, err error) {
	response = pvtz.CreateDescribeZoneRecordsResponse()
	response.Records.Record = append(response.Records.Record, m.records...)
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZones(request *pvtz.DescribeZonesRequest) (response *pvtz.DescribeZonesResponse, err error) {
	response = pvtz.CreateDescribeZonesResponse()
	response.Zones.Zone = append(response.Zones.Zone, m.zone)
	return response, nil
}

func (m *mockAlibabaCloudPrivateZoneAPI) DescribeZoneInfo(request *pvtz.DescribeZoneInfoRequest) (response *pvtz.DescribeZoneInfoResponse, err error) {
	response = pvtz.CreateDescribeZoneInfoResponse()
	response.ZoneId = m.zone.ZoneId
	response.ZoneName = m.zone.ZoneName
	return response, nil
}

func newTestAlibabaCloudProvider(private bool) *AlibabaCloudProvider {
	cfg := alibabaCloudConfig{
		RegionID:        "cn-beijing",
		AccessKeyID:     "xxxxxx",
		AccessKeySecret: "xxxxxx",
		VPCID:           "vpc-xxxxxx",
	}
	//
	//dnsClient, _ := alidns.NewClientWithAccessKey(
	//	cfg.RegionID,
	//	cfg.AccessKeyID,
	//	cfg.AccessKeySecret,
	//)
	//
	//pvtzClient, _ := pvtz.NewClientWithAccessKey(
	//	"cn-hangzhou",
	//	cfg.AccessKeyID,
	//	cfg.AccessKeySecret,
	//)
	domainFilterTest := NewDomainFilter([]string{"container-service.top.", "example.org"})

	return &AlibabaCloudProvider{
		domainFilter: domainFilterTest,
		vpcID:        cfg.VPCID,
		dryRun:       false,
		dnsClient:    NewMockAlibabaCloudDNSAPI(),
		pvtzClient:   &mockAlibabaCloudPrivateZoneAPI{},
		privateZone:  false,
	}
}

func TestAlibabaCloudProvider_Records(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_ApplyChanges(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	changes := plan.Changes{
		Create: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "xyz.container-service.top",
				RecordType: "A",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("4.3.2.1"),
			},
		},
		UpdateNew: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "A",
				RecordTTL:  500,
				Targets:    endpoint.NewTargets("1.2.3.4", "5.6.7.8"),
			},
		},
		Delete: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "TXT",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("\"heritage=external-dns,external-dns/owner=default\""),
			},
		},
	}
	p.ApplyChanges(&changes)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_Records_PrivateZone(t *testing.T) {
	p := newTestAlibabaCloudProvider(true)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_ApplyChanges_PrivateZone(t *testing.T) {
	p := newTestAlibabaCloudProvider(true)
	changes := plan.Changes{
		Create: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "xyz.container-service.top",
				RecordType: "A",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("4.3.2.1"),
			},
		},
		UpdateNew: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "A",
				RecordTTL:  500,
				Targets:    endpoint.NewTargets("1.2.3.4", "5.6.7.8"),
			},
		},
		Delete: []*endpoint.Endpoint{
			&endpoint.Endpoint{
				DNSName:    "abc.container-service.top",
				RecordType: "TXT",
				RecordTTL:  300,
				Targets:    endpoint.NewTargets("\"heritage=external-dns,external-dns/owner=default\""),
			},
		},
	}
	p.ApplyChanges(&changes)
	endpoints, err := p.Records()
	if err != nil {
		t.Errorf("Failed to get records: %v", err)
	} else {
		if len(endpoints) != 2 {
			t.Errorf("Incorrect number of records: %d", len(endpoints))
		}
		for _, endpoint := range endpoints {
			t.Logf("Endpoint for %++v", *endpoint)
		}
	}
}

func TestAlibabaCloudProvider_splitDNSName(t *testing.T) {
	p := newTestAlibabaCloudProvider(false)
	endpoint := &endpoint.Endpoint{}
	endpoint.DNSName = "www.example.org"
	rr, domain := p.splitDNSName(endpoint)
	if rr != "www" || domain != "example.org" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = ".example.org"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "example.org" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "www"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "www" || domain != "" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = ""
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "_30000._tcp.container-service.top"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "_30000._tcp" || domain != "container-service.top" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
	endpoint.DNSName = "container-service.top"
	rr, domain = p.splitDNSName(endpoint)
	if rr != "@" || domain != "container-service.top" {
		t.Errorf("Failed to splitDNSName for %s: rr=%s, domain=%s", endpoint.DNSName, rr, domain)
	}
}

func TestAlibabaCloudProvider_TXTEndpoint(t *testing.T) {

	p := newTestAlibabaCloudProvider(false)
	const recordValue = "heritage=external-dns,external-dns/owner=default"
	const endpointTarget = "\"heritage=external-dns,external-dns/owner=default\""

	if p.escapeTXTRecordValue(endpointTarget) != endpointTarget {
		t.Errorf("Failed to escapeTXTRecordValue: %s", p.escapeTXTRecordValue(endpointTarget))
	}
	if p.unescapeTXTRecordValue(recordValue) != endpointTarget {
		t.Errorf("Failed to unescapeTXTRecordValue: %s", p.unescapeTXTRecordValue(recordValue))
	}
}
<|endoftext|>


<commit_before>package inbound

import (
	"crypto/md5"
	"io"
	"sync"

	"github.com/v2ray/v2ray-core/app"
	"github.com/v2ray/v2ray-core/app/dispatcher"
	"github.com/v2ray/v2ray-core/app/proxyman"
	"github.com/v2ray/v2ray-core/common/alloc"
	v2crypto "github.com/v2ray/v2ray-core/common/crypto"
	v2io "github.com/v2ray/v2ray-core/common/io"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	proto "github.com/v2ray/v2ray-core/common/protocol"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy"
	"github.com/v2ray/v2ray-core/proxy/internal"
	vmessio "github.com/v2ray/v2ray-core/proxy/vmess/io"
	"github.com/v2ray/v2ray-core/proxy/vmess/protocol"
	"github.com/v2ray/v2ray-core/transport/hub"
)

// Inbound connection handler that handles messages in VMess format.
type VMessInboundHandler struct {
	sync.Mutex
	packetDispatcher      dispatcher.PacketDispatcher
	inboundHandlerManager proxyman.InboundHandlerManager
	clients               protocol.UserSet
	user                  *proto.User
	accepting             bool
	listener              *hub.TCPHub
	features              *FeaturesConfig
	listeningPort         v2net.Port
}

func (this *VMessInboundHandler) Port() v2net.Port {
	return this.listeningPort
}

func (this *VMessInboundHandler) Close() {
	this.accepting = false
	if this.listener != nil {
		this.Lock()
		this.listener.Close()
		this.listener = nil
		this.Unlock()
	}
}

func (this *VMessInboundHandler) GetUser() *proto.User {
	return this.user
}

func (this *VMessInboundHandler) Listen(port v2net.Port) error {
	if this.accepting {
		if this.listeningPort == port {
			return nil
		} else {
			return proxy.ErrorAlreadyListening
		}
	}
	this.listeningPort = port

	tcpListener, err := hub.ListenTCP(port, this.HandleConnection)
	if err != nil {
		log.Error("Unable to listen tcp port ", port, ": ", err)
		return err
	}
	this.accepting = true
	this.Lock()
	this.listener = tcpListener
	this.Unlock()
	return nil
}

func (this *VMessInboundHandler) HandleConnection(connection *hub.TCPConn) {
	defer connection.Close()

	connReader := v2net.NewTimeOutReader(16, connection)
	requestReader := protocol.NewVMessRequestReader(this.clients)

	request, err := requestReader.Read(connReader)
	if err != nil {
		log.Access(connection.RemoteAddr(), serial.StringLiteral(""), log.AccessRejected, serial.StringLiteral(err.Error()))
		log.Warning("VMessIn: Invalid request from ", connection.RemoteAddr(), ": ", err)
		return
	}
	log.Access(connection.RemoteAddr(), request.Address, log.AccessAccepted, serial.StringLiteral(""))
	log.Debug("VMessIn: Received request for ", request.Address)

	ray := this.packetDispatcher.DispatchToOutbound(v2net.NewPacket(request.Destination(), nil, true))
	input := ray.InboundInput()
	output := ray.InboundOutput()
	var readFinish, writeFinish sync.Mutex
	readFinish.Lock()
	writeFinish.Lock()

	userSettings := proto.GetUserSettings(request.User.Level)
	connReader.SetTimeOut(userSettings.PayloadReadTimeout)
	go handleInput(request, connReader, input, &readFinish)

	responseKey := md5.Sum(request.RequestKey)
	responseIV := md5.Sum(request.RequestIV)

	aesStream, err := v2crypto.NewAesEncryptionStream(responseKey[:], responseIV[:])
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		close(input)
		return
	}

	responseWriter := v2crypto.NewCryptionWriter(aesStream, connection)

	// Optimize for small response packet
	buffer := alloc.NewLargeBuffer().Clear()
	defer buffer.Release()
	buffer.AppendBytes(request.ResponseHeader, byte(0))
	this.generateCommand(buffer)

	if data, open := <-output; open {
		if request.IsChunkStream() {
			vmessio.Authenticate(data)
		}
		buffer.Append(data.Value)
		data.Release()
		responseWriter.Write(buffer.Value)
		go func(finish *sync.Mutex) {
			var writer v2io.Writer = v2io.NewAdaptiveWriter(responseWriter)
			if request.IsChunkStream() {
				writer = vmessio.NewAuthChunkWriter(writer)
			}
			v2io.ChanToWriter(writer, output)
			finish.Unlock()
		}(&writeFinish)
		writeFinish.Lock()
	}

	connection.CloseWrite()
	readFinish.Lock()
}

func handleInput(request *protocol.VMessRequest, reader io.Reader, input chan<- *alloc.Buffer, finish *sync.Mutex) {
	defer close(input)
	defer finish.Unlock()

	aesStream, err := v2crypto.NewAesDecryptionStream(request.RequestKey, request.RequestIV)
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		return
	}
	descriptionReader := v2crypto.NewCryptionReader(aesStream, reader)
	var requestReader v2io.Reader
	if request.IsChunkStream() {
		requestReader = vmessio.NewAuthChunkReader(descriptionReader)
	} else {
		requestReader = v2io.NewAdaptiveReader(descriptionReader)
	}
	v2io.ReaderToChan(input, requestReader)
}

func init() {
	internal.MustRegisterInboundHandlerCreator("vmess",
		func(space app.Space, rawConfig interface{}) (proxy.InboundHandler, error) {
			if !space.HasApp(dispatcher.APP_ID) {
				return nil, internal.ErrorBadConfiguration
			}
			config := rawConfig.(*Config)

			allowedClients := protocol.NewTimedUserSet()
			for _, user := range config.AllowedUsers {
				allowedClients.AddUser(user)
			}

			handler := &VMessInboundHandler{
				packetDispatcher: space.GetApp(dispatcher.APP_ID).(dispatcher.PacketDispatcher),
				clients:          allowedClients,
				features:         config.Features,
				user:             config.AllowedUsers[0],
			}

			if space.HasApp(proxyman.APP_ID_INBOUND_MANAGER) {
				handler.inboundHandlerManager = space.GetApp(proxyman.APP_ID_INBOUND_MANAGER).(proxyman.InboundHandlerManager)
			}

			return handler, nil
		})
}
<commit_msg>log whole destination in vmess out<commit_after>package inbound

import (
	"crypto/md5"
	"io"
	"sync"

	"github.com/v2ray/v2ray-core/app"
	"github.com/v2ray/v2ray-core/app/dispatcher"
	"github.com/v2ray/v2ray-core/app/proxyman"
	"github.com/v2ray/v2ray-core/common/alloc"
	v2crypto "github.com/v2ray/v2ray-core/common/crypto"
	v2io "github.com/v2ray/v2ray-core/common/io"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	proto "github.com/v2ray/v2ray-core/common/protocol"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy"
	"github.com/v2ray/v2ray-core/proxy/internal"
	vmessio "github.com/v2ray/v2ray-core/proxy/vmess/io"
	"github.com/v2ray/v2ray-core/proxy/vmess/protocol"
	"github.com/v2ray/v2ray-core/transport/hub"
)

// Inbound connection handler that handles messages in VMess format.
type VMessInboundHandler struct {
	sync.Mutex
	packetDispatcher      dispatcher.PacketDispatcher
	inboundHandlerManager proxyman.InboundHandlerManager
	clients               protocol.UserSet
	user                  *proto.User
	accepting             bool
	listener              *hub.TCPHub
	features              *FeaturesConfig
	listeningPort         v2net.Port
}

func (this *VMessInboundHandler) Port() v2net.Port {
	return this.listeningPort
}

func (this *VMessInboundHandler) Close() {
	this.accepting = false
	if this.listener != nil {
		this.Lock()
		this.listener.Close()
		this.listener = nil
		this.Unlock()
	}
}

func (this *VMessInboundHandler) GetUser() *proto.User {
	return this.user
}

func (this *VMessInboundHandler) Listen(port v2net.Port) error {
	if this.accepting {
		if this.listeningPort == port {
			return nil
		} else {
			return proxy.ErrorAlreadyListening
		}
	}
	this.listeningPort = port

	tcpListener, err := hub.ListenTCP(port, this.HandleConnection)
	if err != nil {
		log.Error("Unable to listen tcp port ", port, ": ", err)
		return err
	}
	this.accepting = true
	this.Lock()
	this.listener = tcpListener
	this.Unlock()
	return nil
}

func (this *VMessInboundHandler) HandleConnection(connection *hub.TCPConn) {
	defer connection.Close()

	connReader := v2net.NewTimeOutReader(16, connection)
	requestReader := protocol.NewVMessRequestReader(this.clients)

	request, err := requestReader.Read(connReader)
	if err != nil {
		log.Access(connection.RemoteAddr(), serial.StringLiteral(""), log.AccessRejected, serial.StringLiteral(err.Error()))
		log.Warning("VMessIn: Invalid request from ", connection.RemoteAddr(), ": ", err)
		return
	}
	log.Access(connection.RemoteAddr(), request.Destination(), log.AccessAccepted, serial.StringLiteral(""))
	log.Debug("VMessIn: Received request for ", request.Destination())

	ray := this.packetDispatcher.DispatchToOutbound(v2net.NewPacket(request.Destination(), nil, true))
	input := ray.InboundInput()
	output := ray.InboundOutput()
	var readFinish, writeFinish sync.Mutex
	readFinish.Lock()
	writeFinish.Lock()

	userSettings := proto.GetUserSettings(request.User.Level)
	connReader.SetTimeOut(userSettings.PayloadReadTimeout)
	go handleInput(request, connReader, input, &readFinish)

	responseKey := md5.Sum(request.RequestKey)
	responseIV := md5.Sum(request.RequestIV)

	aesStream, err := v2crypto.NewAesEncryptionStream(responseKey[:], responseIV[:])
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		close(input)
		return
	}

	responseWriter := v2crypto.NewCryptionWriter(aesStream, connection)

	// Optimize for small response packet
	buffer := alloc.NewLargeBuffer().Clear()
	defer buffer.Release()
	buffer.AppendBytes(request.ResponseHeader, byte(0))
	this.generateCommand(buffer)

	if data, open := <-output; open {
		if request.IsChunkStream() {
			vmessio.Authenticate(data)
		}
		buffer.Append(data.Value)
		data.Release()
		responseWriter.Write(buffer.Value)
		go func(finish *sync.Mutex) {
			var writer v2io.Writer = v2io.NewAdaptiveWriter(responseWriter)
			if request.IsChunkStream() {
				writer = vmessio.NewAuthChunkWriter(writer)
			}
			v2io.ChanToWriter(writer, output)
			finish.Unlock()
		}(&writeFinish)
		writeFinish.Lock()
	}

	connection.CloseWrite()
	readFinish.Lock()
}

func handleInput(request *protocol.VMessRequest, reader io.Reader, input chan<- *alloc.Buffer, finish *sync.Mutex) {
	defer close(input)
	defer finish.Unlock()

	aesStream, err := v2crypto.NewAesDecryptionStream(request.RequestKey, request.RequestIV)
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		return
	}
	descriptionReader := v2crypto.NewCryptionReader(aesStream, reader)
	var requestReader v2io.Reader
	if request.IsChunkStream() {
		requestReader = vmessio.NewAuthChunkReader(descriptionReader)
	} else {
		requestReader = v2io.NewAdaptiveReader(descriptionReader)
	}
	v2io.ReaderToChan(input, requestReader)
}

func init() {
	internal.MustRegisterInboundHandlerCreator("vmess",
		func(space app.Space, rawConfig interface{}) (proxy.InboundHandler, error) {
			if !space.HasApp(dispatcher.APP_ID) {
				return nil, internal.ErrorBadConfiguration
			}
			config := rawConfig.(*Config)

			allowedClients := protocol.NewTimedUserSet()
			for _, user := range config.AllowedUsers {
				allowedClients.AddUser(user)
			}

			handler := &VMessInboundHandler{
				packetDispatcher: space.GetApp(dispatcher.APP_ID).(dispatcher.PacketDispatcher),
				clients:          allowedClients,
				features:         config.Features,
				user:             config.AllowedUsers[0],
			}

			if space.HasApp(proxyman.APP_ID_INBOUND_MANAGER) {
				handler.inboundHandlerManager = space.GetApp(proxyman.APP_ID_INBOUND_MANAGER).(proxyman.InboundHandlerManager)
			}

			return handler, nil
		})
}
<|endoftext|>


<commit_before>package inbound

import (
	"crypto/md5"
	"io"
	"net"
	"sync"

	"github.com/v2ray/v2ray-core/app"
	"github.com/v2ray/v2ray-core/common/alloc"
	v2crypto "github.com/v2ray/v2ray-core/common/crypto"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/retry"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy"
	"github.com/v2ray/v2ray-core/proxy/internal"
	"github.com/v2ray/v2ray-core/proxy/vmess"
	"github.com/v2ray/v2ray-core/proxy/vmess/protocol"
)

// Inbound connection handler that handles messages in VMess format.
type VMessInboundHandler struct {
	sync.Mutex
	space         app.Space
	clients       protocol.UserSet
	user          *vmess.User
	accepting     bool
	listener      *net.TCPListener
	features      *FeaturesConfig
	listeningPort v2net.Port
}

func (this *VMessInboundHandler) Port() v2net.Port {
	return this.listeningPort
}

func (this *VMessInboundHandler) Close() {
	this.accepting = false
	if this.listener != nil {
		this.Lock()
		this.listener.Close()
		this.listener = nil
		this.Unlock()
	}
}

func (this *VMessInboundHandler) GetUser() *vmess.User {
	return this.user
}

func (this *VMessInboundHandler) Listen(port v2net.Port) error {
	if this.accepting {
		if this.listeningPort == port {
			return nil
		} else {
			return proxy.ErrorAlreadyListening
		}
	}
	this.listeningPort = port

	listener, err := net.ListenTCP("tcp", &net.TCPAddr{
		IP:   []byte{0, 0, 0, 0},
		Port: int(port),
		Zone: "",
	})
	if err != nil {
		log.Error("Unable to listen tcp port ", port, ": ", err)
		return err
	}
	this.accepting = true
	this.Lock()
	this.listener = listener
	this.Unlock()
	go this.AcceptConnections()
	return nil
}

func (this *VMessInboundHandler) AcceptConnections() error {
	for this.accepting {
		retry.Timed(100 /* times */, 100 /* ms */).On(func() error {
			this.Lock()
			defer this.Unlock()
			if !this.accepting {
				return nil
			}
			connection, err := this.listener.AcceptTCP()
			if err != nil {
				log.Error("Failed to accpet connection: ", err)
				return err
			}
			go this.HandleConnection(connection)
			return nil
		})

	}
	return nil
}

func (this *VMessInboundHandler) HandleConnection(connection *net.TCPConn) error {
	defer connection.Close()

	connReader := v2net.NewTimeOutReader(16, connection)
	requestReader := protocol.NewVMessRequestReader(this.clients)

	request, err := requestReader.Read(connReader)
	if err != nil {
		log.Access(connection.RemoteAddr(), serial.StringLiteral(""), log.AccessRejected, serial.StringLiteral(err.Error()))
		log.Warning("VMessIn: Invalid request from ", connection.RemoteAddr(), ": ", err)
		return err
	}
	log.Access(connection.RemoteAddr(), request.Address, log.AccessAccepted, serial.StringLiteral(""))
	log.Debug("VMessIn: Received request for ", request.Address)

	ray := this.space.PacketDispatcher().DispatchToOutbound(v2net.NewPacket(request.Destination(), nil, true))
	input := ray.InboundInput()
	output := ray.InboundOutput()
	var readFinish, writeFinish sync.Mutex
	readFinish.Lock()
	writeFinish.Lock()

	userSettings := vmess.GetUserSettings(request.User.Level)
	connReader.SetTimeOut(userSettings.PayloadReadTimeout)
	go handleInput(request, connReader, input, &readFinish)

	responseKey := md5.Sum(request.RequestKey)
	responseIV := md5.Sum(request.RequestIV)

	aesStream, err := v2crypto.NewAesEncryptionStream(responseKey[:], responseIV[:])
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		close(input)
		return err
	}

	responseWriter := v2crypto.NewCryptionWriter(aesStream, connection)

	// Optimize for small response packet
	buffer := alloc.NewLargeBuffer().Clear()
	defer buffer.Release()
	buffer.AppendBytes(request.ResponseHeader, byte(0))
	this.generateCommand(buffer)

	if data, open := <-output; open {
		buffer.Append(data.Value)
		data.Release()
		responseWriter.Write(buffer.Value)
		go handleOutput(request, responseWriter, output, &writeFinish)
		writeFinish.Lock()
	}

	connection.CloseWrite()
	readFinish.Lock()

	return nil
}

func handleInput(request *protocol.VMessRequest, reader io.Reader, input chan<- *alloc.Buffer, finish *sync.Mutex) {
	defer close(input)
	defer finish.Unlock()

	aesStream, err := v2crypto.NewAesDecryptionStream(request.RequestKey, request.RequestIV)
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		return
	}
	requestReader := v2crypto.NewCryptionReader(aesStream, reader)
	v2net.ReaderToChan(input, requestReader)
}

func handleOutput(request *protocol.VMessRequest, writer io.Writer, output <-chan *alloc.Buffer, finish *sync.Mutex) {
	v2net.ChanToWriter(writer, output)
	finish.Unlock()
}

func init() {
	internal.MustRegisterInboundConnectionHandlerCreator("vmess",
		func(space app.Space, rawConfig interface{}) (proxy.InboundConnectionHandler, error) {
			config := rawConfig.(*Config)

			allowedClients := protocol.NewTimedUserSet()
			for _, user := range config.AllowedUsers {
				allowedClients.AddUser(user)
			}

			return &VMessInboundHandler{
				space:    space,
				clients:  allowedClients,
				features: config.Features,
				user:     config.AllowedUsers[0],
			}, nil
		})
}
<commit_msg>revert lock change<commit_after>package inbound

import (
	"crypto/md5"
	"io"
	"net"
	"sync"

	"github.com/v2ray/v2ray-core/app"
	"github.com/v2ray/v2ray-core/common/alloc"
	v2crypto "github.com/v2ray/v2ray-core/common/crypto"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/retry"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy"
	"github.com/v2ray/v2ray-core/proxy/internal"
	"github.com/v2ray/v2ray-core/proxy/vmess"
	"github.com/v2ray/v2ray-core/proxy/vmess/protocol"
)

// Inbound connection handler that handles messages in VMess format.
type VMessInboundHandler struct {
	sync.Mutex
	space         app.Space
	clients       protocol.UserSet
	user          *vmess.User
	accepting     bool
	listener      *net.TCPListener
	features      *FeaturesConfig
	listeningPort v2net.Port
}

func (this *VMessInboundHandler) Port() v2net.Port {
	return this.listeningPort
}

func (this *VMessInboundHandler) Close() {
	this.accepting = false
	if this.listener != nil {
		this.listener.Close()
		this.Lock()
		this.listener = nil
		this.Unlock()
	}
}

func (this *VMessInboundHandler) GetUser() *vmess.User {
	return this.user
}

func (this *VMessInboundHandler) Listen(port v2net.Port) error {
	if this.accepting {
		if this.listeningPort == port {
			return nil
		} else {
			return proxy.ErrorAlreadyListening
		}
	}
	this.listeningPort = port

	listener, err := net.ListenTCP("tcp", &net.TCPAddr{
		IP:   []byte{0, 0, 0, 0},
		Port: int(port),
		Zone: "",
	})
	if err != nil {
		log.Error("Unable to listen tcp port ", port, ": ", err)
		return err
	}
	this.accepting = true
	this.Lock()
	this.listener = listener
	this.Unlock()
	go this.AcceptConnections()
	return nil
}

func (this *VMessInboundHandler) AcceptConnections() error {
	for this.accepting {
		retry.Timed(100 /* times */, 100 /* ms */).On(func() error {
			this.Lock()
			defer this.Unlock()
			if !this.accepting {
				return nil
			}
			connection, err := this.listener.AcceptTCP()
			if err != nil {
				log.Error("Failed to accpet connection: ", err)
				return err
			}
			go this.HandleConnection(connection)
			return nil
		})

	}
	return nil
}

func (this *VMessInboundHandler) HandleConnection(connection *net.TCPConn) error {
	defer connection.Close()

	connReader := v2net.NewTimeOutReader(16, connection)
	requestReader := protocol.NewVMessRequestReader(this.clients)

	request, err := requestReader.Read(connReader)
	if err != nil {
		log.Access(connection.RemoteAddr(), serial.StringLiteral(""), log.AccessRejected, serial.StringLiteral(err.Error()))
		log.Warning("VMessIn: Invalid request from ", connection.RemoteAddr(), ": ", err)
		return err
	}
	log.Access(connection.RemoteAddr(), request.Address, log.AccessAccepted, serial.StringLiteral(""))
	log.Debug("VMessIn: Received request for ", request.Address)

	ray := this.space.PacketDispatcher().DispatchToOutbound(v2net.NewPacket(request.Destination(), nil, true))
	input := ray.InboundInput()
	output := ray.InboundOutput()
	var readFinish, writeFinish sync.Mutex
	readFinish.Lock()
	writeFinish.Lock()

	userSettings := vmess.GetUserSettings(request.User.Level)
	connReader.SetTimeOut(userSettings.PayloadReadTimeout)
	go handleInput(request, connReader, input, &readFinish)

	responseKey := md5.Sum(request.RequestKey)
	responseIV := md5.Sum(request.RequestIV)

	aesStream, err := v2crypto.NewAesEncryptionStream(responseKey[:], responseIV[:])
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		close(input)
		return err
	}

	responseWriter := v2crypto.NewCryptionWriter(aesStream, connection)

	// Optimize for small response packet
	buffer := alloc.NewLargeBuffer().Clear()
	defer buffer.Release()
	buffer.AppendBytes(request.ResponseHeader, byte(0))
	this.generateCommand(buffer)

	if data, open := <-output; open {
		buffer.Append(data.Value)
		data.Release()
		responseWriter.Write(buffer.Value)
		go handleOutput(request, responseWriter, output, &writeFinish)
		writeFinish.Lock()
	}

	connection.CloseWrite()
	readFinish.Lock()

	return nil
}

func handleInput(request *protocol.VMessRequest, reader io.Reader, input chan<- *alloc.Buffer, finish *sync.Mutex) {
	defer close(input)
	defer finish.Unlock()

	aesStream, err := v2crypto.NewAesDecryptionStream(request.RequestKey, request.RequestIV)
	if err != nil {
		log.Error("VMessIn: Failed to create AES decryption stream: ", err)
		return
	}
	requestReader := v2crypto.NewCryptionReader(aesStream, reader)
	v2net.ReaderToChan(input, requestReader)
}

func handleOutput(request *protocol.VMessRequest, writer io.Writer, output <-chan *alloc.Buffer, finish *sync.Mutex) {
	v2net.ChanToWriter(writer, output)
	finish.Unlock()
}

func init() {
	internal.MustRegisterInboundConnectionHandlerCreator("vmess",
		func(space app.Space, rawConfig interface{}) (proxy.InboundConnectionHandler, error) {
			config := rawConfig.(*Config)

			allowedClients := protocol.NewTimedUserSet()
			for _, user := range config.AllowedUsers {
				allowedClients.AddUser(user)
			}

			return &VMessInboundHandler{
				space:    space,
				clients:  allowedClients,
				features: config.Features,
				user:     config.AllowedUsers[0],
			}, nil
		})
}
<|endoftext|>


<commit_before>package nsqd

import (
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"strconv"
	"testing"
	"time"

	"github.com/bitly/go-simplejson"
)

func assert(t *testing.T, condition bool, msg string, v ...interface{}) {
	if !condition {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d: "+msg+"\033[39m\n\n",
			append([]interface{}{filepath.Base(file), line}, v...)...)
		t.FailNow()
	}
}

func equal(t *testing.T, act, exp interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n",
			filepath.Base(file), line, exp, act)
		t.FailNow()
	}
}

func nequal(t *testing.T, act, exp interface{}) {
	if reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n",
			filepath.Base(file), line, exp, act)
		t.FailNow()
	}
}

type tbLog interface {
	Log(...interface{})
}

type testLogger struct {
	tbLog
}

func (tl *testLogger) Output(maxdepth int, s string) error {
	tl.Log(s)
	return nil
}

func newTestLogger(tbl tbLog) logger {
	return &testLogger{tbl}
}

func getMetadata(n *NSQD) (*simplejson.Json, error) {
	fn := fmt.Sprintf(path.Join(n.opts.DataPath, "nsqd.%d.dat"), n.opts.ID)
	data, err := ioutil.ReadFile(fn)
	if err != nil {
		return nil, err
	}

	js, err := simplejson.NewJson(data)
	if err != nil {
		return nil, err
	}
	return js, nil
}

func TestStartup(t *testing.T) {
	iterations := 300
	doneExitChan := make(chan int)

	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	opts.MaxBytesPerFile = 10240
	_, _, nsqd := mustStartNSQD(opts)

	topicName := "nsqd_test" + strconv.Itoa(int(time.Now().Unix()))

	exitChan := make(chan int)
	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	// verify nsqd metadata shows no topics
	err := nsqd.PersistMetadata()
	equal(t, err, nil)
	metaData, err := getMetadata(nsqd)
	equal(t, err, nil)
	topics, err := metaData.Get("topics").Array()
	equal(t, err, nil)
	equal(t, len(topics), 0)

	body := make([]byte, 256)
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < iterations; i++ {
		msg := NewMessage(<-nsqd.idChan, body)
		topic.PutMessage(msg)
	}

	t.Logf("pulling from channel")
	channel1 := topic.GetChannel("ch1")

	t.Logf("read %d msgs", iterations/2)
	for i := 0; i < iterations/2; i++ {
		msg := <-channel1.clientMsgChan
		t.Logf("read message %d", i+1)
		equal(t, msg.Body, body)
	}

	for {
		if channel1.Depth() == int64(iterations/2) {
			break
		}
		time.Sleep(50 * time.Millisecond)
	}

	// make sure metadata shows the topic
	metaData, err = getMetadata(nsqd)
	equal(t, err, nil)
	topics, err = metaData.Get("topics").Array()
	equal(t, err, nil)
	equal(t, len(topics), 1)
	observedTopicName, err := metaData.Get("topics").GetIndex(0).Get("name").String()
	equal(t, observedTopicName, topicName)
	equal(t, err, nil)

	exitChan <- 1
	<-doneExitChan

	// start up a new nsqd w/ the same folder

	opts = NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	opts.MaxBytesPerFile = 10240
	_, _, nsqd = mustStartNSQD(opts)

	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	topic = nsqd.GetTopic(topicName)
	// should be empty; channel should have drained everything
	count := topic.Depth()
	equal(t, count, int64(0))

	channel1 = topic.GetChannel("ch1")

	for {
		if channel1.Depth() == int64(iterations/2) {
			break
		}
		time.Sleep(50 * time.Millisecond)
	}

	// read the other half of the messages
	for i := 0; i < iterations/2; i++ {
		msg := <-channel1.clientMsgChan
		t.Logf("read message %d", i+1)
		equal(t, msg.Body, body)
	}

	// verify we drained things
	equal(t, len(topic.memoryMsgChan), 0)
	equal(t, topic.backend.Depth(), int64(0))

	exitChan <- 1
	<-doneExitChan
}

func TestEphemeralTopicsAndChannels(t *testing.T) {
	// ephemeral topics/channels are lazily removed after the last channel/client is removed
	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	_, _, nsqd := mustStartNSQD(opts)

	topicName := "ephemeral_topic" + strconv.Itoa(int(time.Now().Unix())) + "#ephemeral"
	doneExitChan := make(chan int)

	exitChan := make(chan int)
	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	body := []byte("an_ephemeral_message")
	topic := nsqd.GetTopic(topicName)
	ephemeralChannel := topic.GetChannel("ch1#ephemeral")
	client := newClientV2(0, nil, &context{nsqd})
	ephemeralChannel.AddClient(client.ID, client)

	msg := NewMessage(<-nsqd.idChan, body)
	topic.PutMessage(msg)
	msg = <-ephemeralChannel.clientMsgChan
	equal(t, msg.Body, body)

	ephemeralChannel.RemoveClient(client.ID)

	time.Sleep(50 * time.Millisecond)

	topic.Lock()
	numChannels := len(topic.channelMap)
	topic.Unlock()
	equal(t, numChannels, 0)

	nsqd.Lock()
	numTopics := len(nsqd.topicMap)
	nsqd.Unlock()
	equal(t, numTopics, 0)

	exitChan <- 1
	<-doneExitChan
}

func metadataForChannel(n *NSQD, topicIndex int, channelIndex int) *simplejson.Json {
	metadata, _ := getMetadata(n)
	mChannels := metadata.Get("topics").GetIndex(topicIndex).Get("channels")
	return mChannels.GetIndex(channelIndex)
}

func TestPauseMetadata(t *testing.T) {
	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	_, _, nsqd := mustStartNSQD(opts)

	topicName := "pause_metadata" + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel("ch")

	b, _ := metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, false)

	channel.Pause()

	b, _ = metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, true)

	channel.UnPause()

	b, _ = metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, false)
}
<commit_msg>Gracefully shut down nsqd after running TestPauseMetadata()<commit_after>package nsqd

import (
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"strconv"
	"testing"
	"time"

	"github.com/bitly/go-simplejson"
)

func assert(t *testing.T, condition bool, msg string, v ...interface{}) {
	if !condition {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d: "+msg+"\033[39m\n\n",
			append([]interface{}{filepath.Base(file), line}, v...)...)
		t.FailNow()
	}
}

func equal(t *testing.T, act, exp interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n",
			filepath.Base(file), line, exp, act)
		t.FailNow()
	}
}

func nequal(t *testing.T, act, exp interface{}) {
	if reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n",
			filepath.Base(file), line, exp, act)
		t.FailNow()
	}
}

type tbLog interface {
	Log(...interface{})
}

type testLogger struct {
	tbLog
}

func (tl *testLogger) Output(maxdepth int, s string) error {
	tl.Log(s)
	return nil
}

func newTestLogger(tbl tbLog) logger {
	return &testLogger{tbl}
}

func getMetadata(n *NSQD) (*simplejson.Json, error) {
	fn := fmt.Sprintf(path.Join(n.opts.DataPath, "nsqd.%d.dat"), n.opts.ID)
	data, err := ioutil.ReadFile(fn)
	if err != nil {
		return nil, err
	}

	js, err := simplejson.NewJson(data)
	if err != nil {
		return nil, err
	}
	return js, nil
}

func TestStartup(t *testing.T) {
	iterations := 300
	doneExitChan := make(chan int)

	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	opts.MaxBytesPerFile = 10240
	_, _, nsqd := mustStartNSQD(opts)

	topicName := "nsqd_test" + strconv.Itoa(int(time.Now().Unix()))

	exitChan := make(chan int)
	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	// verify nsqd metadata shows no topics
	err := nsqd.PersistMetadata()
	equal(t, err, nil)
	metaData, err := getMetadata(nsqd)
	equal(t, err, nil)
	topics, err := metaData.Get("topics").Array()
	equal(t, err, nil)
	equal(t, len(topics), 0)

	body := make([]byte, 256)
	topic := nsqd.GetTopic(topicName)
	for i := 0; i < iterations; i++ {
		msg := NewMessage(<-nsqd.idChan, body)
		topic.PutMessage(msg)
	}

	t.Logf("pulling from channel")
	channel1 := topic.GetChannel("ch1")

	t.Logf("read %d msgs", iterations/2)
	for i := 0; i < iterations/2; i++ {
		msg := <-channel1.clientMsgChan
		t.Logf("read message %d", i+1)
		equal(t, msg.Body, body)
	}

	for {
		if channel1.Depth() == int64(iterations/2) {
			break
		}
		time.Sleep(50 * time.Millisecond)
	}

	// make sure metadata shows the topic
	metaData, err = getMetadata(nsqd)
	equal(t, err, nil)
	topics, err = metaData.Get("topics").Array()
	equal(t, err, nil)
	equal(t, len(topics), 1)
	observedTopicName, err := metaData.Get("topics").GetIndex(0).Get("name").String()
	equal(t, observedTopicName, topicName)
	equal(t, err, nil)

	exitChan <- 1
	<-doneExitChan

	// start up a new nsqd w/ the same folder

	opts = NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	opts.MaxBytesPerFile = 10240
	_, _, nsqd = mustStartNSQD(opts)

	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	topic = nsqd.GetTopic(topicName)
	// should be empty; channel should have drained everything
	count := topic.Depth()
	equal(t, count, int64(0))

	channel1 = topic.GetChannel("ch1")

	for {
		if channel1.Depth() == int64(iterations/2) {
			break
		}
		time.Sleep(50 * time.Millisecond)
	}

	// read the other half of the messages
	for i := 0; i < iterations/2; i++ {
		msg := <-channel1.clientMsgChan
		t.Logf("read message %d", i+1)
		equal(t, msg.Body, body)
	}

	// verify we drained things
	equal(t, len(topic.memoryMsgChan), 0)
	equal(t, topic.backend.Depth(), int64(0))

	exitChan <- 1
	<-doneExitChan
}

func TestEphemeralTopicsAndChannels(t *testing.T) {
	// ephemeral topics/channels are lazily removed after the last channel/client is removed
	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	opts.MemQueueSize = 100
	_, _, nsqd := mustStartNSQD(opts)

	topicName := "ephemeral_topic" + strconv.Itoa(int(time.Now().Unix())) + "#ephemeral"
	doneExitChan := make(chan int)

	exitChan := make(chan int)
	go func() {
		<-exitChan
		nsqd.Exit()
		doneExitChan <- 1
	}()

	body := []byte("an_ephemeral_message")
	topic := nsqd.GetTopic(topicName)
	ephemeralChannel := topic.GetChannel("ch1#ephemeral")
	client := newClientV2(0, nil, &context{nsqd})
	ephemeralChannel.AddClient(client.ID, client)

	msg := NewMessage(<-nsqd.idChan, body)
	topic.PutMessage(msg)
	msg = <-ephemeralChannel.clientMsgChan
	equal(t, msg.Body, body)

	ephemeralChannel.RemoveClient(client.ID)

	time.Sleep(50 * time.Millisecond)

	topic.Lock()
	numChannels := len(topic.channelMap)
	topic.Unlock()
	equal(t, numChannels, 0)

	nsqd.Lock()
	numTopics := len(nsqd.topicMap)
	nsqd.Unlock()
	equal(t, numTopics, 0)

	exitChan <- 1
	<-doneExitChan
}

func metadataForChannel(n *NSQD, topicIndex int, channelIndex int) *simplejson.Json {
	metadata, _ := getMetadata(n)
	mChannels := metadata.Get("topics").GetIndex(topicIndex).Get("channels")
	return mChannels.GetIndex(channelIndex)
}

func TestPauseMetadata(t *testing.T) {
	opts := NewNSQDOptions()
	opts.Logger = newTestLogger(t)
	_, _, nsqd := mustStartNSQD(opts)
	defer nsqd.Exit()

	topicName := "pause_metadata" + strconv.Itoa(int(time.Now().Unix()))
	topic := nsqd.GetTopic(topicName)
	channel := topic.GetChannel("ch")

	b, _ := metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, false)

	channel.Pause()

	b, _ = metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, true)

	channel.UnPause()

	b, _ = metadataForChannel(nsqd, 0, 0).Get("paused").Bool()
	equal(t, b, false)
}
<|endoftext|>


<commit_before>// Package gzip implements a gzip compression handler middleware for Negroni.
package gzip

import (
	"compress/gzip"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/codegangsta/negroni"
)

// These compression constants are copied from the compress/gzip package.
const (
	encodingGzip = "gzip"

	headerAcceptEncoding  = "Accept-Encoding"
	headerContentEncoding = "Content-Encoding"
	headerContentLength   = "Content-Length"
	headerContentType     = "Content-Type"
	headerVary            = "Vary"
	headerSecWebSocketKey = "Sec-WebSocket-Key"

	BestCompression    = gzip.BestCompression
	BestSpeed          = gzip.BestSpeed
	DefaultCompression = gzip.DefaultCompression
	NoCompression      = gzip.NoCompression
)

// gzipResponseWriter is the ResponseWriter that negroni.ResponseWriter is
// wrapped in.
type gzipResponseWriter struct {
	w *gzip.Writer
	negroni.ResponseWriter
}

// Write writes bytes to the gzip.Writer. It will also set the Content-Type
// header using the net/http library content type detection if the Content-Type
// header was not set yet.
func (grw gzipResponseWriter) Write(b []byte) (int, error) {
	if len(grw.Header().Get(headerContentType)) == 0 {
		grw.Header().Set(headerContentType, http.DetectContentType(b))
	}
	return grw.w.Write(b)
}

// handler struct contains the ServeHTTP method
type handler struct {
	pool sync.Pool
}

// Gzip returns a handler which will handle the Gzip compression in ServeHTTP.
// Valid values for level are identical to those in the compress/gzip package.
func Gzip(level int) *handler {
	h := &handler{}
	h.pool.New = func() interface{} {
		gz, err := gzip.NewWriterLevel(ioutil.Discard, level)
		if err != nil {
			panic(err)
		}
		return gz
	}
	return h
}

// ServeHTTP wraps the http.ResponseWriter with a gzip.Writer.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	// Skip compression if the client doesn't accept gzip encoding.
	if !strings.Contains(r.Header.Get(headerAcceptEncoding), encodingGzip) {
		next(w, r)
		return
	}

	// Skip compression if client attempt WebSocket connection
	if len(r.Header.Get(headerSecWebSocketKey)) > 0 {
		next(w, r)
		return
	}

	// Skip compression if already compressed
	if w.Header().Get(headerContentEncoding) == encodingGzip {
		next(w, r)
		return
	}

	// Retrieve gzip writer from the pool. Reset it to use the ResponseWriter.
	// This allows us to re-use an already allocated buffer rather than
	// allocating a new buffer for every request.
	// We defer g.pool.Put here so that the gz writer is returned to the
	// pool if any thing after here fails for some reason (functions in
	// next could potentially panic, etc)
	gz := h.pool.Get().(*gzip.Writer)
	defer h.pool.Put(gz)
	gz.Reset(w)

	// Set the appropriate gzip headers.
	headers := w.Header()
	headers.Set(headerContentEncoding, encodingGzip)
	headers.Set(headerVary, headerAcceptEncoding)

	// Wrap the original http.ResponseWriter with negroni.ResponseWriter
	// and create the gzipResponseWriter.
	nrw := negroni.NewResponseWriter(w)
	grw := gzipResponseWriter{
		gz,
		nrw,
	}

	// Call the next handler supplying the gzipResponseWriter instead of
	// the original.
	next(grw, r)

	// Delete the content length after we know we have been written to.
	grw.Header().Del(headerContentLength)

	gz.Close()
}
<commit_msg>Update Negroni import path<commit_after>// Package gzip implements a gzip compression handler middleware for Negroni.
package gzip

import (
	"compress/gzip"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/urfave/negroni"
)

// These compression constants are copied from the compress/gzip package.
const (
	encodingGzip = "gzip"

	headerAcceptEncoding  = "Accept-Encoding"
	headerContentEncoding = "Content-Encoding"
	headerContentLength   = "Content-Length"
	headerContentType     = "Content-Type"
	headerVary            = "Vary"
	headerSecWebSocketKey = "Sec-WebSocket-Key"

	BestCompression    = gzip.BestCompression
	BestSpeed          = gzip.BestSpeed
	DefaultCompression = gzip.DefaultCompression
	NoCompression      = gzip.NoCompression
)

// gzipResponseWriter is the ResponseWriter that negroni.ResponseWriter is
// wrapped in.
type gzipResponseWriter struct {
	w *gzip.Writer
	negroni.ResponseWriter
}

// Write writes bytes to the gzip.Writer. It will also set the Content-Type
// header using the net/http library content type detection if the Content-Type
// header was not set yet.
func (grw gzipResponseWriter) Write(b []byte) (int, error) {
	if len(grw.Header().Get(headerContentType)) == 0 {
		grw.Header().Set(headerContentType, http.DetectContentType(b))
	}
	return grw.w.Write(b)
}

// handler struct contains the ServeHTTP method
type handler struct {
	pool sync.Pool
}

// Gzip returns a handler which will handle the Gzip compression in ServeHTTP.
// Valid values for level are identical to those in the compress/gzip package.
func Gzip(level int) *handler {
	h := &handler{}
	h.pool.New = func() interface{} {
		gz, err := gzip.NewWriterLevel(ioutil.Discard, level)
		if err != nil {
			panic(err)
		}
		return gz
	}
	return h
}

// ServeHTTP wraps the http.ResponseWriter with a gzip.Writer.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	// Skip compression if the client doesn't accept gzip encoding.
	if !strings.Contains(r.Header.Get(headerAcceptEncoding), encodingGzip) {
		next(w, r)
		return
	}

	// Skip compression if client attempt WebSocket connection
	if len(r.Header.Get(headerSecWebSocketKey)) > 0 {
		next(w, r)
		return
	}

	// Skip compression if already compressed
	if w.Header().Get(headerContentEncoding) == encodingGzip {
		next(w, r)
		return
	}

	// Retrieve gzip writer from the pool. Reset it to use the ResponseWriter.
	// This allows us to re-use an already allocated buffer rather than
	// allocating a new buffer for every request.
	// We defer g.pool.Put here so that the gz writer is returned to the
	// pool if any thing after here fails for some reason (functions in
	// next could potentially panic, etc)
	gz := h.pool.Get().(*gzip.Writer)
	defer h.pool.Put(gz)
	gz.Reset(w)

	// Set the appropriate gzip headers.
	headers := w.Header()
	headers.Set(headerContentEncoding, encodingGzip)
	headers.Set(headerVary, headerAcceptEncoding)

	// Wrap the original http.ResponseWriter with negroni.ResponseWriter
	// and create the gzipResponseWriter.
	nrw := negroni.NewResponseWriter(w)
	grw := gzipResponseWriter{
		gz,
		nrw,
	}

	// Call the next handler supplying the gzipResponseWriter instead of
	// the original.
	next(grw, r)

	// Delete the content length after we know we have been written to.
	grw.Header().Del(headerContentLength)

	gz.Close()
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io"

// hash index is a number, uint64, representing a deterministic non-redundant []byte.
// all index values have a unique byte sequence, but not all 8-byte sequences can be produced, because variable length means some states are encoded by the length.
// the un=indexable []byte's are above {0xFE,0xFE,0xFE,0xFE,0xFE,0xFE,0xFE,0xFE}
type hashIndexType uint64

// reader implemenation returning the []byte represented by hashindex
func (hi hashIndexType) Read(b []byte) (n int, err error) {
	if hi == 0 {
		return 0, io.EOF
	}
	if hi < 0x0101 {
		hi -= 1
		b[0] = uint8(hi)
		return 1, io.EOF
	}
	if hi < 0x010101 {
		hi -= 0x0101
		b[1] = uint8(hi)
		b[0] = uint8(hi >> 8)
		return 2, io.EOF
	}
	if hi < 0x01010101 {
		hi -= 0x010101
		b[2] = uint8(hi)
		b[1] = uint8(hi >> 8)
		b[0] = uint8(hi >> 16)
		return 3, io.EOF
	}
	if hi < 0x0101010101 {
		hi -= 0x01010101
		b[3] = uint8(hi)
		b[2] = uint8(hi >> 8)
		b[1] = uint8(hi >> 16)
		b[0] = uint8(hi >> 24)
		return 4, io.EOF
	}
	if hi < 0x010101010101 {
		hi -= 0x0101010101
		b[4] = uint8(hi)
		b[3] = uint8(hi >> 8)
		b[2] = uint8(hi >> 16)
		b[1] = uint8(hi >> 24)
		b[0] = uint8(hi >> 32)
		return 5, io.EOF
	}
	if hi < 0x01010101010101 {
		hi -= 0x010101010101
		b[5] = uint8(hi)
		b[4] = uint8(hi >> 8)
		b[3] = uint8(hi >> 16)
		b[2] = uint8(hi >> 24)
		b[1] = uint8(hi >> 32)
		b[0] = uint8(hi >> 40)
		return 6, io.EOF
	}
	if hi < 0x0101010101010101 {
		hi -= 0x01010101010101
		b[6] = uint8(hi)
		b[5] = uint8(hi >> 8)
		b[4] = uint8(hi >> 16)
		b[3] = uint8(hi >> 24)
		b[2] = uint8(hi >> 32)
		b[1] = uint8(hi >> 40)
		b[0] = uint8(hi >> 48)
		return 7, io.EOF
	}
	hi -= 0x0101010101010101
	b[7] = uint8(hi)
	b[6] = uint8(hi >> 8)
	b[5] = uint8(hi >> 16)
	b[4] = uint8(hi >> 24)
	b[3] = uint8(hi >> 32)
	b[2] = uint8(hi >> 40)
	b[1] = uint8(hi >> 48)
	b[0] = uint8(hi >> 56)
	return 8, io.EOF
}

// new hash index from a []byte
func NewHashIndexType(b []byte) (hi hashIndexType) {
	switch len(b) {
	case 0:
		return hashIndexType(0)
	case 1:
		return hashIndexType(1 + uint64(b[0]))
	case 2:
		return hashIndexType(0x0101 + uint64(b[1]) + uint64(b[0])<<8)
	case 3:
		return hashIndexType(0x010101 + uint64(b[2]) + uint64(b[1])<<8 + uint64(b[0])<<16)
	case 4:
		return hashIndexType(0x01010101 + uint64(b[3]) + uint64(b[2])<<8 + uint64(b[1])<<16 + uint64(b[0])<<24)
	case 5:
		return hashIndexType(0x0101010101 + uint64(b[4]) + uint64(b[3])<<8 + uint64(b[2])<<16 + uint64(b[1])<<24 + uint64(b[0])<<32)
	case 6:
		return hashIndexType(0x010101010101 + uint64(b[5]) + uint64(b[4])<<8 + uint64(b[3])<<16 + uint64(b[2])<<24 + uint64(b[1])<<32 + uint64(b[0])<<40)
	case 7:
		return hashIndexType(0x01010101010101 + uint64(b[6]) + uint64(b[5])<<8 + uint64(b[4])<<16 + uint64(b[3])<<24 + uint64(b[2])<<32 + uint64(b[1])<<40 + uint64(b[0])<<48)
	default:
		return hashIndexType(0x0101010101010101 + uint64(b[7]) + uint64(b[6])<<8 + uint64(b[5])<<16 + uint64(b[4])<<24 + uint64(b[3])<<32 + uint64(b[2])<<40 + uint64(b[1])<<48 + uint64(b[0])<<56)
	}
}

// string rep as hexadecimal
func (hi hashIndexType) String() string {
	b := make([]byte, 8, 8)
	n, _ := hi.Read(b)
	return fmt.Sprintf("% x", b[:n])
}

// return new hashindex whoses rep is as the source heashindex but with added byte(s)
func (hi hashIndexType) Append(b byte) hashIndexType{
	buf := make([]byte, 8, 8)
	n, _ := hi.Read(buf)
	buf[n]=b
	return NewHashIndexType(buf[:n+1])
}

// return new hashindex whoses rep is as the source heashindex but with removed byte(s)
func (hi hashIndexType) Truncate(c int) hashIndexType{
	buf := make([]byte, 8, 8)
	n, _ := hi.Read(buf)
	return NewHashIndexType(buf[:n-c])
}

<commit_msg>comment<commit_after>package main

import "fmt"
import "io"

// hash index is a number, uint64, with continuous, variable length []byte representation (binary encoding).
// all index values have a unique byte sequence, but not all 8-byte sequences can be produced, because variable length means some states are encoded by the length.
// the un=indexable []byte's are above {0xFE,0xFE,0xFE,0xFE,0xFE,0xFE,0xFE,0xFE}
type hashIndexType uint64

// reader implemenation returning the []byte represented by hashindex
func (hi hashIndexType) Read(b []byte) (n int, err error) {
	if hi == 0 {
		return 0, io.EOF
	}
	if hi < 0x0101 {
		hi -= 1
		b[0] = uint8(hi)
		return 1, io.EOF
	}
	if hi < 0x010101 {
		hi -= 0x0101
		b[1] = uint8(hi)
		b[0] = uint8(hi >> 8)
		return 2, io.EOF
	}
	if hi < 0x01010101 {
		hi -= 0x010101
		b[2] = uint8(hi)
		b[1] = uint8(hi >> 8)
		b[0] = uint8(hi >> 16)
		return 3, io.EOF
	}
	if hi < 0x0101010101 {
		hi -= 0x01010101
		b[3] = uint8(hi)
		b[2] = uint8(hi >> 8)
		b[1] = uint8(hi >> 16)
		b[0] = uint8(hi >> 24)
		return 4, io.EOF
	}
	if hi < 0x010101010101 {
		hi -= 0x0101010101
		b[4] = uint8(hi)
		b[3] = uint8(hi >> 8)
		b[2] = uint8(hi >> 16)
		b[1] = uint8(hi >> 24)
		b[0] = uint8(hi >> 32)
		return 5, io.EOF
	}
	if hi < 0x01010101010101 {
		hi -= 0x010101010101
		b[5] = uint8(hi)
		b[4] = uint8(hi >> 8)
		b[3] = uint8(hi >> 16)
		b[2] = uint8(hi >> 24)
		b[1] = uint8(hi >> 32)
		b[0] = uint8(hi >> 40)
		return 6, io.EOF
	}
	if hi < 0x0101010101010101 {
		hi -= 0x01010101010101
		b[6] = uint8(hi)
		b[5] = uint8(hi >> 8)
		b[4] = uint8(hi >> 16)
		b[3] = uint8(hi >> 24)
		b[2] = uint8(hi >> 32)
		b[1] = uint8(hi >> 40)
		b[0] = uint8(hi >> 48)
		return 7, io.EOF
	}
	hi -= 0x0101010101010101
	b[7] = uint8(hi)
	b[6] = uint8(hi >> 8)
	b[5] = uint8(hi >> 16)
	b[4] = uint8(hi >> 24)
	b[3] = uint8(hi >> 32)
	b[2] = uint8(hi >> 40)
	b[1] = uint8(hi >> 48)
	b[0] = uint8(hi >> 56)
	return 8, io.EOF
}

// new hash index from a []byte
func NewHashIndexType(b []byte) (hi hashIndexType) {
	switch len(b) {
	case 0:
		return hashIndexType(0)
	case 1:
		return hashIndexType(1 + uint64(b[0]))
	case 2:
		return hashIndexType(0x0101 + uint64(b[1]) + uint64(b[0])<<8)
	case 3:
		return hashIndexType(0x010101 + uint64(b[2]) + uint64(b[1])<<8 + uint64(b[0])<<16)
	case 4:
		return hashIndexType(0x01010101 + uint64(b[3]) + uint64(b[2])<<8 + uint64(b[1])<<16 + uint64(b[0])<<24)
	case 5:
		return hashIndexType(0x0101010101 + uint64(b[4]) + uint64(b[3])<<8 + uint64(b[2])<<16 + uint64(b[1])<<24 + uint64(b[0])<<32)
	case 6:
		return hashIndexType(0x010101010101 + uint64(b[5]) + uint64(b[4])<<8 + uint64(b[3])<<16 + uint64(b[2])<<24 + uint64(b[1])<<32 + uint64(b[0])<<40)
	case 7:
		return hashIndexType(0x01010101010101 + uint64(b[6]) + uint64(b[5])<<8 + uint64(b[4])<<16 + uint64(b[3])<<24 + uint64(b[2])<<32 + uint64(b[1])<<40 + uint64(b[0])<<48)
	default:
		return hashIndexType(0x0101010101010101 + uint64(b[7]) + uint64(b[6])<<8 + uint64(b[5])<<16 + uint64(b[4])<<24 + uint64(b[3])<<32 + uint64(b[2])<<40 + uint64(b[1])<<48 + uint64(b[0])<<56)
	}
}

// string rep as hexadecimal
func (hi hashIndexType) String() string {
	b := make([]byte, 8, 8)
	n, _ := hi.Read(b)
	return fmt.Sprintf("% x", b[:n])
}

// return new hashindex whoses rep is as the source heashindex but with added byte(s)
func (hi hashIndexType) Append(b byte) hashIndexType{
	buf := make([]byte, 8, 8)
	n, _ := hi.Read(buf)
	buf[n]=b
	return NewHashIndexType(buf[:n+1])
}

// return new hashindex whoses rep is as the source heashindex but with removed byte(s)
func (hi hashIndexType) Truncate(c int) hashIndexType{
	buf := make([]byte, 8, 8)
	n, _ := hi.Read(buf)
	return NewHashIndexType(buf[:n-c])
}

<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"encoding/hex"
	"fmt"
	"os"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/directoryBlock"
	"github.com/FactomProject/factomd/common/factoid"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
)

// Database keys and key prefixes
var (
	fblockDBPrefix = []byte("FBlock")
)

type TXDatabaseOverlay struct {
	DBO databaseOverlay.Overlay

	// To indicate to sub processes to quit
	quit bool
}

func NewTXOverlay(db interfaces.IDatabase) *TXDatabaseOverlay {
	answer := new(TXDatabaseOverlay)
	answer.DBO.DB = db
	return answer
}

func NewTXMapDB() *TXDatabaseOverlay {
	return NewTXOverlay(new(mapdb.MapDB))
}

func NewTXLevelDB(ldbpath string) (*TXDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening transaction db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new transaction db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Transaction database started from: " + ldbpath)
	return NewTXOverlay(db), nil
}

func NewTXBoltDB(boltPath string) (*TXDatabaseOverlay, error) {
	fileInfo, err := os.Stat(boltPath)
	if err == nil {
		if fileInfo.IsDir() {
			return nil, fmt.Errorf("%s is not a Bolt databse file", boltPath)
		}
	}
	if err != nil && !os.IsNotExist(err) {
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet cache database file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewTXOverlay(db), nil
}

func (db *TXDatabaseOverlay) Close() error {
	db.quit = true
	return db.DBO.Close()
}

// GetAllTXs returns a list of all transactions in the history of Factom. A
// local database is used to cache the factoid blocks.
func (db *TXDatabaseOverlay) GetAllTXs() ([]interfaces.ITransaction, error) {
	// update the database and get the newest fblock
	_, err := db.Update()
	if err != nil {
		return nil, err
	}
	fblock, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return nil, err
	}
	if fblock == nil {
		return nil, fmt.Errorf("FBlock Chain has not finished syncing")
	}
	txs := make([]interfaces.ITransaction, 0)

	for {
		// get all of the txs from the block
		height := fblock.GetDatabaseHeight()
		for _, tx := range fblock.GetTransactions() {
			ins, err := tx.TotalInputs()
			if err != nil {
				return nil, err
			}
			outs, err := tx.TotalOutputs()
			if err != nil {
				return nil, err
			}

			if ins != 0 || outs != 0 {
				tx.SetBlockHeight(height)
				txs = append(txs, tx)
			}
		}

		if pre := fblock.GetPrevKeyMR().String(); pre != factom.ZeroHash {
			// get the previous block
			fblock, err = db.GetFBlock(pre)
			if err != nil {
				return nil, err
			} else if fblock == nil {
				return nil, fmt.Errorf("Missing fblock in database: %s", pre)
			}
		} else {
			break
		}
	}

	return txs, nil
}

// GetTX gets a transaction by the transaction id
func (db *TXDatabaseOverlay) GetTX(txid string) (interfaces.ITransaction, error) {
	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if tx.GetSigHash().String() == txid {
			return tx, nil
		}
	}

	return nil, fmt.Errorf("Transaction not found")
}

// GetTXAddress returns a list of all transactions in the history of Factom that
// include a specific address.
func (db *TXDatabaseOverlay) GetTXAddress(adr string) (
	[]interfaces.ITransaction, error) {
	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	if factom.AddressStringType(adr) == factom.FactoidPub {
		for _, tx := range txs {
			for _, in := range tx.GetInputs() {
				if primitives.ConvertFctAddressToUserStr(in.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
			for _, out := range tx.GetOutputs() {
				if primitives.ConvertFctAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else if factom.AddressStringType(adr) == factom.ECPub {
		for _, tx := range txs {
			for _, out := range tx.GetECOutputs() {
				if primitives.ConvertECAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else {
		return nil, fmt.Errorf("not a valid address")
	}

	return filtered, nil
}

func (db *TXDatabaseOverlay) GetTXRange(start, end int) (
	[]interfaces.ITransaction, error) {
	if start < 0 || end < 0 {
		return nil, fmt.Errorf("Range cannot have negative numbers")
	}
	s, e := uint32(start), uint32(end)

	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if s <= tx.GetBlockHeight() && tx.GetBlockHeight() <= e {
			filtered = append(filtered, tx)
		}
	}

	return filtered, nil
}

// GetFBlock retrives a Factoid Block from Factom
func (db *TXDatabaseOverlay) GetFBlock(keymr string) (interfaces.IFBlock, error) {
	h, err := primitives.NewShaHashFromStr(keymr)
	if err != nil {
		return nil, err
	}

	fBlock, err := db.DBO.FetchFBlock(h)
	if err != nil {
		return nil, err
	}
	return fBlock, nil
}

func (db *TXDatabaseOverlay) FetchNextFBlockHeight() (uint32, error) {
	block, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return 0, err
	}
	if block == nil {
		return 0, nil
	}
	return block.GetDBHeight() + 1, nil
}

func (db *TXDatabaseOverlay) InsertFBlockHead(fblock interfaces.IFBlock) error {
	return db.DBO.SaveFactoidBlockHead(fblock)
}

// update gets all fblocks written since the database was last updated, and
// returns the most recent fblock keymr.
func (db *TXDatabaseOverlay) Update() (string, error) {
	newestFBlock, err := fblockHead()
	if err != nil {
		return "", err
	}

	start, err := db.FetchNextFBlockHeight()
	if err != nil {
		return "", err
	}

	// Make sure we didn't switch networks
	genesis, err := db.DBO.FetchFBlockByHeight(0)
	if err != nil {
		return "", err
	}
	if genesis != nil {
		genesis2, err := getdblockbyheight(0)
		if err != nil {
			return "", err
		}

		var gensisFBlockKeyMr interfaces.IHash
		for _, e := range genesis2.GetDBEntries() {
			if e.GetChainID().String() == "000000000000000000000000000000000000000000000000000000000000000f" {
				gensisFBlockKeyMr = e.GetKeyMR()
				break
			}
		}

		if gensisFBlockKeyMr == nil {
			return "", fmt.Errorf("unable to fetch the genesis block via the api")
		}

		if !gensisFBlockKeyMr.IsSameAs(genesis.GetKeyMR()) {
			start = 0
		}
	}

	newestHeight := newestFBlock.GetDatabaseHeight()

	fmt.Println("DEBUG: checking tx cache")
	// If the newest block in the tx cashe has a greater height than the newest
	// fblock then clear the cashe and start from 0.
	if start >= newestHeight {
		fmt.Printf("DEBUG: tx cache has height of %d, higher than block height of %d\n", start, newestHeight)
		db.DBO.Clear(databaseOverlay.FACTOIDBLOCK)
		return newestFBlock.GetKeyMR().String(), nil
	}

	// If the latest block from the database is not available from the blockchain
	// then clear the cashe and start from 0.
	if f, err := getfblockbyheight(start); err != nil {
		fmt.Println("DEBUG: fblock was not found in the blockchain", err)
		db.DBO.Clear(databaseOverlay.FACTOIDBLOCK)
		return f.GetKeyMR().String(), err
	} else {
		fmt.Println("DEBUG: got fblock from factomd: ", f.GetKeyMR().String())
	}

	db.DBO.StartMultiBatch()
	for i := start; i <= newestHeight; i++ {
		if i%1000 == 0 {
			if newestHeight-start > 1000 {
				fmt.Printf("Fetching block %v/%v\n", i, newestHeight)
			}
		}
		fblock, err := getfblockbyheight(i)
		if err != nil {
			db.DBO.ExecuteMultiBatch()
			return "", err
		}
		db.DBO.ProcessFBlockMultiBatch(fblock)

		// Save to DB every 500 blocks
		if i%500 == 0 {
			db.DBO.ExecuteMultiBatch()
			db.DBO.StartMultiBatch()
		}

		// If the wallet is stopped, this process becomes hard to kill. Have it exit
		if db.quit {
			break
		}
	}

	if !db.quit {
		fmt.Printf("Fetching block %v/%v\n", newestHeight, newestHeight)
	}

	// Save the remaining blocks
	if err = db.DBO.ExecuteMultiBatch(); err != nil {
		return "", err
	}

	return newestFBlock.GetKeyMR().String(), nil
}

// fblockHead gets the most recent fblock.
func fblockHead() (interfaces.IFBlock, error) {
	fblockID := "000000000000000000000000000000000000000000000000000000000000000f"

	dbhead, err := factom.GetDBlockHead()
	if err != nil {
		return nil, err
	}
	dblock, err := factom.GetDBlock(dbhead)
	if err != nil {
		return nil, err
	}

	var fblockmr string
	for _, eblock := range dblock.EntryBlockList {
		if eblock.ChainID == fblockID {
			fblockmr = eblock.KeyMR
		}
	}
	if fblockmr == "" {
		return nil, err
	}

	return getfblock(fblockmr)
}

func getfblock(keymr string) (interfaces.IFBlock, error) {
	p, err := factom.GetRaw(keymr)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(p)
}

func getfblockbyheight(height uint32) (interfaces.IFBlock, error) {
	p, err := factom.GetFBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(h)
}

func getdblockbyheight(height uint32) (interfaces.IDirectoryBlock, error) {
	p, err := factom.GetDBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return directoryBlock.UnmarshalDBlock(h)
}
<commit_msg>removed DEBUG statements<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"encoding/hex"
	"fmt"
	"os"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/directoryBlock"
	"github.com/FactomProject/factomd/common/factoid"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
)

// Database keys and key prefixes
var (
	fblockDBPrefix = []byte("FBlock")
)

type TXDatabaseOverlay struct {
	DBO databaseOverlay.Overlay

	// To indicate to sub processes to quit
	quit bool
}

func NewTXOverlay(db interfaces.IDatabase) *TXDatabaseOverlay {
	answer := new(TXDatabaseOverlay)
	answer.DBO.DB = db
	return answer
}

func NewTXMapDB() *TXDatabaseOverlay {
	return NewTXOverlay(new(mapdb.MapDB))
}

func NewTXLevelDB(ldbpath string) (*TXDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening transaction db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new transaction db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Transaction database started from: " + ldbpath)
	return NewTXOverlay(db), nil
}

func NewTXBoltDB(boltPath string) (*TXDatabaseOverlay, error) {
	fileInfo, err := os.Stat(boltPath)
	if err == nil {
		if fileInfo.IsDir() {
			return nil, fmt.Errorf("%s is not a Bolt databse file", boltPath)
		}
	}
	if err != nil && !os.IsNotExist(err) {
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet cache database file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewTXOverlay(db), nil
}

func (db *TXDatabaseOverlay) Close() error {
	db.quit = true
	return db.DBO.Close()
}

// GetAllTXs returns a list of all transactions in the history of Factom. A
// local database is used to cache the factoid blocks.
func (db *TXDatabaseOverlay) GetAllTXs() ([]interfaces.ITransaction, error) {
	// update the database and get the newest fblock
	_, err := db.Update()
	if err != nil {
		return nil, err
	}
	fblock, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return nil, err
	}
	if fblock == nil {
		return nil, fmt.Errorf("FBlock Chain has not finished syncing")
	}
	txs := make([]interfaces.ITransaction, 0)

	for {
		// get all of the txs from the block
		height := fblock.GetDatabaseHeight()
		for _, tx := range fblock.GetTransactions() {
			ins, err := tx.TotalInputs()
			if err != nil {
				return nil, err
			}
			outs, err := tx.TotalOutputs()
			if err != nil {
				return nil, err
			}

			if ins != 0 || outs != 0 {
				tx.SetBlockHeight(height)
				txs = append(txs, tx)
			}
		}

		if pre := fblock.GetPrevKeyMR().String(); pre != factom.ZeroHash {
			// get the previous block
			fblock, err = db.GetFBlock(pre)
			if err != nil {
				return nil, err
			} else if fblock == nil {
				return nil, fmt.Errorf("Missing fblock in database: %s", pre)
			}
		} else {
			break
		}
	}

	return txs, nil
}

// GetTX gets a transaction by the transaction id
func (db *TXDatabaseOverlay) GetTX(txid string) (interfaces.ITransaction, error) {
	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if tx.GetSigHash().String() == txid {
			return tx, nil
		}
	}

	return nil, fmt.Errorf("Transaction not found")
}

// GetTXAddress returns a list of all transactions in the history of Factom that
// include a specific address.
func (db *TXDatabaseOverlay) GetTXAddress(adr string) (
	[]interfaces.ITransaction, error) {
	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	if factom.AddressStringType(adr) == factom.FactoidPub {
		for _, tx := range txs {
			for _, in := range tx.GetInputs() {
				if primitives.ConvertFctAddressToUserStr(in.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
			for _, out := range tx.GetOutputs() {
				if primitives.ConvertFctAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else if factom.AddressStringType(adr) == factom.ECPub {
		for _, tx := range txs {
			for _, out := range tx.GetECOutputs() {
				if primitives.ConvertECAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else {
		return nil, fmt.Errorf("not a valid address")
	}

	return filtered, nil
}

func (db *TXDatabaseOverlay) GetTXRange(start, end int) (
	[]interfaces.ITransaction, error) {
	if start < 0 || end < 0 {
		return nil, fmt.Errorf("Range cannot have negative numbers")
	}
	s, e := uint32(start), uint32(end)

	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if s <= tx.GetBlockHeight() && tx.GetBlockHeight() <= e {
			filtered = append(filtered, tx)
		}
	}

	return filtered, nil
}

// GetFBlock retrives a Factoid Block from Factom
func (db *TXDatabaseOverlay) GetFBlock(keymr string) (interfaces.IFBlock, error) {
	h, err := primitives.NewShaHashFromStr(keymr)
	if err != nil {
		return nil, err
	}

	fBlock, err := db.DBO.FetchFBlock(h)
	if err != nil {
		return nil, err
	}
	return fBlock, nil
}

func (db *TXDatabaseOverlay) FetchNextFBlockHeight() (uint32, error) {
	block, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return 0, err
	}
	if block == nil {
		return 0, nil
	}
	return block.GetDBHeight() + 1, nil
}

func (db *TXDatabaseOverlay) InsertFBlockHead(fblock interfaces.IFBlock) error {
	return db.DBO.SaveFactoidBlockHead(fblock)
}

// update gets all fblocks written since the database was last updated, and
// returns the most recent fblock keymr.
func (db *TXDatabaseOverlay) Update() (string, error) {
	newestFBlock, err := fblockHead()
	if err != nil {
		return "", err
	}

	start, err := db.FetchNextFBlockHeight()
	if err != nil {
		return "", err
	}

	// Make sure we didn't switch networks
	genesis, err := db.DBO.FetchFBlockByHeight(0)
	if err != nil {
		return "", err
	}
	if genesis != nil {
		genesis2, err := getdblockbyheight(0)
		if err != nil {
			return "", err
		}

		var gensisFBlockKeyMr interfaces.IHash
		for _, e := range genesis2.GetDBEntries() {
			if e.GetChainID().String() == "000000000000000000000000000000000000000000000000000000000000000f" {
				gensisFBlockKeyMr = e.GetKeyMR()
				break
			}
		}

		if gensisFBlockKeyMr == nil {
			return "", fmt.Errorf("unable to fetch the genesis block via the api")
		}

		if !gensisFBlockKeyMr.IsSameAs(genesis.GetKeyMR()) {
			start = 0
		}
	}

	newestHeight := newestFBlock.GetDatabaseHeight()

	// If the newest block in the tx cashe has a greater height than the newest
	// fblock then clear the cashe and start from 0.
	if start >= newestHeight {
		db.DBO.Clear(databaseOverlay.FACTOIDBLOCK)
		return newestFBlock.GetKeyMR().String(), nil
	}

	// If the latest block from the database is not available from the blockchain
	// then clear the cashe and start from 0.
	if f, err := getfblockbyheight(start); err != nil {
		db.DBO.Clear(databaseOverlay.FACTOIDBLOCK)
		return f.GetKeyMR().String(), err
	}

	db.DBO.StartMultiBatch()
	for i := start; i <= newestHeight; i++ {
		if i%1000 == 0 {
			if newestHeight-start > 1000 {
				fmt.Printf("Fetching block %v/%v\n", i, newestHeight)
			}
		}
		fblock, err := getfblockbyheight(i)
		if err != nil {
			db.DBO.ExecuteMultiBatch()
			return "", err
		}
		db.DBO.ProcessFBlockMultiBatch(fblock)

		// Save to DB every 500 blocks
		if i%500 == 0 {
			db.DBO.ExecuteMultiBatch()
			db.DBO.StartMultiBatch()
		}

		// If the wallet is stopped, this process becomes hard to kill. Have it exit
		if db.quit {
			break
		}
	}

	if !db.quit {
		fmt.Printf("Fetching block %v/%v\n", newestHeight, newestHeight)
	}

	// Save the remaining blocks
	if err = db.DBO.ExecuteMultiBatch(); err != nil {
		return "", err
	}

	return newestFBlock.GetKeyMR().String(), nil
}

// fblockHead gets the most recent fblock.
func fblockHead() (interfaces.IFBlock, error) {
	fblockID := "000000000000000000000000000000000000000000000000000000000000000f"

	dbhead, err := factom.GetDBlockHead()
	if err != nil {
		return nil, err
	}
	dblock, err := factom.GetDBlock(dbhead)
	if err != nil {
		return nil, err
	}

	var fblockmr string
	for _, eblock := range dblock.EntryBlockList {
		if eblock.ChainID == fblockID {
			fblockmr = eblock.KeyMR
		}
	}
	if fblockmr == "" {
		return nil, err
	}

	return getfblock(fblockmr)
}

func getfblock(keymr string) (interfaces.IFBlock, error) {
	p, err := factom.GetRaw(keymr)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(p)
}

func getfblockbyheight(height uint32) (interfaces.IFBlock, error) {
	p, err := factom.GetFBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(h)
}

func getdblockbyheight(height uint32) (interfaces.IDirectoryBlock, error) {
	p, err := factom.GetDBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return directoryBlock.UnmarshalDBlock(h)
}
<|endoftext|>


<commit_before>package project

import (
	"golang.org/x/net/context"
)

// Ps list containers for the specified services.
func (p *Project) Ps(ctx context.Context, services ...string) (InfoSet, error) {
	allInfo := InfoSet{}
	for _, name := range p.ServiceConfigs.Keys() {
		service, err := p.CreateService(name)
		if err != nil {
			return nil, err
		}

		info, err := service.Info(ctx)
		if err != nil {
			return nil, err
		}

		allInfo = append(allInfo, info...)
	}
	return allInfo, nil
}
<commit_msg>Add filtering on project ps<commit_after>package project

import "golang.org/x/net/context"

// Ps list containers for the specified services.
func (p *Project) Ps(ctx context.Context, services ...string) (InfoSet, error) {
	allInfo := InfoSet{}

	if len(services) == 0 {
		services = p.ServiceConfigs.Keys()
	}

	for _, name := range services {

		service, err := p.CreateService(name)
		if err != nil {
			return nil, err
		}

		info, err := service.Info(ctx)
		if err != nil {
			return nil, err
		}

		allInfo = append(allInfo, info...)
	}
	return allInfo, nil
}
<|endoftext|>