<commit_before>package operatorstatus

import (
	"fmt"
	"os"
	"reflect"
	"time"

	configv1 "github.com/openshift/api/config/v1"
	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
	log "github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/discovery"

	olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
)

const (
	clusterOperatorOLM           = "operator-lifecycle-manager"
	clusterOperatorCatalogSource = "operator-lifecycle-manager-catalog"
	openshiftNamespace           = "openshift-operator-lifecycle-manager"
)

func MonitorClusterStatus(name string, syncCh <-chan error, stopCh <-chan struct{}, opClient operatorclient.ClientInterface, configClient configv1client.ConfigV1Interface, crClient versioned.Interface) {
	var (
		syncs              int
		successfulSyncs    int
		hasClusterOperator bool
	)
	go wait.Until(func() {
		// slow poll until we see a cluster operator API, which could be never
		if !hasClusterOperator {
			opStatusGV := schema.GroupVersion{
				Group:   "config.openshift.io",
				Version: "v1",
			}
			err := discovery.ServerSupportsVersion(opClient.KubernetesInterface().Discovery(), opStatusGV)
			if err != nil {
				log.Infof("ClusterOperator api not present, skipping update (%v)", err)
				time.Sleep(time.Minute)
				return
			}
			hasClusterOperator = true
		}

		// Sample the sync channel and see whether we're successfully retiring syncs as a
		// proxy for "working" (we can't know when we hit level, but we can at least verify
		// we are seeing some syncs succeeding). Once we observe at least one successful
		// sync we can begin reporting available and level.
		select {
		case err, ok := <-syncCh:
			if !ok {
				// syncCh should only close if the Run() loop exits
				time.Sleep(5 * time.Second)
				log.Fatalf("Status sync channel closed but process did not exit in time")
			}
			syncs++
			if err == nil {
				successfulSyncs++
			}
			// grab any other sync events that have accumulated
			for len(syncCh) > 0 {
				if err := <-syncCh; err == nil {
					successfulSyncs++
				}
				syncs++
			}
			// if we haven't yet accumulated enough syncs, wait longer
			// TODO: replace these magic numbers with a better measure of syncs across all queueInformers
			if successfulSyncs < 5 || syncs < 10 {
				log.Printf("Waiting to observe more successful syncs")
				return
			}
		}

		// create the cluster operator in an initial state if it does not exist
		existing, err := configClient.ClusterOperators().Get(name, metav1.GetOptions{})
		if k8serrors.IsNotFound(err) {
			log.Info("Existing operator status not found, creating")
			created, createErr := configClient.ClusterOperators().Create(&configv1.ClusterOperator{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Status: configv1.ClusterOperatorStatus{
					Conditions: []configv1.ClusterOperatorStatusCondition{
						{
							Type:               configv1.OperatorProgressing,
							Status:             configv1.ConditionTrue,
							Message:            fmt.Sprintf("Installing %s", olmversion.OLMVersion),
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorDegraded,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorAvailable,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorUpgradeable,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
					},
				},
			})
			if createErr != nil {
				log.Errorf("Failed to create cluster operator: %v\n", createErr)
				return
			}
			created.Status.RelatedObjects, err = relatedObjects(name, opClient, crClient)
			if err != nil {
				log.Errorf("Failed to get related objects: %v", err)
			}
			existing = created
			err = nil
		}
		if err != nil {
			log.Errorf("Unable to retrieve cluster operator: %v", err)
			return
		}

		// update the status with the appropriate state
		previousStatus := existing.Status.DeepCopy()
		previousOperatorVersion := func(vs []configv1.OperandVersion) string {
			for _, v := range vs {
				if v.Name == "operator" {
					return v.Version
				}
			}
			return ""
		}(previousStatus.Versions)
		targetOperatorVersion := os.Getenv("RELEASE_VERSION")
		switch {
		case successfulSyncs > 0:
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorDegraded,
				Status: configv1.ConditionFalse,
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorProgressing,
				Status:  configv1.ConditionFalse,
				Message: fmt.Sprintf("Deployed %s", olmversion.OLMVersion),
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorAvailable,
				Status: configv1.ConditionTrue,
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorUpgradeable,
				Status: configv1.ConditionTrue,
			})
			// we set the versions array when all the latest code is deployed and running - in this case,
			// the sync method is responsible for guaranteeing that happens before it returns nil
			if len(targetOperatorVersion) > 0 {
				existing.Status.Versions = []configv1.OperandVersion{
					{
						Name:    "operator",
						Version: targetOperatorVersion,
					},
					{
						Name:    "operator-lifecycle-manager",
						Version: olmversion.OLMVersion,
					},
				}
				if targetOperatorVersion != previousOperatorVersion {
					setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
						Type:    configv1.OperatorProgressing,
						Status:  configv1.ConditionTrue,
						Message: fmt.Sprintf("Deployed %s", olmversion.OLMVersion),
					})
				}
			} else {
				existing.Status.Versions = nil
			}
		default:
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorDegraded,
				Status:  configv1.ConditionTrue,
				Message: "Waiting for updates to take effect",
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorProgressing,
				Status:  configv1.ConditionFalse,
				Message: fmt.Sprintf("Waiting to see update %s succeed", olmversion.OLMVersion),
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorUpgradeable,
				Status:  configv1.ConditionFalse,
				Message: "Waiting for updates to take effect",
			})
			// TODO: use % errors within a window to report available
		}

		// update the status
		if !reflect.DeepEqual(previousStatus, &existing.Status) {
			if _, err := configClient.ClusterOperators().UpdateStatus(existing); err != nil {
				log.Errorf("Unable to update cluster operator status: %v", err)
			}
		}

		// if we've reported success, we can sleep longer, otherwise we want to keep watching for
		// successful
		if successfulSyncs > 0 {
			time.Sleep(5 * time.Minute)
		}

	}, 5*time.Second, stopCh)
}

func setOperatorStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, newCondition configv1.ClusterOperatorStatusCondition) {
	if conditions == nil {
		conditions = &[]configv1.ClusterOperatorStatusCondition{}
	}
	existingCondition := findOperatorStatusCondition(*conditions, newCondition.Type)
	if existingCondition == nil {
		newCondition.LastTransitionTime = metav1.NewTime(time.Now())
		*conditions = append(*conditions, newCondition)
		return
	}

	if existingCondition.Status != newCondition.Status {
		existingCondition.Status = newCondition.Status
		existingCondition.LastTransitionTime = metav1.NewTime(time.Now())
	}

	existingCondition.Reason = newCondition.Reason
	existingCondition.Message = newCondition.Message
}

func findOperatorStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition {
	for i := range conditions {
		if conditions[i].Type == conditionType {
			return &conditions[i]
		}
	}

	return nil
}

// relatedObjects returns RelatedObjects in the ClusterOperator.Status.
// RelatedObjects are consumed by https://github.com/openshift/must-gather
func relatedObjects(name string, opClient operatorclient.ClientInterface, crClient versioned.Interface) ([]configv1.ObjectReference, error) {
	var objectReferences []configv1.ObjectReference
	log.Infof("Adding related objects for %v", name)
	namespace := openshiftNamespace // hard-coded to constant

	switch name {
	case clusterOperatorOLM:
		csvList, err := crClient.OperatorsV1alpha1().ClusterServiceVersions(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		for _, csv := range csvList.Items {
			if csv.IsCopied() {
				continue
			}
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.ClusterServiceVersionKind,
				Namespace: csv.GetNamespace(),
				Name:      csv.GetName(),
			})
		}
	case clusterOperatorCatalogSource:
		subList, err := crClient.OperatorsV1alpha1().Subscriptions(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		installPlanList, err := crClient.OperatorsV1alpha1().InstallPlans(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		for _, sub := range subList.Items {
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.SubscriptionKind,
				Namespace: sub.GetNamespace(),
				Name:      sub.GetName(),
			})
		}
		for _, ip := range installPlanList.Items {
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.InstallPlanKind,
				Namespace: ip.GetNamespace(),
				Name:      ip.GetName(),
			})
		}
	}
	namespaces := configv1.ObjectReference{
		Group:    corev1.GroupName,
		Resource: "namespaces",
		Name:     namespace,
	}
	objectReferences = append(objectReferences, namespaces)
	return objectReferences, nil
}
<commit_msg>fix(clusteroperator): update status after initial creation<commit_after>package operatorstatus

import (
	"fmt"
	"os"
	"reflect"
	"time"

	configv1 "github.com/openshift/api/config/v1"
	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
	log "github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/diff"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/discovery"

	olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
)

const (
	clusterOperatorOLM           = "operator-lifecycle-manager"
	clusterOperatorCatalogSource = "operator-lifecycle-manager-catalog"
	openshiftNamespace           = "openshift-operator-lifecycle-manager"
)

func MonitorClusterStatus(name string, syncCh <-chan error, stopCh <-chan struct{}, opClient operatorclient.ClientInterface, configClient configv1client.ConfigV1Interface, crClient versioned.Interface) {
	var (
		syncs              int
		successfulSyncs    int
		hasClusterOperator bool
	)
	go wait.Until(func() {
		// slow poll until we see a cluster operator API, which could be never
		if !hasClusterOperator {
			opStatusGV := schema.GroupVersion{
				Group:   "config.openshift.io",
				Version: "v1",
			}
			err := discovery.ServerSupportsVersion(opClient.KubernetesInterface().Discovery(), opStatusGV)
			if err != nil {
				log.Infof("ClusterOperator api not present, skipping update (%v)", err)
				time.Sleep(time.Minute)
				return
			}
			hasClusterOperator = true
		}

		// Sample the sync channel and see whether we're successfully retiring syncs as a
		// proxy for "working" (we can't know when we hit level, but we can at least verify
		// we are seeing some syncs succeeding). Once we observe at least one successful
		// sync we can begin reporting available and level.
		select {
		case err, ok := <-syncCh:
			if !ok {
				// syncCh should only close if the Run() loop exits
				time.Sleep(5 * time.Second)
				log.Fatalf("Status sync channel closed but process did not exit in time")
			}
			syncs++
			if err == nil {
				successfulSyncs++
			}
			// grab any other sync events that have accumulated
			for len(syncCh) > 0 {
				if err := <-syncCh; err == nil {
					successfulSyncs++
				}
				syncs++
			}
			// if we haven't yet accumulated enough syncs, wait longer
			// TODO: replace these magic numbers with a better measure of syncs across all queueInformers
			if successfulSyncs < 5 || syncs < 10 {
				log.Printf("Waiting to observe more successful syncs")
				return
			}
		}

		// create the cluster operator in an initial state if it does not exist
		existing, err := configClient.ClusterOperators().Get(name, metav1.GetOptions{})
		if k8serrors.IsNotFound(err) {
			log.Info("Existing operator status not found, creating")
			created, createErr := configClient.ClusterOperators().Create(&configv1.ClusterOperator{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Status: configv1.ClusterOperatorStatus{
					Conditions: []configv1.ClusterOperatorStatusCondition{
						{
							Type:               configv1.OperatorProgressing,
							Status:             configv1.ConditionTrue,
							Message:            fmt.Sprintf("Installing %s", olmversion.OLMVersion),
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorDegraded,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorAvailable,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
						{
							Type:               configv1.OperatorUpgradeable,
							Status:             configv1.ConditionFalse,
							LastTransitionTime: metav1.Now(),
						},
					},
				},
			})
			if createErr != nil {
				log.Errorf("Failed to create cluster operator: %v\n", createErr)
				return
			}
			created.Status.RelatedObjects, err = relatedObjects(name, opClient, crClient)
			if err != nil {
				log.Errorf("Failed to get related objects: %v", err)
			}
			existing = created
			err = nil
		}
		if err != nil {
			log.Errorf("Unable to retrieve cluster operator: %v", err)
			return
		}

		// update the status with the appropriate state
		previousStatus := existing.Status.DeepCopy()
		previousOperatorVersion := func(vs []configv1.OperandVersion) string {
			for _, v := range vs {
				if v.Name == "operator" {
					return v.Version
				}
			}
			return ""
		}(previousStatus.Versions)
		targetOperatorVersion := os.Getenv("RELEASE_VERSION")
		switch {
		case successfulSyncs > 0:
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorDegraded,
				Status: configv1.ConditionFalse,
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorProgressing,
				Status:  configv1.ConditionFalse,
				Message: fmt.Sprintf("Deployed %s", olmversion.OLMVersion),
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorAvailable,
				Status: configv1.ConditionTrue,
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:   configv1.OperatorUpgradeable,
				Status: configv1.ConditionTrue,
			})
			// we set the versions array when all the latest code is deployed and running - in this case,
			// the sync method is responsible for guaranteeing that happens before it returns nil
			if len(targetOperatorVersion) > 0 {
				existing.Status.Versions = []configv1.OperandVersion{
					{
						Name:    "operator",
						Version: targetOperatorVersion,
					},
					{
						Name:    "operator-lifecycle-manager",
						Version: olmversion.OLMVersion,
					},
				}
				if targetOperatorVersion != previousOperatorVersion {
					setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
						Type:    configv1.OperatorProgressing,
						Status:  configv1.ConditionTrue,
						Message: fmt.Sprintf("Deployed %s", olmversion.OLMVersion),
					})
				}
			} else {
				existing.Status.Versions = nil
			}
		default:
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorDegraded,
				Status:  configv1.ConditionTrue,
				Message: "Waiting for updates to take effect",
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorProgressing,
				Status:  configv1.ConditionFalse,
				Message: fmt.Sprintf("Waiting to see update %s succeed", olmversion.OLMVersion),
			})
			setOperatorStatusCondition(&existing.Status.Conditions, configv1.ClusterOperatorStatusCondition{
				Type:    configv1.OperatorUpgradeable,
				Status:  configv1.ConditionFalse,
				Message: "Waiting for updates to take effect",
			})
			// TODO: use % errors within a window to report available
		}

		// always update the related objects in case changes have occurred
		existing.Status.RelatedObjects, err = relatedObjects(name, opClient, crClient)
		if err != nil {
			log.Errorf("Failed to get related objects: %v", err)
		}
		if !reflect.DeepEqual(previousStatus.RelatedObjects, existing.Status.RelatedObjects) {
			diffString := diff.ObjectDiff(previousStatus.RelatedObjects, existing.Status.RelatedObjects)
			log.Debugf("Update required for related objects: %v", diffString)
		}

		// update the status
		if !reflect.DeepEqual(previousStatus, &existing.Status) {
			if _, err := configClient.ClusterOperators().UpdateStatus(existing); err != nil {
				log.Errorf("Unable to update cluster operator status: %v", err)
			}
		}

		// if we've reported success, we can sleep longer, otherwise we want to keep watching for
		// successful
		if successfulSyncs > 0 {
			time.Sleep(5 * time.Minute)
		}

	}, 5*time.Second, stopCh)
}

func setOperatorStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, newCondition configv1.ClusterOperatorStatusCondition) {
	if conditions == nil {
		conditions = &[]configv1.ClusterOperatorStatusCondition{}
	}
	existingCondition := findOperatorStatusCondition(*conditions, newCondition.Type)
	if existingCondition == nil {
		newCondition.LastTransitionTime = metav1.NewTime(time.Now())
		*conditions = append(*conditions, newCondition)
		return
	}

	if existingCondition.Status != newCondition.Status {
		existingCondition.Status = newCondition.Status
		existingCondition.LastTransitionTime = metav1.NewTime(time.Now())
	}

	existingCondition.Reason = newCondition.Reason
	existingCondition.Message = newCondition.Message
}

func findOperatorStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition {
	for i := range conditions {
		if conditions[i].Type == conditionType {
			return &conditions[i]
		}
	}

	return nil
}

// relatedObjects returns RelatedObjects in the ClusterOperator.Status.
// RelatedObjects are consumed by https://github.com/openshift/must-gather
func relatedObjects(name string, opClient operatorclient.ClientInterface, crClient versioned.Interface) ([]configv1.ObjectReference, error) {
	var objectReferences []configv1.ObjectReference
	log.Infof("Adding related objects for %v", name)
	namespace := openshiftNamespace // hard-coded to constant

	switch name {
	case clusterOperatorOLM:
		csvList, err := crClient.OperatorsV1alpha1().ClusterServiceVersions(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		for _, csv := range csvList.Items {
			if csv.IsCopied() {
				continue
			}
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.ClusterServiceVersionKind,
				Namespace: csv.GetNamespace(),
				Name:      csv.GetName(),
			})
		}
	case clusterOperatorCatalogSource:
		subList, err := crClient.OperatorsV1alpha1().Subscriptions(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		installPlanList, err := crClient.OperatorsV1alpha1().InstallPlans(namespace).List(metav1.ListOptions{})
		if err != nil {
			return nil, err
		}

		for _, sub := range subList.Items {
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.SubscriptionKind,
				Namespace: sub.GetNamespace(),
				Name:      sub.GetName(),
			})
		}
		for _, ip := range installPlanList.Items {
			objectReferences = append(objectReferences, configv1.ObjectReference{
				Group:     olmv1alpha1.GroupName,
				Resource:  olmv1alpha1.InstallPlanKind,
				Namespace: ip.GetNamespace(),
				Name:      ip.GetName(),
			})
		}
	}
	namespaces := configv1.ObjectReference{
		Group:    corev1.GroupName,
		Resource: "namespaces",
		Name:     namespace,
	}
	objectReferences = append(objectReferences, namespaces)
	return objectReferences, nil
}
<|endoftext|>


<commit_before>package dos

func GetDiskFreeSpace(rootPathName string) (free uint64, total uint64, totalFree uint64, err error) {
	rc, err1 := getDiskFreeSpaceEx(rootPathName, &free, &total, &totalFree)
	if rc == 0 {
		err = err1
	}
	return
}
<commit_msg>Fix drivespace.go for golint<commit_after>package dos

// GetDiskFreeSpace retunrs disk information.
//   rootPathName - string like "C:"
func GetDiskFreeSpace(rootPathName string) (free uint64, total uint64, totalFree uint64, err error) {
	rc, err1 := getDiskFreeSpaceEx(rootPathName, &free, &total, &totalFree)
	if rc == 0 {
		err = err1
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stack

import (
	"fmt"
	"sync"
	"time"

	"gvisor.googlesource.com/gvisor/pkg/sleep"
	"gvisor.googlesource.com/gvisor/pkg/tcpip"
)

const linkAddrCacheSize = 512 // max cache entries

// linkAddrCache is a fixed-sized cache mapping IP addresses to link addresses.
//
// The entries are stored in a ring buffer, oldest entry replaced first.
//
// This struct is safe for concurrent use.
type linkAddrCache struct {
	// ageLimit is how long a cache entry is valid for.
	ageLimit time.Duration

	// resolutionTimeout is the amount of time to wait for a link request to
	// resolve an address.
	resolutionTimeout time.Duration

	// resolutionAttempts is the number of times an address is attempted to be
	// resolved before failing.
	resolutionAttempts int

	mu      sync.Mutex
	cache   map[tcpip.FullAddress]*linkAddrEntry
	next    int // array index of next available entry
	entries [linkAddrCacheSize]linkAddrEntry
}

// entryState controls the state of a single entry in the cache.
type entryState int

const (
	// incomplete means that there is an outstanding request to resolve the
	// address. This is the initial state.
	incomplete entryState = iota
	// ready means that the address has been resolved and can be used.
	ready
	// failed means that address resolution timed out and the address
	// could not be resolved.
	failed
	// expired means that the cache entry has expired and the address must be
	// resolved again.
	expired
)

// String implements Stringer.
func (s entryState) String() string {
	switch s {
	case incomplete:
		return "incomplete"
	case ready:
		return "ready"
	case failed:
		return "failed"
	case expired:
		return "expired"
	default:
		return fmt.Sprintf("invalid entryState: %d", s)
	}
}

// A linkAddrEntry is an entry in the linkAddrCache.
// This struct is thread-compatible.
type linkAddrEntry struct {
	addr       tcpip.FullAddress
	linkAddr   tcpip.LinkAddress
	expiration time.Time
	s          entryState
	resDone    bool

	// wakers is a set of waiters for address resolution result. Anytime
	// state transitions out of 'incomplete' these waiters are notified.
	wakers map[*sleep.Waker]struct{}

	cancel chan struct{}
	resCh  chan struct{}
}

func (e *linkAddrEntry) state() entryState {
	if e.s != expired && time.Now().After(e.expiration) {
		// Force the transition to ensure waiters are notified.
		e.changeState(expired)
	}
	return e.s
}

func (e *linkAddrEntry) changeState(ns entryState) {
	if e.s == ns {
		return
	}

	// Validate state transition.
	switch e.s {
	case incomplete:
		// All transitions are valid.
	case ready, failed:
		if ns != expired {
			panic(fmt.Sprintf("invalid state transition from %v to %v", e.s, ns))
		}
	case expired:
		// Terminal state.
		panic(fmt.Sprintf("invalid state transition from %v to %v", e.s, ns))
	default:
		panic(fmt.Sprintf("invalid state: %v", e.s))
	}

	// Notify whoever is waiting on address resolution when transitioning
	// out of 'incomplete'.
	if e.s == incomplete {
		for w := range e.wakers {
			w.Assert()
		}
		e.wakers = nil
	}
	e.s = ns
}

func (e *linkAddrEntry) addWaker(w *sleep.Waker) {
	e.wakers[w] = struct{}{}
}

func (e *linkAddrEntry) removeWaker(w *sleep.Waker) {
	delete(e.wakers, w)
}

// add adds a k -> v mapping to the cache.
func (c *linkAddrCache) add(k tcpip.FullAddress, v tcpip.LinkAddress) {
	c.mu.Lock()
	defer c.mu.Unlock()

	entry := c.cache[k]
	if entry != nil {
		s := entry.state()
		if s != expired && entry.linkAddr == v {
			// Disregard repeated calls.
			return
		}
		// Check if entry is waiting for address resolution.
		if s == incomplete {
			entry.linkAddr = v
		} else {
			// Otherwise create a new entry to replace it.
			entry = c.makeAndAddEntry(k, v)
		}
	} else {
		entry = c.makeAndAddEntry(k, v)
	}

	entry.changeState(ready)
}

// makeAndAddEntry is a helper function to create and add a new
// entry to the cache map and evict older entry as needed.
func (c *linkAddrCache) makeAndAddEntry(k tcpip.FullAddress, v tcpip.LinkAddress) *linkAddrEntry {
	// Take over the next entry.
	entry := &c.entries[c.next]
	if c.cache[entry.addr] == entry {
		delete(c.cache, entry.addr)
	}

	// Mark the soon-to-be-replaced entry as expired, just in case there is
	// someone waiting for address resolution on it.
	entry.changeState(expired)
	if entry.cancel != nil {
		if !entry.resDone {
			close(entry.resCh)
		}
		close(entry.cancel)
	}

	*entry = linkAddrEntry{
		addr:       k,
		linkAddr:   v,
		expiration: time.Now().Add(c.ageLimit),
		resDone:    false,
		wakers:     make(map[*sleep.Waker]struct{}),
		cancel:     make(chan struct{}, 1),
		resCh:      make(chan struct{}, 1),
	}

	c.cache[k] = entry
	c.next++
	if c.next == len(c.entries) {
		c.next = 0
	}
	return entry
}

// get reports any known link address for k.
func (c *linkAddrCache) get(k tcpip.FullAddress, linkRes LinkAddressResolver, localAddr tcpip.Address, linkEP LinkEndpoint, waker *sleep.Waker) (tcpip.LinkAddress, <-chan struct{}, *tcpip.Error) {
	if linkRes != nil {
		if addr, ok := linkRes.ResolveStaticAddress(k.Addr); ok {
			return addr, nil, nil
		}
	}

	c.mu.Lock()
	entry := c.cache[k]
	if entry == nil || entry.state() == expired {
		c.mu.Unlock()
		if linkRes == nil {
			return "", nil, tcpip.ErrNoLinkAddress
		}

		ch := c.startAddressResolution(k, linkRes, localAddr, linkEP, waker)
		return "", ch, tcpip.ErrWouldBlock
	}
	defer c.mu.Unlock()

	switch s := entry.state(); s {
	case expired:
		// It's possible that entry expired between state() call above and here
		// in that case it's safe to consider it ready.
		fallthrough
	case ready:
		return entry.linkAddr, nil, nil
	case failed:
		return "", nil, tcpip.ErrNoLinkAddress
	case incomplete:
		// Address resolution is still in progress.
		entry.addWaker(waker)
		return "", entry.resCh, tcpip.ErrWouldBlock
	default:
		panic(fmt.Sprintf("invalid cache entry state: %d", s))
	}
}

// removeWaker removes a waker previously added through get().
func (c *linkAddrCache) removeWaker(k tcpip.FullAddress, waker *sleep.Waker) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if entry := c.cache[k]; entry != nil {
		entry.removeWaker(waker)
	}
}

func (c *linkAddrCache) startAddressResolution(k tcpip.FullAddress, linkRes LinkAddressResolver, localAddr tcpip.Address, linkEP LinkEndpoint, waker *sleep.Waker) <-chan struct{} {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Look up again with lock held to ensure entry wasn't added by someone else.
	if e := c.cache[k]; e != nil && e.state() != expired {
		return nil
	}

	// Add 'incomplete' entry in the cache to mark that resolution is in progress.
	e := c.makeAndAddEntry(k, "")
	e.addWaker(waker)

	go func() { // S/R-SAFE: link non-savable; wakers dropped synchronously.
		for i := 0; ; i++ {
			// Send link request, then wait for the timeout limit and check
			// whether the request succeeded.
			linkRes.LinkAddressRequest(k.Addr, localAddr, linkEP)
			c.mu.Lock()
			cancel := e.cancel
			c.mu.Unlock()

			select {
			case <-time.After(c.resolutionTimeout):
				if stop := c.checkLinkRequest(k, i); stop {
					// If entry is evicted then resCh is already closed.
					c.mu.Lock()
					if e, ok := c.cache[k]; ok {
						if !e.resDone {
							e.resDone = true
							close(e.resCh)
						}
					}
					c.mu.Unlock()
					return
				}
			case <-cancel:
				return
			}
		}
	}()
	return e.resCh
}

// checkLinkRequest checks whether previous attempt to resolve address has succeeded
// and mark the entry accordingly, e.g. ready, failed, etc. Return true if request
// can stop, false if another request should be sent.
func (c *linkAddrCache) checkLinkRequest(k tcpip.FullAddress, attempt int) bool {
	c.mu.Lock()
	defer c.mu.Unlock()

	entry, ok := c.cache[k]
	if !ok {
		// Entry was evicted from the cache.
		return true
	}

	switch s := entry.state(); s {
	case ready, failed, expired:
		// Entry was made ready by resolver or failed. Either way we're done.
		return true
	case incomplete:
		if attempt+1 >= c.resolutionAttempts {
			// Max number of retries reached, mark entry as failed.
			entry.changeState(failed)
			return true
		}
		// No response yet, need to send another ARP request.
		return false
	default:
		panic(fmt.Sprintf("invalid cache entry state: %d", s))
	}
}

func newLinkAddrCache(ageLimit, resolutionTimeout time.Duration, resolutionAttempts int) *linkAddrCache {
	return &linkAddrCache{
		ageLimit:           ageLimit,
		resolutionTimeout:  resolutionTimeout,
		resolutionAttempts: resolutionAttempts,
		cache:              make(map[tcpip.FullAddress]*linkAddrEntry, linkAddrCacheSize),
	}
}
<commit_msg>Simplify channel management<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stack

import (
	"fmt"
	"sync"
	"time"

	"gvisor.googlesource.com/gvisor/pkg/sleep"
	"gvisor.googlesource.com/gvisor/pkg/tcpip"
)

const linkAddrCacheSize = 512 // max cache entries

// linkAddrCache is a fixed-sized cache mapping IP addresses to link addresses.
//
// The entries are stored in a ring buffer, oldest entry replaced first.
//
// This struct is safe for concurrent use.
type linkAddrCache struct {
	// ageLimit is how long a cache entry is valid for.
	ageLimit time.Duration

	// resolutionTimeout is the amount of time to wait for a link request to
	// resolve an address.
	resolutionTimeout time.Duration

	// resolutionAttempts is the number of times an address is attempted to be
	// resolved before failing.
	resolutionAttempts int

	mu      sync.Mutex
	cache   map[tcpip.FullAddress]*linkAddrEntry
	next    int // array index of next available entry
	entries [linkAddrCacheSize]linkAddrEntry
}

// entryState controls the state of a single entry in the cache.
type entryState int

const (
	// incomplete means that there is an outstanding request to resolve the
	// address. This is the initial state.
	incomplete entryState = iota
	// ready means that the address has been resolved and can be used.
	ready
	// failed means that address resolution timed out and the address
	// could not be resolved.
	failed
	// expired means that the cache entry has expired and the address must be
	// resolved again.
	expired
)

// String implements Stringer.
func (s entryState) String() string {
	switch s {
	case incomplete:
		return "incomplete"
	case ready:
		return "ready"
	case failed:
		return "failed"
	case expired:
		return "expired"
	default:
		return fmt.Sprintf("unknown(%d)", s)
	}
}

// A linkAddrEntry is an entry in the linkAddrCache.
// This struct is thread-compatible.
type linkAddrEntry struct {
	addr       tcpip.FullAddress
	linkAddr   tcpip.LinkAddress
	expiration time.Time
	s          entryState

	// wakers is a set of waiters for address resolution result. Anytime
	// state transitions out of 'incomplete' these waiters are notified.
	wakers map[*sleep.Waker]struct{}

	done chan struct{}
}

func (e *linkAddrEntry) state() entryState {
	if e.s != expired && time.Now().After(e.expiration) {
		// Force the transition to ensure waiters are notified.
		e.changeState(expired)
	}
	return e.s
}

func (e *linkAddrEntry) changeState(ns entryState) {
	if e.s == ns {
		return
	}

	// Validate state transition.
	switch e.s {
	case incomplete:
		// All transitions are valid.
	case ready, failed:
		if ns != expired {
			panic(fmt.Sprintf("invalid state transition from %s to %s", e.s, ns))
		}
	case expired:
		// Terminal state.
		panic(fmt.Sprintf("invalid state transition from %s to %s", e.s, ns))
	default:
		panic(fmt.Sprintf("invalid state: %s", e.s))
	}

	// Notify whoever is waiting on address resolution when transitioning
	// out of 'incomplete'.
	if e.s == incomplete {
		for w := range e.wakers {
			w.Assert()
		}
		e.wakers = nil
		if e.done != nil {
			close(e.done)
		}
	}
	e.s = ns
}

func (e *linkAddrEntry) addWaker(w *sleep.Waker) {
	e.wakers[w] = struct{}{}
}

func (e *linkAddrEntry) removeWaker(w *sleep.Waker) {
	delete(e.wakers, w)
}

// add adds a k -> v mapping to the cache.
func (c *linkAddrCache) add(k tcpip.FullAddress, v tcpip.LinkAddress) {
	c.mu.Lock()
	defer c.mu.Unlock()

	entry, ok := c.cache[k]
	if ok {
		s := entry.state()
		if s != expired && entry.linkAddr == v {
			// Disregard repeated calls.
			return
		}
		// Check if entry is waiting for address resolution.
		if s == incomplete {
			entry.linkAddr = v
		} else {
			// Otherwise create a new entry to replace it.
			entry = c.makeAndAddEntry(k, v)
		}
	} else {
		entry = c.makeAndAddEntry(k, v)
	}

	entry.changeState(ready)
}

// makeAndAddEntry is a helper function to create and add a new
// entry to the cache map and evict older entry as needed.
func (c *linkAddrCache) makeAndAddEntry(k tcpip.FullAddress, v tcpip.LinkAddress) *linkAddrEntry {
	// Take over the next entry.
	entry := &c.entries[c.next]
	if c.cache[entry.addr] == entry {
		delete(c.cache, entry.addr)
	}

	// Mark the soon-to-be-replaced entry as expired, just in case there is
	// someone waiting for address resolution on it.
	entry.changeState(expired)

	*entry = linkAddrEntry{
		addr:       k,
		linkAddr:   v,
		expiration: time.Now().Add(c.ageLimit),
		wakers:     make(map[*sleep.Waker]struct{}),
		done:       make(chan struct{}),
	}

	c.cache[k] = entry
	c.next = (c.next + 1) % len(c.entries)
	return entry
}

// get reports any known link address for k.
func (c *linkAddrCache) get(k tcpip.FullAddress, linkRes LinkAddressResolver, localAddr tcpip.Address, linkEP LinkEndpoint, waker *sleep.Waker) (tcpip.LinkAddress, <-chan struct{}, *tcpip.Error) {
	if linkRes != nil {
		if addr, ok := linkRes.ResolveStaticAddress(k.Addr); ok {
			return addr, nil, nil
		}
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	if entry, ok := c.cache[k]; ok {
		switch s := entry.state(); s {
		case expired:
		case ready:
			return entry.linkAddr, nil, nil
		case failed:
			return "", nil, tcpip.ErrNoLinkAddress
		case incomplete:
			// Address resolution is still in progress.
			entry.addWaker(waker)
			return "", entry.done, tcpip.ErrWouldBlock
		default:
			panic(fmt.Sprintf("invalid cache entry state: %s", s))
		}
	}

	if linkRes == nil {
		return "", nil, tcpip.ErrNoLinkAddress
	}

	// Add 'incomplete' entry in the cache to mark that resolution is in progress.
	e := c.makeAndAddEntry(k, "")
	e.addWaker(waker)

	go c.startAddressResolution(k, linkRes, localAddr, linkEP, e.done) // S/R-SAFE: link non-savable; wakers dropped synchronously.

	return "", e.done, tcpip.ErrWouldBlock
}

// removeWaker removes a waker previously added through get().
func (c *linkAddrCache) removeWaker(k tcpip.FullAddress, waker *sleep.Waker) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if entry, ok := c.cache[k]; ok {
		entry.removeWaker(waker)
	}
}

func (c *linkAddrCache) startAddressResolution(k tcpip.FullAddress, linkRes LinkAddressResolver, localAddr tcpip.Address, linkEP LinkEndpoint, done <-chan struct{}) {
	for i := 0; ; i++ {
		// Send link request, then wait for the timeout limit and check
		// whether the request succeeded.
		linkRes.LinkAddressRequest(k.Addr, localAddr, linkEP)

		select {
		case <-time.After(c.resolutionTimeout):
			if stop := c.checkLinkRequest(k, i); stop {
				return
			}
		case <-done:
			return
		}
	}
}

// checkLinkRequest checks whether previous attempt to resolve address has succeeded
// and mark the entry accordingly, e.g. ready, failed, etc. Return true if request
// can stop, false if another request should be sent.
func (c *linkAddrCache) checkLinkRequest(k tcpip.FullAddress, attempt int) bool {
	c.mu.Lock()
	defer c.mu.Unlock()

	entry, ok := c.cache[k]
	if !ok {
		// Entry was evicted from the cache.
		return true
	}

	switch s := entry.state(); s {
	case ready, failed, expired:
		// Entry was made ready by resolver or failed. Either way we're done.
		return true
	case incomplete:
		if attempt+1 >= c.resolutionAttempts {
			// Max number of retries reached, mark entry as failed.
			entry.changeState(failed)
			return true
		}
		// No response yet, need to send another ARP request.
		return false
	default:
		panic(fmt.Sprintf("invalid cache entry state: %s", s))
	}
}

func newLinkAddrCache(ageLimit, resolutionTimeout time.Duration, resolutionAttempts int) *linkAddrCache {
	return &linkAddrCache{
		ageLimit:           ageLimit,
		resolutionTimeout:  resolutionTimeout,
		resolutionAttempts: resolutionAttempts,
		cache:              make(map[tcpip.FullAddress]*linkAddrEntry, linkAddrCacheSize),
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nonetdev

package collector

import (
	"bytes"
	"encoding/binary"
	"errors"
	"net"
	"regexp"
	"strconv"

	"github.com/prometheus/common/log"
	"golang.org/x/sys/unix"
)

func getNetDevStats(ignore *regexp.Regexp, accept *regexp.Regexp) (map[string]map[string]string, error) {
	netDev := map[string]map[string]string{}

	ifs, err := net.Interfaces()
	if err != nil {
		return nil, errors.New("net.Interfaces() failed")
	}

	for _, iface := range ifs {
		ifaceData, err := getIfaceData(iface.Index)
		if err != nil {
			log.Debugf("failed to load data for interface %q: %v", iface.Name, err)
			continue
		}

		if ignore.MatchString(iface.Name) {
			log.Debugf("Ignoring device: %s", iface.Name)
			continue
		}
		if accept != nil && !accept.MatchString(iface.Name) {
			log.Debugf("Ignoring device: %s", iface.Name)
			continue
		}

		devStats := map[string]string{}
		devStats["receive_packets"] = strconv.FormatUint(ifaceData.Data.Ipackets, 10)
		devStats["transmit_packets"] = strconv.FormatUint(ifaceData.Data.Opackets, 10)
		devStats["receive_errs"] = strconv.FormatUint(ifaceData.Data.Ierrors, 10)
		devStats["transmit_errs"] = strconv.FormatUint(ifaceData.Data.Oerrors, 10)
		devStats["receive_bytes"] = strconv.FormatUint(ifaceData.Data.Ibytes, 10)
		devStats["transmit_bytes"] = strconv.FormatUint(ifaceData.Data.Obytes, 10)
		devStats["receive_multicast"] = strconv.FormatUint(ifaceData.Data.Imcasts, 10)
		devStats["transmit_multicast"] = strconv.FormatUint(ifaceData.Data.Omcasts, 10)
		netDev[iface.Name] = devStats
	}

	return netDev, nil
}

func getIfaceData(index int) (*ifMsghdr2, error) {
	var data ifMsghdr2
	rawData, err := unix.SysctlRaw("net", unix.AF_ROUTE, 0, 0, unix.NET_RT_IFLIST2, index)
	if err != nil {
		return nil, err
	}
	err = binary.Read(bytes.NewReader(rawData), binary.LittleEndian, &data)
	return &data, err
}

type ifMsghdr2 struct {
	Msglen    uint16
	Version   uint8
	Type      uint8
	Addrs     int32
	Flags     int32
	Index     uint16
	_         [2]byte
	SndLen    int32
	SndMaxlen int32
	SndDrops  int32
	Timer     int32
	Data      ifData64
}

type ifData64 struct {
	Type       uint8
	Typelen    uint8
	Physical   uint8
	Addrlen    uint8
	Hdrlen     uint8
	Recvquota  uint8
	Xmitquota  uint8
	Unused1    uint8
	Mtu        uint32
	Metric     uint32
	Baudrate   uint64
	Ipackets   uint64
	Ierrors    uint64
	Opackets   uint64
	Oerrors    uint64
	Collisions uint64
	Ibytes     uint64
	Obytes     uint64
	Imcasts    uint64
	Omcasts    uint64
	Iqdrops    uint64
	Noproto    uint64
	Recvtiming uint32
	Xmittiming uint32
	Lastchange unix.Timeval32
}
<commit_msg>bugfix: avoid nil reference when ignore is nil (#1414)<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nonetdev

package collector

import (
	"bytes"
	"encoding/binary"
	"errors"
	"net"
	"regexp"
	"strconv"

	"github.com/prometheus/common/log"
	"golang.org/x/sys/unix"
)

func getNetDevStats(ignore *regexp.Regexp, accept *regexp.Regexp) (map[string]map[string]string, error) {
	netDev := map[string]map[string]string{}

	ifs, err := net.Interfaces()
	if err != nil {
		return nil, errors.New("net.Interfaces() failed")
	}

	for _, iface := range ifs {
		ifaceData, err := getIfaceData(iface.Index)
		if err != nil {
			log.Debugf("failed to load data for interface %q: %v", iface.Name, err)
			continue
		}

		if ignore != nil && ignore.MatchString(iface.Name) {
			log.Debugf("Ignoring device: %s", iface.Name)
			continue
		}
		if accept != nil && !accept.MatchString(iface.Name) {
			log.Debugf("Ignoring device: %s", iface.Name)
			continue
		}

		devStats := map[string]string{}
		devStats["receive_packets"] = strconv.FormatUint(ifaceData.Data.Ipackets, 10)
		devStats["transmit_packets"] = strconv.FormatUint(ifaceData.Data.Opackets, 10)
		devStats["receive_errs"] = strconv.FormatUint(ifaceData.Data.Ierrors, 10)
		devStats["transmit_errs"] = strconv.FormatUint(ifaceData.Data.Oerrors, 10)
		devStats["receive_bytes"] = strconv.FormatUint(ifaceData.Data.Ibytes, 10)
		devStats["transmit_bytes"] = strconv.FormatUint(ifaceData.Data.Obytes, 10)
		devStats["receive_multicast"] = strconv.FormatUint(ifaceData.Data.Imcasts, 10)
		devStats["transmit_multicast"] = strconv.FormatUint(ifaceData.Data.Omcasts, 10)
		netDev[iface.Name] = devStats
	}

	return netDev, nil
}

func getIfaceData(index int) (*ifMsghdr2, error) {
	var data ifMsghdr2
	rawData, err := unix.SysctlRaw("net", unix.AF_ROUTE, 0, 0, unix.NET_RT_IFLIST2, index)
	if err != nil {
		return nil, err
	}
	err = binary.Read(bytes.NewReader(rawData), binary.LittleEndian, &data)
	return &data, err
}

type ifMsghdr2 struct {
	Msglen    uint16
	Version   uint8
	Type      uint8
	Addrs     int32
	Flags     int32
	Index     uint16
	_         [2]byte
	SndLen    int32
	SndMaxlen int32
	SndDrops  int32
	Timer     int32
	Data      ifData64
}

type ifData64 struct {
	Type       uint8
	Typelen    uint8
	Physical   uint8
	Addrlen    uint8
	Hdrlen     uint8
	Recvquota  uint8
	Xmitquota  uint8
	Unused1    uint8
	Mtu        uint32
	Metric     uint32
	Baudrate   uint64
	Ipackets   uint64
	Ierrors    uint64
	Opackets   uint64
	Oerrors    uint64
	Collisions uint64
	Ibytes     uint64
	Obytes     uint64
	Imcasts    uint64
	Omcasts    uint64
	Iqdrops    uint64
	Noproto    uint64
	Recvtiming uint32
	Xmittiming uint32
	Lastchange unix.Timeval32
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/tsuru/action"
	rtesting "github.com/globocom/tsuru/router/testing"
	"github.com/globocom/tsuru/testing"
	"launchpad.net/gocheck"
)

func (s *S) TestCreateContainerName(c *gocheck.C) {
	c.Assert(createContainer.Name, gocheck.Equals, "create-container")
}

func (s *S) TestCreateContainerForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	client, err := dockerClient.NewClient(s.server.URL())
	c.Assert(err, gocheck.IsNil)
	images, err := client.ListImages(true)
	c.Assert(err, gocheck.IsNil)
	cmds := []string{"ps", "-ef"}
	context := action.FWContext{Params: []interface{}{images[0].ID, cmds}}
	r, err := createContainer.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont := r.(container)
	defer cont.remove()
	c.Assert(cont, gocheck.FitsTypeOf, container{})
}

func (s *S) TestCreateContainerBackward(c *gocheck.C) {
	cont := container{ID: "ble"}
	context := action.BWContext{Params: []interface{}{cont}}
	createContainer.Backward(context)
}

func (s *S) TestInsertContainerName(c *gocheck.C) {
	c.Assert(insertContainer.Name, gocheck.Equals, "insert-container")
}

func (s *S) TestInsertContainerForward(c *gocheck.C) {
	cont := container{ID: "someid"}
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := insertContainer.Forward(context)
	c.Assert(err, gocheck.IsNil)
	coll := s.conn.Collection(s.collName)
	defer coll.RemoveId(cont.ID)
	cont = r.(container)
	var retrieved container
	err = coll.FindId(cont.ID).One(&retrieved)
	c.Assert(retrieved.ID, gocheck.Equals, cont.ID)
	c.Assert(retrieved.Status, gocheck.Equals, "created")
	c.Assert(cont, gocheck.FitsTypeOf, container{})
}

func (s *S) TestInsertContainerBackward(c *gocheck.C) {
	cont := container{ID: "someid"}
	coll := s.conn.Collection(s.collName)
	err := coll.Insert(&cont)
	c.Assert(err, gocheck.IsNil)
	context := action.BWContext{Params: []interface{}{cont}}
	insertContainer.Backward(context)
	err = coll.FindId(cont.ID).One(&cont)
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "not found")
}

func (s *S) TestAddRouteName(c *gocheck.C) {
	c.Assert(addRoute.Name, gocheck.Equals, "add-route")
}

func (s *S) TestAddRouteForward(c *gocheck.C) {
	app := testing.NewFakeApp("myapp", "python", 1)
	rtesting.FakeRouter.AddBackend(app.GetName())
	defer rtesting.FakeRouter.RemoveBackend(app.GetName())
	cont := container{ID: "ble", AppName: app.GetName()}
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := addRoute.Forward(context)
	c.Assert(err, gocheck.IsNil)
	c.Assert(r, gocheck.IsNil)
	hasRoute := rtesting.FakeRouter.HasRoute(app.GetName(), cont.getAddress())
	c.Assert(hasRoute, gocheck.Equals, true)
}

func (s *S) TestSetIpName(c *gocheck.C) {
	c.Assert(setIp.Name, gocheck.Equals, "set-ip")
}

func (s *S) TestSetIpForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	conta, err := s.newContainer()
	c.Assert(err, gocheck.IsNil)
	defer rtesting.FakeRouter.RemoveBackend(conta.AppName)
	cont := *conta
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := setIp.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont = r.(container)
	c.Assert(cont, gocheck.FitsTypeOf, container{})
	c.Assert(cont.ip, gocheck.Not(gocheck.Equals), "")
}

func (s *S) TestSetHostPortName(c *gocheck.C) {
	c.Assert(setHostPort.Name, gocheck.Equals, "set-host-port")
}

func (s *S) TestSetHostPortForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	conta, err := s.newContainer()
	c.Assert(err, gocheck.IsNil)
	defer rtesting.FakeRouter.RemoveBackend(conta.AppName)
	cont := *conta
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := setHostPort.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont = r.(container)
	c.Assert(cont, gocheck.FitsTypeOf, container{})
	c.Assert(cont.HostPort, gocheck.Not(gocheck.Equals), "")
}
<commit_msg>docker: fixed setIp test.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/tsuru/action"
	rtesting "github.com/globocom/tsuru/router/testing"
	"github.com/globocom/tsuru/testing"
	"launchpad.net/gocheck"
)

func (s *S) TestCreateContainerName(c *gocheck.C) {
	c.Assert(createContainer.Name, gocheck.Equals, "create-container")
}

func (s *S) TestCreateContainerForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	client, err := dockerClient.NewClient(s.server.URL())
	c.Assert(err, gocheck.IsNil)
	images, err := client.ListImages(true)
	c.Assert(err, gocheck.IsNil)
	cmds := []string{"ps", "-ef"}
	context := action.FWContext{Params: []interface{}{images[0].ID, cmds}}
	r, err := createContainer.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont := r.(container)
	defer cont.remove()
	c.Assert(cont, gocheck.FitsTypeOf, container{})
}

func (s *S) TestCreateContainerBackward(c *gocheck.C) {
	cont := container{ID: "ble"}
	context := action.BWContext{Params: []interface{}{cont}}
	createContainer.Backward(context)
}

func (s *S) TestInsertContainerName(c *gocheck.C) {
	c.Assert(insertContainer.Name, gocheck.Equals, "insert-container")
}

func (s *S) TestInsertContainerForward(c *gocheck.C) {
	cont := container{ID: "someid"}
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := insertContainer.Forward(context)
	c.Assert(err, gocheck.IsNil)
	coll := s.conn.Collection(s.collName)
	defer coll.RemoveId(cont.ID)
	cont = r.(container)
	var retrieved container
	err = coll.FindId(cont.ID).One(&retrieved)
	c.Assert(retrieved.ID, gocheck.Equals, cont.ID)
	c.Assert(retrieved.Status, gocheck.Equals, "created")
	c.Assert(cont, gocheck.FitsTypeOf, container{})
}

func (s *S) TestInsertContainerBackward(c *gocheck.C) {
	cont := container{ID: "someid"}
	coll := s.conn.Collection(s.collName)
	err := coll.Insert(&cont)
	c.Assert(err, gocheck.IsNil)
	context := action.BWContext{Params: []interface{}{cont}}
	insertContainer.Backward(context)
	err = coll.FindId(cont.ID).One(&cont)
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, "not found")
}

func (s *S) TestAddRouteName(c *gocheck.C) {
	c.Assert(addRoute.Name, gocheck.Equals, "add-route")
}

func (s *S) TestAddRouteForward(c *gocheck.C) {
	app := testing.NewFakeApp("myapp", "python", 1)
	rtesting.FakeRouter.AddBackend(app.GetName())
	defer rtesting.FakeRouter.RemoveBackend(app.GetName())
	cont := container{ID: "ble", AppName: app.GetName()}
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := addRoute.Forward(context)
	c.Assert(err, gocheck.IsNil)
	c.Assert(r, gocheck.IsNil)
	hasRoute := rtesting.FakeRouter.HasRoute(app.GetName(), cont.getAddress())
	c.Assert(hasRoute, gocheck.Equals, true)
}

func (s *S) TestSetIpName(c *gocheck.C) {
	c.Assert(setIp.Name, gocheck.Equals, "set-ip")
}

func (s *S) TestSetIpForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	conta, err := s.newContainer()
	c.Assert(err, gocheck.IsNil)
	defer rtesting.FakeRouter.RemoveBackend(conta.AppName)
	cont := *conta
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := setIp.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont = r.(container)
	c.Assert(cont, gocheck.FitsTypeOf, container{})
	c.Assert(cont.IP, gocheck.Not(gocheck.Equals), "")
}

func (s *S) TestSetHostPortName(c *gocheck.C) {
	c.Assert(setHostPort.Name, gocheck.Equals, "set-host-port")
}

func (s *S) TestSetHostPortForward(c *gocheck.C) {
	err := s.newImage()
	c.Assert(err, gocheck.IsNil)
	conta, err := s.newContainer()
	c.Assert(err, gocheck.IsNil)
	defer rtesting.FakeRouter.RemoveBackend(conta.AppName)
	cont := *conta
	context := action.FWContext{Params: []interface{}{cont}}
	r, err := setHostPort.Forward(context)
	c.Assert(err, gocheck.IsNil)
	cont = r.(container)
	c.Assert(cont, gocheck.FitsTypeOf, container{})
	c.Assert(cont.HostPort, gocheck.Not(gocheck.Equals), "")
}
<|endoftext|>


<commit_before>package cli

import (
	"flag"
	"testing"

	rpcpb "github.com/google/shipshape/shipshape/proto/shipshape_rpc_proto"
)

var dockerTag = flag.String("shipshape_test_docker_tag", "", "the docker tag for the images to use for testing")

func countFailures(resp rpcpb.ShipshapeResponse) int {
	failures := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		failures += len(analyzeResp.Failure)
	}
	return failures
}

func countNotes(resp rpcpb.ShipshapeResponse) int {
	notes := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		notes += len(analyzeResp.Note)
	}
	return notes
}

func countCategoryNotes(resp rpcpb.ShipshapeResponse, category string) int {
	notes := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		for _, note := range analyzeResp.Note {
			if *note.Category == category {
				notes += 1
			}
		}
	}
	return notes
}

func TestBasic(t *testing.T) {
	options := Options{
		File:                "shipshape/cli/testdata/workspace1",
		ThirdPartyAnalyzers: []string{},
		Build:               "maven",
		TriggerCats:         []string{"PostMessage", "JSHint"},
		Dind:                false,
		Event:               "manual", // TODO: const
		Repo:                "gcr.io/shipshape_releases", // TODO: const
		StayUp:              true,
		Tag:                 *dockerTag,
		// TODO(rsk): current e2e test can be ru both with & without kythe.
		LocalKythe:          false,
	}
	var allResponses rpcpb.ShipshapeResponse
	options.HandleResponse = func(shipshapeResp *rpcpb.ShipshapeResponse, _ string) error {
		allResponses.AnalyzeResponse = append(allResponses.AnalyzeResponse, shipshapeResp.AnalyzeResponse...)
		return nil
	}

	returnedNotesCount, err := New(options).Run()
	if err != nil {
		t.Fatal(err)
	}
	testName := "TestBasic"

	if got, want := countFailures(allResponses), 0; got != want {
		t.Errorf("%v: Wrong number of failures; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if countedNotes := countNotes(allResponses); returnedNotesCount != countedNotes {
		t.Errorf("%v: Inconsistent note count: returned %v, counted %v (proto data: %v", testName, returnedNotesCount, countedNotes, allResponses)
	}
	if got, want := returnedNotesCount, 10; got != want {
		t.Errorf("%v: Wrong number of notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "JSHint"), 8; got != want {
		t.Errorf("%v: Wrong number of JSHint notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "PostMessage"), 2; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
}

func TestExternalAnalyzers(t *testing.T) {
	// Replaces part of the e2e test
	// Create a fake maven project with android failures

	// Run CLI using a .shipshape file
}

func TestBuiltInAnalyzersPreBuild(t *testing.T) {
	// Replaces part of the e2e test
	// Test PostMessage and Go, with no kythe build
	options := Options{
		File:                "shipshape/cli/testdata/workspace1",
		ThirdPartyAnalyzers: []string{},
		Build:               "",
		TriggerCats:         []string{"PostMessage", "JSHint", "go vet", "PyLint"},
		Dind:                false,
		Event:               "manual", // TODO: const
		Repo:                "gcr.io/shipshape_releases", // TODO: const
		StayUp:              true,
		// TODO(rsk): current e2e test require tag to be specified on
		// the command line. Furthermore, if the tag is "local" it builds
		// containers and tags them.
		Tag:                 "prod",
		// TODO(rsk): current e2e test can be ru both with & without kythe.
		LocalKythe:          false,
	}
	var allResponses rpcpb.ShipshapeResponse
	options.HandleResponse = func(shipshapeResp *rpcpb.ShipshapeResponse, _ string) error {
		allResponses.AnalyzeResponse = append(allResponses.AnalyzeResponse, shipshapeResp.AnalyzeResponse...)
		return nil
	}
	returnedNotesCount, err := New(options).Run()
	if err != nil {
		t.Fatal(err)
	}
	testName := "TestBuiltInAnalyzerPreBuild"

	if got, want := countFailures(allResponses), 0; got != want {
		t.Errorf("%v: Wrong number of failures; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if countedNotes := countNotes(allResponses); returnedNotesCount != countedNotes {
		t.Errorf("%v: Inconsistent note count: returned %v, counted %v (proto data: %v", testName, returnedNotesCount, countedNotes, allResponses)
	}
	if got, want := returnedNotesCount, 10; got != want {
		t.Errorf("%v: Wrong number of notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "PostMessage"), 2; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "JSHint"), 8; got != want {
		t.Errorf("%v: Wrong number of JSHint notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "go vet"), 0; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "PyLint"), 0; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
}

func TestBuiltInAnalyzersPostBuild(t *testing.T) {
	// Replaces part of the e2e test
	// Test with a kythe maven build
	// PostMessage and ErrorProne
}

func TestStreamsMode(t *testing.T) {
	// Test whether it works in streams mode
	// Before creating this, ensure that streams mode
	// is actually still something we need to support.
}

func TestChangingDirectories(t *testing.T) {
	// Replaces the changedir test
	// Make sure to test changing down, changing up, running on the same directory, running on a single file in the same directory, and changing to a sibling
}

func dumpLogs() {

}

func checkOutput(category string, numResults int) {

}
<commit_msg>Updates test<commit_after>package cli

import (
	"flag"
	"testing"

	rpcpb "github.com/google/shipshape/shipshape/proto/shipshape_rpc_proto"
)

var dockerTag = flag.String("shipshape_test_docker_tag", "", "the docker tag for the images to use for testing")

func countFailures(resp rpcpb.ShipshapeResponse) int {
	failures := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		failures += len(analyzeResp.Failure)
	}
	return failures
}

func countNotes(resp rpcpb.ShipshapeResponse) int {
	notes := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		notes += len(analyzeResp.Note)
	}
	return notes
}

func countCategoryNotes(resp rpcpb.ShipshapeResponse, category string) int {
	notes := 0
	for _, analyzeResp := range resp.AnalyzeResponse {
		for _, note := range analyzeResp.Note {
			if *note.Category == category {
				notes += 1
			}
		}
	}
	return notes
}

func TestExternalAnalyzers(t *testing.T) {
	// Replaces part of the e2e test
	// Create a fake maven project with android failures

	// Run CLI using a .shipshape file
}

func TestBuiltInAnalyzersPreBuild(t *testing.T) {
	options := Options{
		File:                "shipshape/cli/testdata/workspace1",
		ThirdPartyAnalyzers: []string{},
		Build:               "",
		TriggerCats:         []string{"PostMessage", "JSHint", "go vet", "PyLint"},
		Dind:                false,
		Event:               "manual", // TODO: const
		Repo:                "gcr.io/shipshape_releases", // TODO: const
		StayUp:              true,
		Tag:                 *dockerTag,
		// TODO(rsk): current e2e test can be run both with & without kythe.
		LocalKythe:          false,
	}
	var allResponses rpcpb.ShipshapeResponse
	options.HandleResponse = func(shipshapeResp *rpcpb.ShipshapeResponse, _ string) error {
		allResponses.AnalyzeResponse = append(allResponses.AnalyzeResponse, shipshapeResp.AnalyzeResponse...)
		return nil
	}
	returnedNotesCount, err := New(options).Run()
	if err != nil {
		t.Fatal(err)
	}
	testName := "TestBuiltInAnalyzerPreBuild"

	if got, want := countFailures(allResponses), 0; got != want {
		t.Errorf("%v: Wrong number of failures; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if countedNotes := countNotes(allResponses); returnedNotesCount != countedNotes {
		t.Errorf("%v: Inconsistent note count: returned %v, counted %v (proto data: %v", testName, returnedNotesCount, countedNotes, allResponses)
	}
	if got, want := returnedNotesCount, 10; got != want {
		t.Errorf("%v: Wrong number of notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "PostMessage"), 2; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "JSHint"), 8; got != want {
		t.Errorf("%v: Wrong number of JSHint notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "go vet"), 0; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
	if got, want := countCategoryNotes(allResponses, "PyLint"), 0; got != want {
		t.Errorf("%v: Wrong number of PostMessage notes; got %v, want %v (proto data: %v)", testName, got, want, allResponses)
	}
}

func TestBuiltInAnalyzersPostBuild(t *testing.T) {
	// Replaces part of the e2e test
	// Test with a kythe maven build
	// PostMessage and ErrorProne
}

func TestStreamsMode(t *testing.T) {
	// Test whether it works in streams mode
	// Before creating this, ensure that streams mode
	// is actually still something we need to support.
}

func TestChangingDirectories(t *testing.T) {
	// Replaces the changedir test
	// Make sure to test changing down, changing up, running on the same directory, running on a single file in the same directory, and changing to a sibling
}

func dumpLogs() {

}

func checkOutput(category string, numResults int) {

}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package appdir

import (
	"errors"
	"os/user"
)

func userHomeDir() (string, error) {
	curUser, err := user.Current()
	if err != nil {
		return "", errors.New("unable to determine your home dir")
	}
	return curUser.HomeDir, nil
}

func Cache() (string, error) {
	return cache()
}

func CacheList() ([]string, error) {
	return cacheList()
}

func Config() (string, error) {
	return config()
}

func ConfigList() ([]string, error) {
	return configList()
}

func Data() (string, error) {
	return data()
}

func DataList() ([]string, error) {
	return dataList()
}
<commit_msg>Add docs<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package appdir

import (
	"errors"
	"os/user"
)

func userHomeDir() (string, error) {
	curUser, err := user.Current()
	if err != nil {
		return "", errors.New("unable to determine your home dir")
	}
	return curUser.HomeDir, nil
}

// Cache returns the base cache directory and an error, if any.
func Cache() (string, error) {
	return cache()
}

// CacheList returns the base cache directory list and an error, if any. The
// list is ordered by priority and the first element, if present, is the same
// directory returned by Cache.
func CacheList() ([]string, error) {
	return cacheList()
}

// Config returns the base config directory and an error, if any.
func Config() (string, error) {
	return config()
}

// ConfigList returns the base config directory list and an error, if any. The
// list is ordered by priority and the first element, if present, is the same
// directory returned by Config.
func ConfigList() ([]string, error) {
	return configList()
}

// Data returns the base data directory and an error, if any.
func Data() (string, error) {
	return data()
}

// DataList returns the base data directory list and an error, if any. The
// list is ordered by priority and the first element, if present, is the same
// directory returned by Data.
func DataList() ([]string, error) {
	return dataList()
}
<|endoftext|>


<commit_before>package metrics

import (
	"flag"
	"io"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/valyala/fasthttp"
)

var (
	httpClientRequestTimeout  = flag.Duration("httpClientRequestTimeout", time.Second*10, "Maximum time to wait for http response")
	httpClientKeepAlivePeriod = flag.Duration("httpClientKeepAlivePeriod", time.Second*5, "Interval for sending keep-alive messages on keepalive connections. Zero disables keep-alive messages")
	httpClientReadBufferSize  = flag.Int("httpClientReadBufferSize", 8*1024, "Per-connection read buffer size for httpclient")
	httpClientWriteBufferSize = flag.Int("httpClientWriteBufferSize", 8*1024, "Per-connection write buffer size for httpclient")
)

const jobCapacity = 10000
const maxConnDuration = time.Second

type Client struct {
	*fasthttp.HostClient
	wg sync.WaitGroup

	sync.Mutex
	Jobsch  chan struct{}
	workers int
}

var (
	request *fasthttp.Request
	t       time.Duration
)

func New(r *fasthttp.Request, timeout time.Duration) *Client {
	registerMetrics()
	request = r
	t = timeout

	addr, isTLS := acquireAddr(request)
	return &Client{
		Jobsch: make(chan struct{}, jobCapacity),
		HostClient: &fasthttp.HostClient{
			Addr:            addr,
			IsTLS:           isTLS,
			Dial:            dial,
			MaxConnDuration: maxConnDuration,
		},
	}
}

func (c *Client) Amount() int {
	c.Lock()
	defer c.Unlock()

	return c.workers
}

func (c *Client) Overflow() int {
	c.Lock()
	defer c.Unlock()

	return len(c.Jobsch)
}

func drainChan(ch chan struct{}) {
	for {
		select {
		case <-ch:
			continue
		default:
			return
		}
	}
}

func (c *Client) Flush() {
	drainChan(c.Jobsch)
	close(c.Jobsch)

	c.wg.Wait()
	flushMetrics()
	c.workers = 0
	c.Jobsch = make(chan struct{}, jobCapacity)
}

func (c *Client) RunWorkers(n int) {
	for i := 0; i < n; i++ {
		c.wg.Add(1)
		go func() {
			c.Lock()
			c.workers++
			c.Unlock()

			c.run()
			c.wg.Done()
		}()
	}
}

func (c *Client) run() {
	var resp fasthttp.Response
	r := new(fasthttp.Request)
	request.CopyTo(r)
	for range c.Jobsch {
		s := time.Now()
		err := c.DoTimeout(r, &resp, t)
		if err != nil {
			if err == fasthttp.ErrTimeout {
				timeouts.Inc()
			}
			errors.Inc()
		} else {
			requestSuccess.Inc()
		}
		requestDuration.Observe(float64(time.Since(s).Seconds()))
		requestSum.Inc()
	}
}

type hostConn struct {
	net.Conn
	addr         string
	closed       uint32
	connOpen     prometheus.Gauge
	readError    prometheus.Counter
	writeError   prometheus.Counter
	bytesWritten prometheus.Counter
	bytesRead    prometheus.Counter
}

func dial(addr string) (net.Conn, error) {
	conn, err := fasthttp.DialTimeout(addr, *httpClientRequestTimeout)
	if err != nil {
		return nil, err
	}
	if err = setupTCPConn(conn); err != nil {
		connError.Inc()
		conn.Close()
		return nil, err
	}

	connOpen.Inc()
	return &hostConn{
		Conn:         conn,
		addr:         addr,
		connOpen:     connOpen,
		readError:    readError,
		writeError:   writeError,
		bytesWritten: bytesWritten,
		bytesRead:    bytesRead,
	}, nil
}

func setupTCPConn(conn net.Conn) error {
	c, ok := conn.(*net.TCPConn)
	if !ok {
		return nil
	}

	var err error
	if *httpClientReadBufferSize > 0 {
		if err = c.SetReadBuffer(*httpClientReadBufferSize); err != nil {
			return err
		}
	}
	if *httpClientWriteBufferSize > 0 {
		if err = c.SetWriteBuffer(*httpClientWriteBufferSize); err != nil {
			return err
		}
	}
	if *httpClientKeepAlivePeriod > 0 {
		if err = c.SetKeepAlive(true); err != nil {
			return err
		}
		if err = c.SetKeepAlivePeriod(*httpClientKeepAlivePeriod); err != nil {
			return err
		}
	}
	return nil
}

func (hc *hostConn) Close() error {
	if atomic.AddUint32(&hc.closed, 1) == 1 {
		hc.connOpen.Dec()
	}

	return hc.Conn.Close()
}

func (hc *hostConn) Write(p []byte) (int, error) {
	n, err := hc.Conn.Write(p)
	hc.bytesWritten.Add(float64(n))
	if err != nil {
		hc.writeError.Inc()
	}
	return n, err
}

func (hc *hostConn) Read(p []byte) (int, error) {
	n, err := hc.Conn.Read(p)
	hc.bytesRead.Add(float64(n))
	if err != nil && err != io.EOF {
		hc.readError.Inc()
	}
	return n, err
}

func acquireAddr(req *fasthttp.Request) (string, bool) {
	addr := string(req.URI().Host())
	if len(addr) == 0 {
		log.Fatalf("address cannot be empty")
	}
	isTLS := string(request.URI().Scheme()) == "https"
	tmp := strings.SplitN(addr, ":", 2)
	if len(tmp) != 2 {
		port := ":80"
		if isTLS {
			port = ":443"
		}
		return tmp[0] + port, isTLS
	}
	port := tmp[1]
	portInt, err := strconv.Atoi(port)
	if err != nil {
		log.Fatalf("cannot parse port %q of addr %q: %s", port, addr, err)
	}
	if portInt < 0 {
		log.Fatalf("upstreamHosts port %d cannot be negative: %q", portInt, addr)
	}

	return addr, isTLS
}
<commit_msg>fix hostClient conf<commit_after>package metrics

import (
	"flag"
	"io"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/valyala/fasthttp"
)

var (
	httpClientRequestTimeout  = flag.Duration("httpClientRequestTimeout", time.Second*10, "Maximum time to wait for http response")
	httpClientKeepAlivePeriod = flag.Duration("httpClientKeepAlivePeriod", time.Second*5, "Interval for sending keep-alive messages on keepalive connections. Zero disables keep-alive messages")
	httpClientReadBufferSize  = flag.Int("httpClientReadBufferSize", 8*1024, "Per-connection read buffer size for httpclient")
	httpClientWriteBufferSize = flag.Int("httpClientWriteBufferSize", 8*1024, "Per-connection write buffer size for httpclient")
)

const jobCapacity = 10000
const maxIdleConnDuration = time.Second
const maxConns = 1<<31 - 1

type Client struct {
	*fasthttp.HostClient
	wg sync.WaitGroup

	sync.Mutex
	Jobsch  chan struct{}
	workers int
}

var (
	request *fasthttp.Request
	t       time.Duration
)

func New(r *fasthttp.Request, timeout time.Duration) *Client {
	registerMetrics()
	request = r
	t = timeout

	addr, isTLS := acquireAddr(request)
	return &Client{
		Jobsch: make(chan struct{}, jobCapacity),
		HostClient: &fasthttp.HostClient{
			Addr:                addr,
			IsTLS:               isTLS,
			Dial:                dial,
			MaxIdleConnDuration: maxIdleConnDuration,
			MaxConns:            maxConns,
		},
	}
}

func (c *Client) Amount() int {
	c.Lock()
	defer c.Unlock()

	return c.workers
}

func (c *Client) Overflow() int {
	c.Lock()
	defer c.Unlock()

	return len(c.Jobsch)
}

func drainChan(ch chan struct{}) {
	for {
		select {
		case <-ch:
			continue
		default:
			return
		}
	}
}

func (c *Client) Flush() {
	drainChan(c.Jobsch)
	close(c.Jobsch)

	c.wg.Wait()
	flushMetrics()
	c.workers = 0
	c.Jobsch = make(chan struct{}, jobCapacity)
}

func (c *Client) RunWorkers(n int) {
	for i := 0; i < n; i++ {
		c.wg.Add(1)
		go func() {
			c.Lock()
			c.workers++
			c.Unlock()

			c.run()
			c.wg.Done()
		}()
	}
}

func (c *Client) run() {
	var resp fasthttp.Response
	r := new(fasthttp.Request)
	request.CopyTo(r)
	for range c.Jobsch {
		s := time.Now()
		err := c.DoTimeout(r, &resp, t)
		if err != nil {
			if err == fasthttp.ErrTimeout {
				timeouts.Inc()
			}
			errors.Inc()
		} else {
			requestSuccess.Inc()
		}
		requestDuration.Observe(float64(time.Since(s).Seconds()))
		requestSum.Inc()
	}
}

type hostConn struct {
	net.Conn
	addr         string
	closed       uint32
	connOpen     prometheus.Gauge
	readError    prometheus.Counter
	writeError   prometheus.Counter
	bytesWritten prometheus.Counter
	bytesRead    prometheus.Counter
}

func dial(addr string) (net.Conn, error) {
	conn, err := fasthttp.DialTimeout(addr, *httpClientRequestTimeout)
	if err != nil {
		return nil, err
	}
	if err = setupTCPConn(conn); err != nil {
		connError.Inc()
		conn.Close()
		return nil, err
	}

	connOpen.Inc()
	return &hostConn{
		Conn:         conn,
		addr:         addr,
		connOpen:     connOpen,
		readError:    readError,
		writeError:   writeError,
		bytesWritten: bytesWritten,
		bytesRead:    bytesRead,
	}, nil
}

func setupTCPConn(conn net.Conn) error {
	c, ok := conn.(*net.TCPConn)
	if !ok {
		return nil
	}

	var err error
	if *httpClientReadBufferSize > 0 {
		if err = c.SetReadBuffer(*httpClientReadBufferSize); err != nil {
			return err
		}
	}
	if *httpClientWriteBufferSize > 0 {
		if err = c.SetWriteBuffer(*httpClientWriteBufferSize); err != nil {
			return err
		}
	}
	if *httpClientKeepAlivePeriod > 0 {
		if err = c.SetKeepAlive(true); err != nil {
			return err
		}
		if err = c.SetKeepAlivePeriod(*httpClientKeepAlivePeriod); err != nil {
			return err
		}
	}
	return nil
}

func (hc *hostConn) Close() error {
	if atomic.AddUint32(&hc.closed, 1) == 1 {
		hc.connOpen.Dec()
	}

	return hc.Conn.Close()
}

func (hc *hostConn) Write(p []byte) (int, error) {
	n, err := hc.Conn.Write(p)
	hc.bytesWritten.Add(float64(n))
	if err != nil {
		hc.writeError.Inc()
	}
	return n, err
}

func (hc *hostConn) Read(p []byte) (int, error) {
	n, err := hc.Conn.Read(p)
	hc.bytesRead.Add(float64(n))
	if err != nil && err != io.EOF {
		hc.readError.Inc()
	}
	return n, err
}

func acquireAddr(req *fasthttp.Request) (string, bool) {
	addr := string(req.URI().Host())
	if len(addr) == 0 {
		log.Fatalf("address cannot be empty")
	}
	isTLS := string(request.URI().Scheme()) == "https"
	tmp := strings.SplitN(addr, ":", 2)
	if len(tmp) != 2 {
		port := ":80"
		if isTLS {
			port = ":443"
		}
		return tmp[0] + port, isTLS
	}
	port := tmp[1]
	portInt, err := strconv.Atoi(port)
	if err != nil {
		log.Fatalf("cannot parse port %q of addr %q: %s", port, addr, err)
	}
	if portInt < 0 {
		log.Fatalf("upstreamHosts port %d cannot be negative: %q", portInt, addr)
	}

	return addr, isTLS
}
<|endoftext|>


<commit_before>package qtypes_metrics

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"time"
	"github.com/qframe/types/messages"
)

// The different types of metrics that are supported
const (
	Gauge             = "gauge"
	Counter           = "counter"
	CumulativeCounter = "cumcounter"
)

// Metric type holds all the information for a single metric data
// point. Metrics are generated in collectors and passed to handlers.
type Metric struct {
	qtypes_messages.Base
	Name       string            `json:"name"`
	MetricType string            `json:"type"`
	Value      float64           `json:"value"`
	Dimensions map[string]string `json:"dimensions"`
	Buffered   bool              `json:"buffered"`
}

// New returns a new metric with name. Default metric type is "gauge"
// and timestamp is set to now. Value is initialized to 0.0.
func New(source, name string) Metric {
	return Metric{
		Base: 		qtypes_messages.NewBase(source),
		Name:       sanitizeString(name),
		MetricType: Gauge,
		Value:      0.0,
		Dimensions: make(map[string]string),
		Buffered:   false,
	}
}

// NewExt provides a more controled creation
func NewExt(source, name string, metricTyp string, val float64, dimensions map[string]string, t time.Time, buffered bool) Metric {
	m := Metric{
		Base: 		qtypes_messages.NewTimedBase(source, t),
		Name:       sanitizeString(name),
		MetricType: metricTyp,
		Value:      val,
		Dimensions: dimensions,
		Buffered:   buffered,
	}
	return m
}

func (m *Metric) GetDimensionList() string {
	res := []string{}
	for k, v := range m.Dimensions {
		res = append(res, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(res, ",")
}

// Filter provides a struct that can filter a metric by Name (regex), type, dimension (subset of Dimensions)
type Filter struct {
	Name       string            `json:"name"`
	MetricType string            `json:"type"`
	Dimensions map[string]string `json:"dimensions"`
}

// ToJSON Transforms Filter to JSON
func (f *Filter) ToJSON() string {
	b, err := json.Marshal(f)
	if err != nil {
		fmt.Println(err)
		return "{}"
	}
	return string(b)
}

// NewFilter returns a Filter with compiled regex
func NewFilter(name string, t string, d map[string]string) Filter {
	return Filter{
		Name:       name,
		MetricType: t,
		Dimensions: d,
	}
}

// WithValue returns metric with value of type Gauge
func WithValue(source, name string, value float64) Metric {
	metric := New(source, name)
	metric.Value = value
	return metric
}

// EnableBuffering puts the metric into buffering handlers (e.g. ZmqBUF)
func (m *Metric) EnableBuffering() {
	m.Buffered = true
}

// DisableBuffering takes the metric out of buffering (e.g. ZmqBUF)
func (m *Metric) DisableBuffering() {
	m.Buffered = false
}

// SetTime to metric
func (m *Metric) SetTime(mtime time.Time) {
	m.Time = mtime
}

// AddDimension adds a new dimension to the Metric.
func (m *Metric) AddDimension(name, value string) {
	m.Dimensions[sanitizeString(name)] = sanitizeString(value)
}

// RemoveDimension removes a dimension from the Metric.
func (m *Metric) RemoveDimension(name string) {
	delete(m.Dimensions, name)
}

// AddDimensions adds multiple new dimensions to the Metric.
func (m *Metric) AddDimensions(dimensions map[string]string) {
	for k, v := range dimensions {
		m.AddDimension(k, v)
	}
}

// GetDimensions returns the dimensions of a metric merged with defaults. Defaults win.
func (m *Metric) GetDimensions(defaults map[string]string) (dimensions map[string]string) {
	dimensions = make(map[string]string)
	for name, value := range m.Dimensions {
		dimensions[name] = value
	}
	for name, value := range defaults {
		dimensions[name] = value
	}
	return dimensions
}

// GetDimensionValue returns the value of a dimension if it's set.
func (m *Metric) GetDimensionValue(dimension string) (value string, ok bool) {
	dimension = sanitizeString(dimension)
	value, ok = m.Dimensions[dimension]
	return
}

func (m *Metric) GetDimensionString() string {
	res := []string{}
	for k,v := range m.Dimensions {
		res = append(res, fmt.Sprintf("%s=%s", k,v))
	}
	return strings.Join(res, ",")
}
// AddToAll adds a map of dimensions to a list of metrics
func AddToAll(metrics *[]Metric, dims map[string]string) {
	for _, m := range *metrics {
		for key, value := range dims {
			m.AddDimension(key, value)
		}
	}
}

func sanitizeString(s string) string {
	s = strings.Replace(s, "=", "-", -1)
	s = strings.Replace(s, ":", "-", -1)
	return s
}

// ToJSON Transforms metric to JSON
func (m *Metric) ToJSON() string {
	b, err := json.Marshal(m)
	if err != nil {
		fmt.Println(err)
		return "{}"
	}
	return string(b)
}

// IsSubDim checks if the 2st map contains all items in the second
func (m *Metric) IsSubDim(other map[string]string) bool {
	for k, v := range other {
		val, ok := m.Dimensions[k]
		if !ok || v != val {
			return false
		}
	}
	return true
}

// IsFiltered checks if metrics is filtered with a given filter
func (m *Metric) IsFiltered(f Filter) bool {
	if !m.IsSubDim(f.Dimensions) {
		return false
	}
	if m.MetricType != f.MetricType {
		return false
	}
	// TODO: Precompile regex to speed up matching
	if !regexp.MustCompile(f.Name).MatchString(m.Name) {
		return false
	}

	return true
}

func (m *Metric) ToOpenTSDB() string {
	return fmt.Sprintf("%s %d %f %s", m.Name, m.Time.Unix(), m.Value, m.GetDimensionString())
}
<commit_msg>split tags in OpenTSDB with spaces<commit_after>package qtypes_metrics

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"time"
	"github.com/qframe/types/messages"
)

// The different types of metrics that are supported
const (
	Gauge             = "gauge"
	Counter           = "counter"
	CumulativeCounter = "cumcounter"
)

// Metric type holds all the information for a single metric data
// point. Metrics are generated in collectors and passed to handlers.
type Metric struct {
	qtypes_messages.Base
	Name       string            `json:"name"`
	MetricType string            `json:"type"`
	Value      float64           `json:"value"`
	Dimensions map[string]string `json:"dimensions"`
	Buffered   bool              `json:"buffered"`
}

// New returns a new metric with name. Default metric type is "gauge"
// and timestamp is set to now. Value is initialized to 0.0.
func New(source, name string) Metric {
	return Metric{
		Base: 		qtypes_messages.NewBase(source),
		Name:       sanitizeString(name),
		MetricType: Gauge,
		Value:      0.0,
		Dimensions: make(map[string]string),
		Buffered:   false,
	}
}

// NewExt provides a more controled creation
func NewExt(source, name string, metricTyp string, val float64, dimensions map[string]string, t time.Time, buffered bool) Metric {
	m := Metric{
		Base: 		qtypes_messages.NewTimedBase(source, t),
		Name:       sanitizeString(name),
		MetricType: metricTyp,
		Value:      val,
		Dimensions: dimensions,
		Buffered:   buffered,
	}
	return m
}

func (m *Metric) GetDimensionList() string {
	res := []string{}
	for k, v := range m.Dimensions {
		res = append(res, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(res, ",")
}

// Filter provides a struct that can filter a metric by Name (regex), type, dimension (subset of Dimensions)
type Filter struct {
	Name       string            `json:"name"`
	MetricType string            `json:"type"`
	Dimensions map[string]string `json:"dimensions"`
}

// ToJSON Transforms Filter to JSON
func (f *Filter) ToJSON() string {
	b, err := json.Marshal(f)
	if err != nil {
		fmt.Println(err)
		return "{}"
	}
	return string(b)
}

// NewFilter returns a Filter with compiled regex
func NewFilter(name string, t string, d map[string]string) Filter {
	return Filter{
		Name:       name,
		MetricType: t,
		Dimensions: d,
	}
}

// WithValue returns metric with value of type Gauge
func WithValue(source, name string, value float64) Metric {
	metric := New(source, name)
	metric.Value = value
	return metric
}

// EnableBuffering puts the metric into buffering handlers (e.g. ZmqBUF)
func (m *Metric) EnableBuffering() {
	m.Buffered = true
}

// DisableBuffering takes the metric out of buffering (e.g. ZmqBUF)
func (m *Metric) DisableBuffering() {
	m.Buffered = false
}

// SetTime to metric
func (m *Metric) SetTime(mtime time.Time) {
	m.Time = mtime
}

// AddDimension adds a new dimension to the Metric.
func (m *Metric) AddDimension(name, value string) {
	m.Dimensions[sanitizeString(name)] = sanitizeString(value)
}

// RemoveDimension removes a dimension from the Metric.
func (m *Metric) RemoveDimension(name string) {
	delete(m.Dimensions, name)
}

// AddDimensions adds multiple new dimensions to the Metric.
func (m *Metric) AddDimensions(dimensions map[string]string) {
	for k, v := range dimensions {
		m.AddDimension(k, v)
	}
}

// GetDimensions returns the dimensions of a metric merged with defaults. Defaults win.
func (m *Metric) GetDimensions(defaults map[string]string) (dimensions map[string]string) {
	dimensions = make(map[string]string)
	for name, value := range m.Dimensions {
		dimensions[name] = value
	}
	for name, value := range defaults {
		dimensions[name] = value
	}
	return dimensions
}

// GetDimensionValue returns the value of a dimension if it's set.
func (m *Metric) GetDimensionValue(dimension string) (value string, ok bool) {
	dimension = sanitizeString(dimension)
	value, ok = m.Dimensions[dimension]
	return
}

func (m *Metric) GetDimensionString() string {
	res := []string{}
	for k,v := range m.Dimensions {
		res = append(res, fmt.Sprintf("%s=%s", k,v))
	}
	return strings.Join(res, " ")
}
// AddToAll adds a map of dimensions to a list of metrics
func AddToAll(metrics *[]Metric, dims map[string]string) {
	for _, m := range *metrics {
		for key, value := range dims {
			m.AddDimension(key, value)
		}
	}
}

func sanitizeString(s string) string {
	s = strings.Replace(s, "=", "-", -1)
	s = strings.Replace(s, ":", "-", -1)
	return s
}

// ToJSON Transforms metric to JSON
func (m *Metric) ToJSON() string {
	b, err := json.Marshal(m)
	if err != nil {
		fmt.Println(err)
		return "{}"
	}
	return string(b)
}

// IsSubDim checks if the 2st map contains all items in the second
func (m *Metric) IsSubDim(other map[string]string) bool {
	for k, v := range other {
		val, ok := m.Dimensions[k]
		if !ok || v != val {
			return false
		}
	}
	return true
}

// IsFiltered checks if metrics is filtered with a given filter
func (m *Metric) IsFiltered(f Filter) bool {
	if !m.IsSubDim(f.Dimensions) {
		return false
	}
	if m.MetricType != f.MetricType {
		return false
	}
	// TODO: Precompile regex to speed up matching
	if !regexp.MustCompile(f.Name).MatchString(m.Name) {
		return false
	}

	return true
}

func (m *Metric) ToOpenTSDB() string {
	return m.ToOpenTSDBLine(false)
}

func (m *Metric) ToOpenTSDBLine(dropPut bool) string {
	res := []string{}
	if ! dropPut {
		res = append(res, "put")
	}
	res = append(res, m.Name, fmt.Sprintf("%d", m.Time.Unix()), fmt.Sprintf("%v", m.Value), m.GetDimensionString())
	return strings.Join(res, " ")
}

<|endoftext|>


<commit_before>package makeindex

import (
	//"github.com/indraniel/srasearch/sra"
	"github.com/indraniel/srasearch/utils"
	"bufio"
	//"encoding/json"
	"github.com/blevesearch/bleve"
	"io"
	"log"
	"strings"
	"time"
)

func CreateSearchIndex(input, output string) {
	batchSize := 100
	log.Println(
		"I'm in the CreateSearchIndex function!",
		"Please implement me!",
	)
	f, gzf := utils.OpenGZFile(input)
	defer utils.CloseGZFile(f, gzf)

	mapping := bleve.NewIndexMapping()
	index, err := bleve.New(output, mapping)
	if err != nil {
		log.Fatal("Trouble making a bleve index!")
	}

	reader := bufio.NewReader(gzf)
	line, err := reader.ReadString('\n')

	count := 0
	batch := bleve.NewBatch()
	startTime := time.Now()
	batchCount := 0
	for err == nil {
		elems := strings.SplitN(line, ",", 2)
		docId, json := elems[0], elems[1]
		json = strings.TrimRight(json, "\n")

		batch.Index(docId, []byte(json))
		batchCount++

		if batchCount >= batchSize {
			//log.Println("Submitting a batch to the index")
			err := index.Batch(batch)
			//log.Println("[DONE] Submitting a batch to the index")
			if err != nil {
				log.Fatal("Couldn't batch!")
				//				return err
			}
			batch = bleve.NewBatch()
			batchCount = 0
		}
		count++
		if count%1000 == 0 {
			indexDuration := time.Since(startTime)
			indexDurationSeconds := float64(indexDuration) / float64(time.Second)
			timePerDoc := float64(indexDuration) / float64(count)
			log.Printf("Indexed %d documents, in %.2fs (average %.2fms/doc)\n", count, indexDurationSeconds, timePerDoc/float64(time.Millisecond))
		}
		line, err = reader.ReadString('\n')
	}

	if err != io.EOF {
		log.Fatalln("[err] reading line ", count, "in", f.Name(), ":", err)
	}

	// flush the last batch
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			log.Fatal(err)
		}
	}

	indexDuration := time.Since(startTime)
	indexDurationSeconds := float64(indexDuration) / float64(time.Second)
	timePerDoc := float64(indexDuration) / float64(count)
	log.Printf("Indexed %d documents, in %.2fs (average %.2fms/doc)\n", count, indexDurationSeconds, timePerDoc/float64(time.Millisecond))
	log.Println("All Done")
}

//func buildIndexMapping() (*bleve.IndexMapping, error) {
//}

// v1
//		var si sra.SraItem
//		err := json.Unmarshal([]byte(line), &si)
//		if err != nil {
//			log.Fatal("Trouble unmarshal JSON to SraItem:", err)
//		}
//		//		log.Println("(", i, ")", "Indexing", si.Id)
//
//		err = index.Index(si.Id, si)
//		if err != nil {
//			log.Fatal("Trouble indexing sra item:", err)
//		}
//		i++
//		if i%25000 == 0 {
//			log.Println("Indexed", i, "entries")
//		}
<commit_msg>+ remove needless implementation warning message<commit_after>package makeindex

import (
	//"github.com/indraniel/srasearch/sra"
	"github.com/indraniel/srasearch/utils"
	"bufio"
	//"encoding/json"
	"github.com/blevesearch/bleve"
	"io"
	"log"
	"strings"
	"time"
)

func CreateSearchIndex(input, output string) {
	batchSize := 100
	f, gzf := utils.OpenGZFile(input)
	defer utils.CloseGZFile(f, gzf)

	mapping := bleve.NewIndexMapping()
	index, err := bleve.New(output, mapping)
	if err != nil {
		log.Fatal("Trouble making a bleve index!")
	}

	reader := bufio.NewReader(gzf)
	line, err := reader.ReadString('\n')

	count := 0
	batch := bleve.NewBatch()
	startTime := time.Now()
	batchCount := 0
	for err == nil {
		elems := strings.SplitN(line, ",", 2)
		docId, json := elems[0], elems[1]
		json = strings.TrimRight(json, "\n")

		batch.Index(docId, []byte(json))
		batchCount++

		if batchCount >= batchSize {
			//log.Println("Submitting a batch to the index")
			err := index.Batch(batch)
			//log.Println("[DONE] Submitting a batch to the index")
			if err != nil {
				log.Fatal("Couldn't batch!")
				//				return err
			}
			batch = bleve.NewBatch()
			batchCount = 0
		}
		count++
		if count%1000 == 0 {
			indexDuration := time.Since(startTime)
			indexDurationSeconds := float64(indexDuration) / float64(time.Second)
			timePerDoc := float64(indexDuration) / float64(count)
			log.Printf("Indexed %d documents, in %.2fs (average %.2fms/doc)\n", count, indexDurationSeconds, timePerDoc/float64(time.Millisecond))
		}
		line, err = reader.ReadString('\n')
	}

	if err != io.EOF {
		log.Fatalln("[err] reading line ", count, "in", f.Name(), ":", err)
	}

	// flush the last batch
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			log.Fatal(err)
		}
	}

	indexDuration := time.Since(startTime)
	indexDurationSeconds := float64(indexDuration) / float64(time.Second)
	timePerDoc := float64(indexDuration) / float64(count)
	log.Printf("Indexed %d documents, in %.2fs (average %.2fms/doc)\n", count, indexDurationSeconds, timePerDoc/float64(time.Millisecond))
	log.Println("All Done")
}

//func buildIndexMapping() (*bleve.IndexMapping, error) {
//}

// v1
//		var si sra.SraItem
//		err := json.Unmarshal([]byte(line), &si)
//		if err != nil {
//			log.Fatal("Trouble unmarshal JSON to SraItem:", err)
//		}
//		//		log.Println("(", i, ")", "Indexing", si.Id)
//
//		err = index.Index(si.Id, si)
//		if err != nil {
//			log.Fatal("Trouble indexing sra item:", err)
//		}
//		i++
//		if i%25000 == 0 {
//			log.Println("Indexed", i, "entries")
//		}
<|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	"time"

	"github.com/webx-top/echo"
)

func Log() echo.MiddlewareFunc {
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			req := c.Request()
			res := c.Response()
			logger := c.Logger()

			start := time.Now()
			if err := h.Handle(c); err != nil {
				c.Error(err)
			}

			remoteAddr := req.RealIP()
			stop := time.Now()
			method := req.Method()
			uri := req.URI()
			size := res.Size()
			code := res.Status()
			logger.Info(remoteAddr + " " + method + " " + uri + " " + fmt.Sprint(code) + " " + stop.Sub(start).String() + " " + fmt.Sprint(size))
			return nil
		})
	}
}
<commit_msg>update<commit_after>package middleware

import (
	"fmt"
	"time"

	"github.com/admpub/log"
	"github.com/webx-top/echo"
)

type VisitorInfo struct {
	RealIP       string
	Time         time.Time
	Elapsed      time.Duration
	URI          string
	Method       string
	UserAgent    string
	Referer      string
	RequestSize  int64
	ResponseSize int64
	ResponseCode int
}

func Log(recv ...func(*VisitorInfo)) echo.MiddlewareFunc {
	var logging func(*VisitorInfo)
	if len(recv) > 0 {
		logging = recv[0]
	}
	if logging == nil {
		logger := log.GetLogger(`http`)
		logging = func(v *VisitorInfo) {
			logger.Info(v.RealIP + " " + v.Method + " " + v.URI + " " + fmt.Sprint(v.ResponseCode) + " " + v.Elapsed.String() + " " + fmt.Sprint(v.ResponseSize))
		}
	}
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			req := c.Request()
			res := c.Response()
			info := &VisitorInfo{Time: time.Now()}
			if err := h.Handle(c); err != nil {
				c.Error(err)
			}
			info.RealIP = req.RealIP()
			info.UserAgent = req.UserAgent()
			info.Referer = req.Referer()
			info.RequestSize = req.Size()
			info.Elapsed = time.Now().Sub(info.Time)
			info.Method = req.Method()
			info.URI = req.URI()
			info.ResponseSize = res.Size()
			info.ResponseCode = res.Status()
			logging(info)
			return nil
		})
	}
}
<|endoftext|>