<commit_before>package punkt

// Interface used by the Tokenize function, can be extended to correct sentence
// boundaries that punkt misses.
type SentenceTokenizer interface {
	AnnotateTokens([]*Token, ...AnnotateTokens) []*Token
	Tokenize(string) []string
}

// A sentence tokenizer which uses an unsupervised algorithm to build a model
// for abbreviation words, collocations, and words that start sentences
// and then uses that model to find sentence boundaries.
type DefaultSentenceTokenizer struct {
	*Storage
	WordTokenizer
	PunctStrings
	Annotations []AnnotateTokens
}

// Sane defaults for the sentence tokenizer
func NewSentenceTokenizer(s *Storage) *DefaultSentenceTokenizer {
	lang := NewLanguage()
	word := NewWordTokenizer(lang)

	annotations := NewAnnotations(s, lang, word)

	tokenizer := &DefaultSentenceTokenizer{
		Storage:       s,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer

}

// Wraps around DST doing the work for customizing the tokenizer
func NewTokenizer(s *Storage, word WordTokenizer, lang PunctStrings) *DefaultSentenceTokenizer {
	annotations := NewAnnotations(s, lang, word)

	tokenizer := &DefaultSentenceTokenizer{
		Storage:       s,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer
}

/*
Given a set of tokens augmented with markers for line-start and
paragraph-start, returns an iterator through those tokens with full
annotation including predicted sentence breaks.
*/
func (s *DefaultSentenceTokenizer) AnnotateTokens(tokens []*Token, annotate ...AnnotateTokens) []*Token {
	for _, ann := range annotate {
		tokens = ann.Annotate(tokens)
	}

	return tokens
}

func (s *DefaultSentenceTokenizer) Tokenize(text string) []string {
	// Use the default word tokenizer but only grab the tokens that
	// relate to a sentence ending punctuation.  This means grab the word
	// before and after the punctuation.
	tokens := s.WordTokenizer.Tokenize(text, true)

	lastBreak := 0
	// Think of AnnotateTokens as a pipeline that we send our tokens through to process.
	annotatedTokens := s.AnnotateTokens(tokens, s.Annotations...)
	sentences := make([]string, 0, len(annotatedTokens))
	for _, token := range annotatedTokens {
		if !token.SentBreak {
			continue
		}

		sentence := text[lastBreak:token.Position]
		sentences = append(sentences, sentence)

		lastBreak = token.Position
	}

	sentences = append(sentences, text[lastBreak:])
	return sentences
}
<commit_msg>return text if no tokens can be found<commit_after>package punkt

// Interface used by the Tokenize function, can be extended to correct sentence
// boundaries that punkt misses.
type SentenceTokenizer interface {
	AnnotateTokens([]*Token, ...AnnotateTokens) []*Token
	Tokenize(string) []string
}

// A sentence tokenizer which uses an unsupervised algorithm to build a model
// for abbreviation words, collocations, and words that start sentences
// and then uses that model to find sentence boundaries.
type DefaultSentenceTokenizer struct {
	*Storage
	WordTokenizer
	PunctStrings
	Annotations []AnnotateTokens
}

// Sane defaults for the sentence tokenizer
func NewSentenceTokenizer(s *Storage) *DefaultSentenceTokenizer {
	lang := NewLanguage()
	word := NewWordTokenizer(lang)

	annotations := NewAnnotations(s, lang, word)

	tokenizer := &DefaultSentenceTokenizer{
		Storage:       s,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer

}

// Wraps around DST doing the work for customizing the tokenizer
func NewTokenizer(s *Storage, word WordTokenizer, lang PunctStrings) *DefaultSentenceTokenizer {
	annotations := NewAnnotations(s, lang, word)

	tokenizer := &DefaultSentenceTokenizer{
		Storage:       s,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer
}

/*
Given a set of tokens augmented with markers for line-start and
paragraph-start, returns an iterator through those tokens with full
annotation including predicted sentence breaks.
*/
func (s *DefaultSentenceTokenizer) AnnotateTokens(tokens []*Token, annotate ...AnnotateTokens) []*Token {
	for _, ann := range annotate {
		tokens = ann.Annotate(tokens)
	}

	return tokens
}

func (s *DefaultSentenceTokenizer) Tokenize(text string) []string {
	// Use the default word tokenizer but only grab the tokens that
	// relate to a sentence ending punctuation.  This means grab the word
	// before and after the punctuation.
	tokens := s.WordTokenizer.Tokenize(text, true)

	if len(tokens) == 0 {
		return []string{text}
	}

	lastBreak := 0
	// Think of AnnotateTokens as a pipeline that we send our tokens through to process.
	annotatedTokens := s.AnnotateTokens(tokens, s.Annotations...)
	sentences := make([]string, 0, len(annotatedTokens))
	for _, token := range annotatedTokens {
		if !token.SentBreak {
			continue
		}

		sentence := text[lastBreak:token.Position]
		sentences = append(sentences, sentence)

		lastBreak = token.Position
	}

	sentences = append(sentences, text[lastBreak:])
	return sentences
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcap"
	"fmt"
	"os"
	"time"
)

type SnifferSetup struct {
	pcapHandle     *pcap.Handle
	afpacketHandle *AfpacketHandle
	config         *tomlInterfaces

	DataSource gopacket.PacketDataSource
}

type tomlInterfaces struct {
	Device         string
	Devices        []string
	Type           string
	File           string
	With_vlans     bool
	Bpf_filter     string
	Snaplen        int
	Buffer_size_mb int
}

// Computes the block_size and the num_blocks in such a way that the
// allocated mmap buffer is close to but smaller than target_size_mb.
// The restriction is that the block_size must be divisible by both the
// frame size and page size.
func afpacketComputeSize(target_size_mb int, snaplen int, page_size int) (
	frame_size int, block_size int, num_blocks int, err error) {

	if snaplen > page_size {
		return 0, 0, 0, fmt.Errorf("Snaplen cannot be bigger than the page size")
	}

	frame_size = page_size / (page_size / snaplen)
	// 128 is the default from the gopacket library so just use that
	block_size = frame_size * 128
	num_blocks = (target_size_mb * 1024 * 1024) / block_size

	if num_blocks == 0 {
		return 0, 0, 0, fmt.Errorf("Buffer size too small")
	}

	return frame_size, block_size, num_blocks, nil
}

func CreateSniffer(config *tomlInterfaces, file *string) (*SnifferSetup, error) {
	var sniffer SnifferSetup
	var err error

	sniffer.config = config

	if file != nil && len(*file) > 0 {
		DEBUG("sniffer", "Reading from file: %s", *file)
		// we read file with the pcap provider
		sniffer.config.Type = "pcap"
		sniffer.config.File = *file
	}

	// set defaults
	if len(sniffer.config.Device) == 0 {
		sniffer.config.Device = "any"
	}

	if len(sniffer.config.Devices) == 0 {
		// 'devices' not set but 'device' is set. For backwards compatibility,
		// use the one configured device
		if len(sniffer.config.Device) > 0 {
			sniffer.config.Devices = []string{sniffer.config.Device}
		}
	}
	if sniffer.config.Snaplen == 0 {
		sniffer.config.Snaplen = 1514
	}

	DEBUG("sniffer", "Sniffer type: %s devices: %s", sniffer.config.Type, sniffer.config.Devices)

	if sniffer.config.Type == "pcap" || sniffer.config.Type == "" {
		if len(sniffer.config.File) > 0 {
			sniffer.pcapHandle, err = pcap.OpenOffline(sniffer.config.File)
			if err != nil {
				return nil, err
			}
		} else {
			if len(sniffer.config.Devices) > 1 {
				return nil, fmt.Errorf("Pcap sniffer only supports one device. You can use 'any' if you want")
			}
			sniffer.pcapHandle, err = pcap.OpenLive(
				sniffer.config.Devices[0],
				int32(sniffer.config.Snaplen),
				true,
				500*time.Millisecond)
			if err != nil {
				return nil, err
			}
		}

		sniffer.DataSource = gopacket.PacketDataSource(sniffer.pcapHandle)

	} else if sniffer.config.Type == "af_packet" {
		if sniffer.config.Buffer_size_mb == 0 {
			sniffer.config.Buffer_size_mb = 24
		}

		if len(sniffer.config.Devices) > 1 {
			return nil, fmt.Errorf("Afpacket sniffer only supports one device. You can use 'any' if you want")
		}

		frame_size, block_size, num_blocks, err := afpacketComputeSize(
			sniffer.config.Buffer_size_mb,
			sniffer.config.Snaplen,
			os.Getpagesize())
		if err != nil {
			return nil, err
		}

		sniffer.afpacketHandle, err = NewAfpacketHandle(
			sniffer.config.Devices[0],
			frame_size,
			block_size,
			num_blocks,
			500*time.Millisecond)
		if err != nil {
			return nil, err
		}

		sniffer.DataSource = gopacket.PacketDataSource(sniffer.afpacketHandle)

	} else {
		return nil, fmt.Errorf("Unknown sniffer type: %s", sniffer.config.Type)
	}

	return &sniffer, nil
}

func (sniffer *SnifferSetup) Reopen() error {
	var err error

	if sniffer.config.Type != "pcap" || sniffer.config.File == "" {
		return fmt.Errorf("Reopen is only possible for files")
	}

	sniffer.pcapHandle.Close()
	sniffer.pcapHandle, err = pcap.OpenOffline(sniffer.config.File)
	if err != nil {
		return err
	}

	sniffer.DataSource = gopacket.PacketDataSource(sniffer.pcapHandle)

	return nil
}

func (sniffer *SnifferSetup) Close() {
	switch sniffer.config.Type {
	case "pcap":
		sniffer.pcapHandle.Close()
	}
}

func (sniffer *SnifferSetup) Datalink() layers.LinkType {
	if sniffer.config.Type == "pcap" {
		return sniffer.pcapHandle.LinkType()
	}
	return layers.LinkTypeEthernet
}
<commit_msg>Default sniffer on linux is af_packet from now on<commit_after>package main

import (
	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcap"
	"fmt"
	"os"
	"runtime"
	"time"
)

type SnifferSetup struct {
	pcapHandle     *pcap.Handle
	afpacketHandle *AfpacketHandle
	config         *tomlInterfaces

	DataSource gopacket.PacketDataSource
}

type tomlInterfaces struct {
	Device         string
	Devices        []string
	Type           string
	File           string
	With_vlans     bool
	Bpf_filter     string
	Snaplen        int
	Buffer_size_mb int
}

// Computes the block_size and the num_blocks in such a way that the
// allocated mmap buffer is close to but smaller than target_size_mb.
// The restriction is that the block_size must be divisible by both the
// frame size and page size.
func afpacketComputeSize(target_size_mb int, snaplen int, page_size int) (
	frame_size int, block_size int, num_blocks int, err error) {

	if snaplen > page_size {
		return 0, 0, 0, fmt.Errorf("Snaplen cannot be bigger than the page size")
	}

	frame_size = page_size / (page_size / snaplen)
	// 128 is the default from the gopacket library so just use that
	block_size = frame_size * 128
	num_blocks = (target_size_mb * 1024 * 1024) / block_size

	if num_blocks == 0 {
		return 0, 0, 0, fmt.Errorf("Buffer size too small")
	}

	return frame_size, block_size, num_blocks, nil
}

func CreateSniffer(config *tomlInterfaces, file *string) (*SnifferSetup, error) {
	var sniffer SnifferSetup
	var err error

	sniffer.config = config

	if file != nil && len(*file) > 0 {
		DEBUG("sniffer", "Reading from file: %s", *file)
		// we read file with the pcap provider
		sniffer.config.Type = "pcap"
		sniffer.config.File = *file
	}

	// set defaults
	if len(sniffer.config.Device) == 0 {
		sniffer.config.Device = "any"
	}

	if len(sniffer.config.Devices) == 0 {
		// 'devices' not set but 'device' is set. For backwards compatibility,
		// use the one configured device
		if len(sniffer.config.Device) > 0 {
			sniffer.config.Devices = []string{sniffer.config.Device}
		}
	}
	if sniffer.config.Snaplen == 0 {
		sniffer.config.Snaplen = 1514
	}

	if sniffer.config.Type == "autodetect" || sniffer.config.Type == "" {
		sniffer.config.Type = sniffer.autodetectSnifferType(runtime.GOOS)
	}

	DEBUG("sniffer", "Sniffer type: %s devices: %s", sniffer.config.Type, sniffer.config.Devices)

	if sniffer.config.Type == "pcap" {
		if len(sniffer.config.File) > 0 {
			sniffer.pcapHandle, err = pcap.OpenOffline(sniffer.config.File)
			if err != nil {
				return nil, err
			}
		} else {
			if len(sniffer.config.Devices) > 1 {
				return nil, fmt.Errorf("Pcap sniffer only supports one device. You can use 'any' if you want")
			}
			sniffer.pcapHandle, err = pcap.OpenLive(
				sniffer.config.Devices[0],
				int32(sniffer.config.Snaplen),
				true,
				500*time.Millisecond)
			if err != nil {
				return nil, err
			}
		}

		sniffer.DataSource = gopacket.PacketDataSource(sniffer.pcapHandle)

	} else if sniffer.config.Type == "af_packet" {
		if sniffer.config.Buffer_size_mb == 0 {
			sniffer.config.Buffer_size_mb = 24
		}

		if len(sniffer.config.Devices) > 1 {
			return nil, fmt.Errorf("Afpacket sniffer only supports one device. You can use 'any' if you want")
		}

		frame_size, block_size, num_blocks, err := afpacketComputeSize(
			sniffer.config.Buffer_size_mb,
			sniffer.config.Snaplen,
			os.Getpagesize())
		if err != nil {
			return nil, err
		}

		sniffer.afpacketHandle, err = NewAfpacketHandle(
			sniffer.config.Devices[0],
			frame_size,
			block_size,
			num_blocks,
			500*time.Millisecond)
		if err != nil {
			return nil, err
		}

		sniffer.DataSource = gopacket.PacketDataSource(sniffer.afpacketHandle)

	} else {
		return nil, fmt.Errorf("Unknown sniffer type: %s", sniffer.config.Type)
	}

	return &sniffer, nil
}

func (sniffer *SnifferSetup) autodetectSnifferType(ostype string) string {
	if ostype == "linux" {
		return "af_packet"
	} else {
		return "pcap"
	}
}

func (sniffer *SnifferSetup) Reopen() error {
	var err error

	if sniffer.config.Type != "pcap" || sniffer.config.File == "" {
		return fmt.Errorf("Reopen is only possible for files")
	}

	sniffer.pcapHandle.Close()
	sniffer.pcapHandle, err = pcap.OpenOffline(sniffer.config.File)
	if err != nil {
		return err
	}

	sniffer.DataSource = gopacket.PacketDataSource(sniffer.pcapHandle)

	return nil
}

func (sniffer *SnifferSetup) Close() {
	switch sniffer.config.Type {
	case "pcap":
		sniffer.pcapHandle.Close()
	}
}

func (sniffer *SnifferSetup) Datalink() layers.LinkType {
	if sniffer.config.Type == "pcap" {
		return sniffer.pcapHandle.LinkType()
	}
	return layers.LinkTypeEthernet
}
<|endoftext|>


<commit_before><commit_msg>Set user agent for GCE and GKE API clients<commit_after><|endoftext|>


<commit_before><commit_msg>cmd/protoc-gen-go: add protoc suffix<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package kubernetes listens to Kubernetes for policy updates.
package kubernetes

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"io"
	"github.com/golang/glog"
	"net/http"
	"strings"
)

// readChunk reads the next chunk from the provided reader.
func readChunk(reader io.Reader) ([]byte, error) {
	var result []byte
	for {
		buf := make([]byte, bytes.MinRead)
		n, err := reader.Read(buf)
		if n < bytes.MinRead {
			result = append(result, buf[0:n]...)
			if err != nil {
				return result, err
			}
			break
		}
		result = append(result, buf...)
	}
	return result, nil
}

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

// kubeListener is a Service that listens to updates
// from Kubernetes by connecting to the endpoints specified
// and consuming chunked JSON documents. The endpoints are
// constructed from kubeURL and the following paths:
// 1. namespaceNotificationPath for namespace additions/deletions
// 2. policyNotificationPathPrefix + <namespace name> + policyNotificationPathPostfix
//    for policy additions/deletions.
type kubeListener struct {
	config                        common.ServiceConfig
	restClient                    *common.RestClient
	kubeURL                       string
	namespaceNotificationPath     string
	policyNotificationPathPrefix  string
	policyNotificationPathPostfix string
	segmentLabelName              string
	lastEventPerNamespace         map[string]uint64
	namespaceBufferSize           uint64
}

// Routes returns various routes used in the service.
func (l *kubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *kubeListener) Name() string {
	return "kubernetesListener"
}

// SetConfig implements SetConfig function of the Service interface.
func (l *kubeListener) SetConfig(config common.ServiceConfig) error {
	m := config.ServiceSpecific
	if m["kubernetes_url"] == "" {
		return errors.New("kubernetes_url required")
	}
	l.kubeURL = m["kubernetes_url"].(string)

	if m["namespace_notification_path"] == "" {
		return errors.New("namespace_notification_path required")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if m["policy_notification_path_prefix"] == "" {
		return errors.New("policy_notification_path_prefix required")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if m["policy_notification_path_postfix"] == "" {
		return errors.New("policy_notification_path_postfix required")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if m["segment_label_name"] == "" {
		return errors.New("segment_label_name required")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	if m["wait_for_iface_try"] == nil {
		l.namespaceBufferSize = 10
	} else {
		l.namespaceBufferSize = uint64(m["namespace_buffer_size"].(float64))
	}

	return nil
}

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	kubeListener := &kubeListener{}
	kubeListener.restClient = client
	config, err := client.GetServiceConfig(kubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(kubeListener, *config)
}

// getOrAddSegment finds a segment (based on segment selector).
// If not found, it adds one.
func (l *kubeListener) getOrAddSegment(namespace string, kubeSegmentName string) (*tenant.Segment, error) {
	ten := &tenant.Tenant{}
	ten.Name = namespace
	// TODO this should be changed to find EXACTLY one after deletion functionality is implemented
	err := l.restClient.Find(ten, common.FindLast)
	if err != nil {
		return nil, err
	}

	seg := &tenant.Segment{}
	seg.Name = kubeSegmentName
	seg.TenantID = ten.ID
	err = l.restClient.Find(seg, common.FindExactlyOne)
	if err == nil {
		return seg, nil
	}

	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentName, TenantID: ten.ID}
			segURL, err2 := l.restClient.GetServiceUrl("tenant")
			if err2 != nil {
				return nil, err2
			}
			segURL = fmt.Sprintf("%s/tenants/%d/segments", segURL, ten.ID)
			err2 = l.restClient.Post(segURL, segreq, seg)
			if err2 == nil {
				// Successful creation.
				return seg, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// resolveTenantByName retrieves tenant information from romana.
func (l *kubeListener) resolveTenantByName(tenantName string) (*tenant.Tenant, error) {
	t := &tenant.Tenant{Name: tenantName}
	err := l.restClient.Find(t, common.FindLast)
	if err != nil {
		return t, err
	}
	return t, nil
}

// translateNetworkPolicy translates a Kubernetes policy into
// Romana policy (see common.Policy) with the following rules:
// 1. Kubernetes Namespace corresponds to Romana Tenant
// 2. If Romana Tenant does not exist it is an error (a tenant should
//    automatically have been created when the namespace was added)
func (l *kubeListener) translateNetworkPolicy(kubePolicy *KubeObject) (common.Policy, error) {
	policyName := kubePolicy.Metadata.Name
	romanaPolicy := &common.Policy{Direction: common.PolicyDirectionIngress, Name: policyName, ExternalID: policyName}
	ns := kubePolicy.Metadata.Namespace
	// TODO actually look up tenant K8S ID.
	t, err := l.resolveTenantByName(ns)
	glog.Infof("translateNetworkPolicy(): For namespace %s got %+v / %+v", ns, t, err)
	if err != nil {
		return *romanaPolicy, err
	}
	tenantID := t.ID
	tenantExternalID := t.ExternalID

	kubeSegmentID := kubePolicy.Spec.PodSelector.MatchLabels[l.segmentLabelName]
	if kubeSegmentID == "" {
		return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
	}

	segment, err := l.getOrAddSegment(ns, kubeSegmentID)
	if err != nil {
		return *romanaPolicy, err
	}
	segmentID := segment.ID
	appliedTo := common.Endpoint{TenantID: tenantID, SegmentID: segmentID}
	romanaPolicy.AppliedTo = make([]common.Endpoint, 1)
	romanaPolicy.AppliedTo[0] = appliedTo
	romanaPolicy.Peers = []common.Endpoint{}
	romanaPolicy.Rules = common.Rules{}
	// TODO range
	// from := kubePolicy.Spec.Ingress[0].From
	// This is subject to change once the network specification in Kubernetes is finalized.
	// Right now it is a work in progress.
	for _, ingress := range kubePolicy.Spec.Ingress {
		for _, entry := range ingress.From {
			pods := entry.Pods
			fromKubeSegmentID := pods.MatchLabels[l.segmentLabelName]
			if fromKubeSegmentID == "" {
				return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
			}
			fromSegment, err := l.getOrAddSegment(ns, fromKubeSegmentID)
			if err != nil {
				return *romanaPolicy, err
			}
			peer := common.Endpoint{TenantID: tenantID, TenantExternalID: tenantExternalID, SegmentID: fromSegment.ID, SegmentExternalID: fromSegment.ExternalID}
			romanaPolicy.Peers = append(romanaPolicy.Peers, peer)
		}
		// TODO range
		// toPorts := kubePolicy.Spec.Ingress[0].ToPorts
		for _, toPort := range ingress.ToPorts {
			proto := strings.ToLower(toPort.Protocol)
			ports := []uint{toPort.Port}
			rule := common.Rule{Protocol: proto, Ports: ports}
			romanaPolicy.Rules = append(romanaPolicy.Rules, rule)
		}
	}
	err = romanaPolicy.Validate()
	if err != nil {
		return *romanaPolicy, err
	}
	return *romanaPolicy, nil
}

func (l *kubeListener) applyNetworkPolicy(action networkPolicyAction, romanaNetworkPolicy common.Policy) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies", policyURL)
	policyStr, _ := json.Marshal(romanaNetworkPolicy)
	switch action {
	case networkPolicyActionAdd:
		glog.Infof("Applying policy %s", policyStr)
		err := l.restClient.Post(policyURL, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	case networkPolicyActionDelete:
		glog.Infof("Deleting policy policy %s", policyStr)
		err := l.restClient.Delete(policyURL, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	default:
		return errors.New("Unsupported operation")
	}
	return nil
}

func (l *kubeListener) Initialize() error {
	glog.Infof("%s: Starting server", l.Name())
	nsURL, err := common.CleanURL(fmt.Sprintf("%s%s", l.kubeURL, l.namespaceNotificationPath))
	if err != nil {
		return err
	}
	l.lastEventPerNamespace = make(map[string]uint64)
	glog.Infof("Starting to listen on %s", nsURL)
	done := make(chan Done)
	nsEvents, err := l.nsWatch(done, nsURL)
	if err != nil {
		glog.Fatal("Namespace watcher failed to start", err)
	}

	events := l.conductor(nsEvents, done)
	l.process(events, done)
	glog.Infoln("All routines started")
	return nil
}

// CreateSchema is placeholder for now.
func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<commit_msg>Remove outdated code from kubernetes listener<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package kubernetes listens to Kubernetes for policy updates.
package kubernetes

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"github.com/golang/glog"
	"net/http"
	"strings"
)

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

// kubeListener is a Service that listens to updates
// from Kubernetes by connecting to the endpoints specified
// and consuming chunked JSON documents. The endpoints are
// constructed from kubeURL and the following paths:
// 1. namespaceNotificationPath for namespace additions/deletions
// 2. policyNotificationPathPrefix + <namespace name> + policyNotificationPathPostfix
//    for policy additions/deletions.
type kubeListener struct {
	config                        common.ServiceConfig
	restClient                    *common.RestClient
	kubeURL                       string
	namespaceNotificationPath     string
	policyNotificationPathPrefix  string
	policyNotificationPathPostfix string
	segmentLabelName              string
	lastEventPerNamespace         map[string]uint64
	namespaceBufferSize           uint64
}

// Routes returns various routes used in the service.
func (l *kubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *kubeListener) Name() string {
	return "kubernetesListener"
}

// SetConfig implements SetConfig function of the Service interface.
func (l *kubeListener) SetConfig(config common.ServiceConfig) error {
	m := config.ServiceSpecific
	if m["kubernetes_url"] == "" {
		return errors.New("kubernetes_url required")
	}
	l.kubeURL = m["kubernetes_url"].(string)

	if m["namespace_notification_path"] == "" {
		return errors.New("namespace_notification_path required")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if m["policy_notification_path_prefix"] == "" {
		return errors.New("policy_notification_path_prefix required")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if m["policy_notification_path_postfix"] == "" {
		return errors.New("policy_notification_path_postfix required")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if m["segment_label_name"] == "" {
		return errors.New("segment_label_name required")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	if m["wait_for_iface_try"] == nil {
		l.namespaceBufferSize = 10
	} else {
		l.namespaceBufferSize = uint64(m["namespace_buffer_size"].(float64))
	}

	return nil
}

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	kubeListener := &kubeListener{}
	kubeListener.restClient = client
	config, err := client.GetServiceConfig(kubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(kubeListener, *config)
}

// getOrAddSegment finds a segment (based on segment selector).
// If not found, it adds one.
func (l *kubeListener) getOrAddSegment(namespace string, kubeSegmentName string) (*tenant.Segment, error) {
	ten := &tenant.Tenant{}
	ten.Name = namespace
	// TODO this should be changed to find EXACTLY one after deletion functionality is implemented
	err := l.restClient.Find(ten, common.FindLast)
	if err != nil {
		return nil, err
	}

	seg := &tenant.Segment{}
	seg.Name = kubeSegmentName
	seg.TenantID = ten.ID
	err = l.restClient.Find(seg, common.FindExactlyOne)
	if err == nil {
		return seg, nil
	}

	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentName, TenantID: ten.ID}
			segURL, err2 := l.restClient.GetServiceUrl("tenant")
			if err2 != nil {
				return nil, err2
			}
			segURL = fmt.Sprintf("%s/tenants/%d/segments", segURL, ten.ID)
			err2 = l.restClient.Post(segURL, segreq, seg)
			if err2 == nil {
				// Successful creation.
				return seg, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// resolveTenantByName retrieves tenant information from romana.
func (l *kubeListener) resolveTenantByName(tenantName string) (*tenant.Tenant, error) {
	t := &tenant.Tenant{Name: tenantName}
	err := l.restClient.Find(t, common.FindLast)
	if err != nil {
		return t, err
	}
	return t, nil
}

// translateNetworkPolicy translates a Kubernetes policy into
// Romana policy (see common.Policy) with the following rules:
// 1. Kubernetes Namespace corresponds to Romana Tenant
// 2. If Romana Tenant does not exist it is an error (a tenant should
//    automatically have been created when the namespace was added)
func (l *kubeListener) translateNetworkPolicy(kubePolicy *KubeObject) (common.Policy, error) {
	policyName := kubePolicy.Metadata.Name
	romanaPolicy := &common.Policy{Direction: common.PolicyDirectionIngress, Name: policyName, ExternalID: policyName}
	ns := kubePolicy.Metadata.Namespace
	// TODO actually look up tenant K8S ID.
	t, err := l.resolveTenantByName(ns)
	glog.Infof("translateNetworkPolicy(): For namespace %s got %+v / %+v", ns, t, err)
	if err != nil {
		return *romanaPolicy, err
	}
	tenantID := t.ID
	tenantExternalID := t.ExternalID

	kubeSegmentID := kubePolicy.Spec.PodSelector.MatchLabels[l.segmentLabelName]
	if kubeSegmentID == "" {
		return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
	}

	segment, err := l.getOrAddSegment(ns, kubeSegmentID)
	if err != nil {
		return *romanaPolicy, err
	}
	segmentID := segment.ID
	appliedTo := common.Endpoint{TenantID: tenantID, SegmentID: segmentID}
	romanaPolicy.AppliedTo = make([]common.Endpoint, 1)
	romanaPolicy.AppliedTo[0] = appliedTo
	romanaPolicy.Peers = []common.Endpoint{}
	romanaPolicy.Rules = common.Rules{}
	// TODO range
	// from := kubePolicy.Spec.Ingress[0].From
	// This is subject to change once the network specification in Kubernetes is finalized.
	// Right now it is a work in progress.
	for _, ingress := range kubePolicy.Spec.Ingress {
		for _, entry := range ingress.From {
			pods := entry.Pods
			fromKubeSegmentID := pods.MatchLabels[l.segmentLabelName]
			if fromKubeSegmentID == "" {
				return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
			}
			fromSegment, err := l.getOrAddSegment(ns, fromKubeSegmentID)
			if err != nil {
				return *romanaPolicy, err
			}
			peer := common.Endpoint{TenantID: tenantID, TenantExternalID: tenantExternalID, SegmentID: fromSegment.ID, SegmentExternalID: fromSegment.ExternalID}
			romanaPolicy.Peers = append(romanaPolicy.Peers, peer)
		}
		// TODO range
		// toPorts := kubePolicy.Spec.Ingress[0].ToPorts
		for _, toPort := range ingress.ToPorts {
			proto := strings.ToLower(toPort.Protocol)
			ports := []uint{toPort.Port}
			rule := common.Rule{Protocol: proto, Ports: ports}
			romanaPolicy.Rules = append(romanaPolicy.Rules, rule)
		}
	}
	err = romanaPolicy.Validate()
	if err != nil {
		return *romanaPolicy, err
	}
	return *romanaPolicy, nil
}

func (l *kubeListener) applyNetworkPolicy(action networkPolicyAction, romanaNetworkPolicy common.Policy) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies", policyURL)
	policyStr, _ := json.Marshal(romanaNetworkPolicy)
	switch action {
	case networkPolicyActionAdd:
		glog.Infof("Applying policy %s", policyStr)
		err := l.restClient.Post(policyURL, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	case networkPolicyActionDelete:
		glog.Infof("Deleting policy policy %s", policyStr)
		err := l.restClient.Delete(policyURL, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	default:
		return errors.New("Unsupported operation")
	}
	return nil
}

func (l *kubeListener) Initialize() error {
	glog.Infof("%s: Starting server", l.Name())
	nsURL, err := common.CleanURL(fmt.Sprintf("%s%s", l.kubeURL, l.namespaceNotificationPath))
	if err != nil {
		return err
	}
	l.lastEventPerNamespace = make(map[string]uint64)
	glog.Infof("Starting to listen on %s", nsURL)
	done := make(chan Done)
	nsEvents, err := l.nsWatch(done, nsURL)
	if err != nil {
		glog.Fatal("Namespace watcher failed to start", err)
	}

	events := l.conductor(nsEvents, done)
	l.process(events, done)
	glog.Infoln("All routines started")
	return nil
}

// CreateSchema is placeholder for now.
func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<|endoftext|>


<commit_before>package tasks

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/jackspirou/govend/app/deps"
	"github.com/jackspirou/govend/app/helpers"
	"github.com/jackspirou/govend/app/utils/copyrecur"
)

func Govend() {

	// Run "$ go get" for all deps.
	for _, dep := range deps.List {
		// Command to go get {dep/repo}
		err := exec.Command("go", "get", "-d", dep).Run()
		helpers.Check(err)
	}

	// Read the $GOPATH env.
	gopath := os.Getenv("GOPATH")
	if len(gopath) < 1 {
		log.Fatalln("Unable to read the $GOPATH environment variable.")
	}

	// Remove the vendor directory if it exists.
	err := os.RemoveAll(deps.Dir)
	helpers.Check(err)

	// Copy all dependency code into the vendor directory.
	for _, dep := range deps.List {

		// Source and destination paths.
		src := gopath + "/src/" + dep
		dest := deps.Dir + "/" + dep

		// Tell the user we are pulling this dependency into their project.
		fmt.Println(" ↓ " + dep)

		// Recursively copy the dependency into the vendors directory.
		err := copyrecur.CopyDir(src, dest)
		helpers.Check(err)
	}

	// Tell the user we are done vendoring!
	fmt.Println("")
	fmt.Println("Vending complete")

}
<commit_msg>added gormimports<commit_after>package tasks

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/jackspirou/govend/app/deps"
	"github.com/jackspirou/govend/app/helpers"
	"github.com/jackspirou/govend/app/utils/copyrecur"
)

func Govend() {

	// deps for govend.
	err := exec.Command("go", "get", "-d", "github.com/jackspirou/rmimports").Run()
	helpers.Check(err)

	// deps for govend.
	err = exec.Command("go", "get", "github.com/jackspirou/gormimports").Run()
	helpers.Check(err)

	// Run "$ go get" for all deps.
	for _, dep := range deps.List {
		// Command to go get {dep/repo}
		err := exec.Command("go", "get", "-d", dep).Run()
		helpers.Check(err)
	}

	// Read the $GOPATH env.
	gopath := os.Getenv("GOPATH")
	if len(gopath) < 1 {
		log.Fatalln("Unable to read the $GOPATH environment variable.")
	}

	// Remove the vendor directory if it exists.
	err = os.RemoveAll(deps.Dir)
	helpers.Check(err)

	// Copy all dependency code into the vendor directory.
	for _, dep := range deps.List {

		// Source and destination paths.
		src := gopath + "/src/" + dep
		dest := deps.Dir + "/" + dep

		// Tell the user we are pulling this dependency into their project.
		fmt.Println(" ↓ " + dep)

		// Recursively copy the dependency into the vendors directory.
		err := copyrecur.CopyDir(src, dest)
		helpers.Check(err)
	}

	// Tell the user we are now attempting to fix imports.
	fmt.Println("")
	fmt.Println("Attempting to fix imports...")

	// Delete all files go get downloaded.
	for _, dep := range deps.List {

		src := gopath + "/src/" + dep

		err := os.RemoveAll(src)
		helpers.Check(err)
	}

	// Run gormimports
	err = exec.Command("gormimports", "-w", "./*").Run()
	helpers.Check(err)

	// Run goimports
	err = exec.Command("goimports", "-w", "./*").Run()
	helpers.Check(err)

	// Tell the user we are done vendoring!
	fmt.Println("")
	fmt.Println("Vending complete")

}
<|endoftext|>


<commit_before>package router

import (
	"context"
	"github.com/gin-gonic/gin"
	"github.com/pkg/errors"
	"github.com/pterodactyl/wings/router/tokens"
	"github.com/pterodactyl/wings/server"
	"github.com/pterodactyl/wings/server/filesystem"
	"golang.org/x/sync/errgroup"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

// Returns the contents of a file on the server.
func getServerFileContents(c *gin.Context) {
	s := GetServer(c.Param("server"))

	p, err := url.QueryUnescape(c.Query("file"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}
	p = "/" + strings.TrimLeft(p, "/")

	st, err := s.Filesystem().Stat(p)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Header("X-Mime-Type", st.Mimetype)
	c.Header("Content-Length", strconv.Itoa(int(st.Info.Size())))

	// If a download parameter is included in the URL go ahead and attach the necessary headers
	// so that the file can be downloaded.
	if c.Query("download") != "" {
		c.Header("Content-Disposition", "attachment; filename="+st.Info.Name())
		c.Header("Content-Type", "application/octet-stream")
	}

	// TODO(dane): should probably come up with a different approach here. If an error is encountered
	//  by this Readfile call you'll end up causing a (recovered) panic in the program because so many
	//  headers have already been set. We should probably add a RawReadfile that just returns the file
	//  to be read and then we can stream from that safely without error.
	//
	// Until that becomes a problem though I'm just going to leave this how it is. The panic is recovered
	// and a normal 500 error is returned to the client to my knowledge. It is also very unlikely to
	// happen since we're doing so much before this point that would normally throw an error if there
	// was a problem with the file.
	if err := s.Filesystem().Readfile(p, c.Writer); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}
}

// Returns the contents of a directory for a server.
func getServerListDirectory(c *gin.Context) {
	s := GetServer(c.Param("server"))

	d, err := url.QueryUnescape(c.Query("directory"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	stats, err := s.Filesystem().ListDirectory(d)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.JSON(http.StatusOK, stats)
}

type renameFile struct {
	To   string `json:"to"`
	From string `json:"from"`
}

// Renames (or moves) files for a server.
func putServerRenameFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string       `json:"root"`
		Files []renameFile `json:"files"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files to move or rename were provided.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and perform the move or rename action against each.
	for _, p := range data.Files {
		pf := path.Join(data.Root, p.From)
		pt := path.Join(data.Root, p.To)

		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				if err := s.Filesystem().Rename(pf, pt); err != nil {
					// Return nil if the error is an is not exists.
					// NOTE: os.IsNotExist() does not work if the error is wrapped.
					if errors.Is(err, os.ErrNotExist) {
						return nil
					}

					return err
				}

				return nil
			}
		})
	}

	if err := g.Wait(); err != nil {
		if errors.Is(err, os.ErrExist) {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Cannot move or rename file, destination already exists.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Copies a server file.
func postServerCopyFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Location string `json:"location"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if err := s.Filesystem().Copy(data.Location); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Deletes files from a server.
func postServerDeleteFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string   `json:"root"`
		Files []string `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files were specified for deletion.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and delete them. If any of the file deletions
	// fail just abort the process entirely.
	for _, p := range data.Files {
		pi := path.Join(data.Root, p)

		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				return s.Filesystem().Delete(pi)
			}
		})
	}

	if err := g.Wait(); err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Writes the contents of the request to a file on a server.
func postServerWriteFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	f, err := url.QueryUnescape(c.Query("file"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}
	f = "/" + strings.TrimLeft(f, "/")

	if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil {
		if errors.Is(err, filesystem.ErrIsDirectory) {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Cannot write file, name conflicts with an existing directory by the same name.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Create a directory on a server.
func postServerCreateDirectory(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Name string `json:"name"`
		Path string `json:"path"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if err := s.Filesystem().CreateDirectory(data.Name, data.Path); err != nil {
		if err.Error() == "not a directory" {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Part of the path being created is not a directory (ENOTDIR).",
			})
			return
		}

		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

func postServerCompressFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		RootPath string   `json:"root"`
		Files    []string `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files were passed through to be compressed.",
		})
		return
	}

	if !s.Filesystem().HasSpaceAvailable(true) {
		c.AbortWithStatusJSON(http.StatusConflict, gin.H{
			"error": "This server does not have enough available disk space to generate a compressed archive.",
		})
		return
	}

	f, err := s.Filesystem().CompressFiles(data.RootPath, data.Files)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.JSON(http.StatusOK, &filesystem.Stat{
		Info:     f,
		Mimetype: "application/tar+gzip",
	})
}

func postServerDecompressFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		RootPath string `json:"root"`
		File     string `json:"file"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File)
	if err != nil {
		// Handle an unknown format error.
		if errors.Is(err, filesystem.ErrUnknownArchiveFormat) {
			s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format")

			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "unknown archive format",
			})
			return
		}

		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	if !hasSpace {
		c.AbortWithStatusJSON(http.StatusConflict, gin.H{
			"error": "This server does not have enough available disk space to decompress this archive.",
		})
		return
	}

	if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
				"error": "The requested archive was not found.",
			})
			return
		}

		// If the file is busy for some reason just return a nicer error to the user since there is not
		// much we specifically can do. They'll need to stop the running server process in order to overwrite
		// a file like this.
		if strings.Contains(err.Error(), "text file busy") {
			s.Log().WithField("error", err).Warn("failed to decompress file due to busy text file")

			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "One or more files this archive is attempting to overwrite are currently in use by another process. Please try again.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

type chmodFile struct {
	File string      `json:"file"`
	Mode os.FileMode `json:"mode"`
}

func postServerChmodFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string      `json:"root"`
		Files []chmodFile `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files to chmod were provided.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and perform the move or rename action against each.
	for _, p := range data.Files {
		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				if err := s.Filesystem().Chmod(path.Join(data.Root, p.File), p.Mode); err != nil {
					// Return nil if the error is an is not exists.
					// NOTE: os.IsNotExist() does not work if the error is wrapped.
					if errors.Is(err, os.ErrNotExist) {
						return nil
					}

					return err
				}

				return nil
			}
		})
	}

	if err := g.Wait(); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

func postServerUploadFiles(c *gin.Context) {
	token := tokens.UploadPayload{}
	if err := tokens.ParseToken([]byte(c.Query("token")), &token); err != nil {
		TrackedError(err).AbortWithServerError(c)
		return
	}

	s := GetServer(token.ServerUuid)
	if s == nil || !token.IsUniqueRequest() {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
			"error": "The requested resource was not found on this server.",
		})
		return
	}

	form, err := c.MultipartForm()
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
			"error": "Failed to get multipart form data from request.",
		})
		return
	}

	headers, ok := form.File["files"]
	if !ok {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
			"error": "No files were found on the request body.",
		})
		return
	}

	directory := c.Query("directory")

	var totalSize int64
	for _, header := range headers {
		totalSize += header.Size
	}

	for _, header := range headers {
		p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename))
		if err != nil {
			TrackedServerError(err, s).AbortFilesystemError(c)
			return
		}

		// We run this in a different method so I can use defer without any of
		// the consequences caused by calling it in a loop.
		if err := handleFileUpload(p, s, header); err != nil {
			TrackedServerError(err, s).AbortFilesystemError(c)
			return
		}
	}
}

func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) error {
	file, err := header.Open()
	if err != nil {
		return err
	}
	defer file.Close()

	if err := s.Filesystem().Writefile(p, file); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix chmod endpoint<commit_after>package router

import (
	"context"
	"github.com/apex/log"
	"github.com/gin-gonic/gin"
	"github.com/pkg/errors"
	"github.com/pterodactyl/wings/router/tokens"
	"github.com/pterodactyl/wings/server"
	"github.com/pterodactyl/wings/server/filesystem"
	"golang.org/x/sync/errgroup"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

// Returns the contents of a file on the server.
func getServerFileContents(c *gin.Context) {
	s := GetServer(c.Param("server"))

	p, err := url.QueryUnescape(c.Query("file"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}
	p = "/" + strings.TrimLeft(p, "/")

	st, err := s.Filesystem().Stat(p)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Header("X-Mime-Type", st.Mimetype)
	c.Header("Content-Length", strconv.Itoa(int(st.Info.Size())))

	// If a download parameter is included in the URL go ahead and attach the necessary headers
	// so that the file can be downloaded.
	if c.Query("download") != "" {
		c.Header("Content-Disposition", "attachment; filename="+st.Info.Name())
		c.Header("Content-Type", "application/octet-stream")
	}

	// TODO(dane): should probably come up with a different approach here. If an error is encountered
	//  by this Readfile call you'll end up causing a (recovered) panic in the program because so many
	//  headers have already been set. We should probably add a RawReadfile that just returns the file
	//  to be read and then we can stream from that safely without error.
	//
	// Until that becomes a problem though I'm just going to leave this how it is. The panic is recovered
	// and a normal 500 error is returned to the client to my knowledge. It is also very unlikely to
	// happen since we're doing so much before this point that would normally throw an error if there
	// was a problem with the file.
	if err := s.Filesystem().Readfile(p, c.Writer); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}
}

// Returns the contents of a directory for a server.
func getServerListDirectory(c *gin.Context) {
	s := GetServer(c.Param("server"))

	d, err := url.QueryUnescape(c.Query("directory"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	stats, err := s.Filesystem().ListDirectory(d)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.JSON(http.StatusOK, stats)
}

type renameFile struct {
	To   string `json:"to"`
	From string `json:"from"`
}

// Renames (or moves) files for a server.
func putServerRenameFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string       `json:"root"`
		Files []renameFile `json:"files"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files to move or rename were provided.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and perform the move or rename action against each.
	for _, p := range data.Files {
		pf := path.Join(data.Root, p.From)
		pt := path.Join(data.Root, p.To)

		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				if err := s.Filesystem().Rename(pf, pt); err != nil {
					// Return nil if the error is an is not exists.
					// NOTE: os.IsNotExist() does not work if the error is wrapped.
					if errors.Is(err, os.ErrNotExist) {
						return nil
					}

					return err
				}

				return nil
			}
		})
	}

	if err := g.Wait(); err != nil {
		if errors.Is(err, os.ErrExist) {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Cannot move or rename file, destination already exists.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Copies a server file.
func postServerCopyFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Location string `json:"location"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if err := s.Filesystem().Copy(data.Location); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Deletes files from a server.
func postServerDeleteFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string   `json:"root"`
		Files []string `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files were specified for deletion.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and delete them. If any of the file deletions
	// fail just abort the process entirely.
	for _, p := range data.Files {
		pi := path.Join(data.Root, p)

		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				return s.Filesystem().Delete(pi)
			}
		})
	}

	if err := g.Wait(); err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Writes the contents of the request to a file on a server.
func postServerWriteFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	f, err := url.QueryUnescape(c.Query("file"))
	if err != nil {
		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}
	f = "/" + strings.TrimLeft(f, "/")

	if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil {
		if errors.Is(err, filesystem.ErrIsDirectory) {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Cannot write file, name conflicts with an existing directory by the same name.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

// Create a directory on a server.
func postServerCreateDirectory(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Name string `json:"name"`
		Path string `json:"path"`
	}
	// BindJSON sends 400 if the request fails, all we need to do is return
	if err := c.BindJSON(&data); err != nil {
		return
	}

	if err := s.Filesystem().CreateDirectory(data.Name, data.Path); err != nil {
		if err.Error() == "not a directory" {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "Part of the path being created is not a directory (ENOTDIR).",
			})
			return
		}

		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

func postServerCompressFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		RootPath string   `json:"root"`
		Files    []string `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files were passed through to be compressed.",
		})
		return
	}

	if !s.Filesystem().HasSpaceAvailable(true) {
		c.AbortWithStatusJSON(http.StatusConflict, gin.H{
			"error": "This server does not have enough available disk space to generate a compressed archive.",
		})
		return
	}

	f, err := s.Filesystem().CompressFiles(data.RootPath, data.Files)
	if err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.JSON(http.StatusOK, &filesystem.Stat{
		Info:     f,
		Mimetype: "application/tar+gzip",
	})
}

func postServerDecompressFiles(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		RootPath string `json:"root"`
		File     string `json:"file"`
	}

	if err := c.BindJSON(&data); err != nil {
		return
	}

	hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File)
	if err != nil {
		// Handle an unknown format error.
		if errors.Is(err, filesystem.ErrUnknownArchiveFormat) {
			s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format")

			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "unknown archive format",
			})
			return
		}

		TrackedServerError(err, s).AbortWithServerError(c)
		return
	}

	if !hasSpace {
		c.AbortWithStatusJSON(http.StatusConflict, gin.H{
			"error": "This server does not have enough available disk space to decompress this archive.",
		})
		return
	}

	if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
				"error": "The requested archive was not found.",
			})
			return
		}

		// If the file is busy for some reason just return a nicer error to the user since there is not
		// much we specifically can do. They'll need to stop the running server process in order to overwrite
		// a file like this.
		if strings.Contains(err.Error(), "text file busy") {
			s.Log().WithField("error", err).Warn("failed to decompress file due to busy text file")

			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
				"error": "One or more files this archive is attempting to overwrite are currently in use by another process. Please try again.",
			})
			return
		}

		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

type chmodFile struct {
	File string `json:"file"`
	Mode string `json:"mode"`
}

func postServerChmodFile(c *gin.Context) {
	s := GetServer(c.Param("server"))

	var data struct {
		Root  string      `json:"root"`
		Files []chmodFile `json:"files"`
	}

	if err := c.BindJSON(&data); err != nil {
		log.Debug(err.Error())
		return
	}

	if len(data.Files) == 0 {
		c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{
			"error": "No files to chmod were provided.",
		})
		return
	}

	g, ctx := errgroup.WithContext(context.Background())

	// Loop over the array of files passed in and perform the move or rename action against each.
	for _, p := range data.Files {
		g.Go(func() error {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
				mode, err := strconv.ParseUint(p.Mode, 10, 32)
				if err != nil {
					return err
				}

				if err := s.Filesystem().Chmod(path.Join(data.Root, p.File), os.FileMode(uint32(mode))); err != nil {
					// Return nil if the error is an is not exists.
					// NOTE: os.IsNotExist() does not work if the error is wrapped.
					if errors.Is(err, os.ErrNotExist) {
						return nil
					}

					return err
				}

				return nil
			}
		})
	}

	if err := g.Wait(); err != nil {
		TrackedServerError(err, s).AbortFilesystemError(c)
		return
	}

	c.Status(http.StatusNoContent)
}

func postServerUploadFiles(c *gin.Context) {
	token := tokens.UploadPayload{}
	if err := tokens.ParseToken([]byte(c.Query("token")), &token); err != nil {
		TrackedError(err).AbortWithServerError(c)
		return
	}

	s := GetServer(token.ServerUuid)
	if s == nil || !token.IsUniqueRequest() {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
			"error": "The requested resource was not found on this server.",
		})
		return
	}

	form, err := c.MultipartForm()
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
			"error": "Failed to get multipart form data from request.",
		})
		return
	}

	headers, ok := form.File["files"]
	if !ok {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
			"error": "No files were found on the request body.",
		})
		return
	}

	directory := c.Query("directory")

	var totalSize int64
	for _, header := range headers {
		totalSize += header.Size
	}

	for _, header := range headers {
		p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename))
		if err != nil {
			TrackedServerError(err, s).AbortFilesystemError(c)
			return
		}

		// We run this in a different method so I can use defer without any of
		// the consequences caused by calling it in a loop.
		if err := handleFileUpload(p, s, header); err != nil {
			TrackedServerError(err, s).AbortFilesystemError(c)
			return
		}
	}
}

func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) error {
	file, err := header.Open()
	if err != nil {
		return err
	}
	defer file.Close()

	if err := s.Filesystem().Writefile(p, file); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package lex

import (
	"fmt"
	"regexp"
	"strings"
)

var (
	// key[:val]
	kv_re     *regexp.Regexp = regexp.MustCompile(`^([A-z][0-9A-z_]*)(:(("[^"\\]*(?:\\.[^"\\]*)*")|([^\s:"]+)))?\s+`)
	escape_re *regexp.Regexp = regexp.MustCompile(`\\.`)
)

func parseKV(src string) func() (string, string, error) {
	remain := append([]byte(strings.TrimSpace(src)), ' ')
	return func() (string, string, error) {
		// drained
		if len(remain) == 0 {
			return "", "", nil
		}

		m := kv_re.FindSubmatch(remain)
		if m == nil {
			return "", "", fmt.Errorf("Illegal kv format near: %q", string(remain))
		}

		l := len(m[0])
		k := m[1]
		v := m[3]

		// if it is quoted. Unquote it, see: https://golang.org/ref/spec#Rune_literals
		if len(m[4]) != 0 {
			v = escape_re.ReplaceAllFunc(v[1:len(v)-1], func(x []byte) []byte {
				switch c := x[1]; c {
				default:
					return []byte{c}
				case 'a':
					return []byte{'\a'}
				case 'b':
					return []byte{'\b'}
				case 'f':
					return []byte{'\f'}
				case 'n':
					return []byte{'\n'}
				case 'r':
					return []byte{'\r'}
				case 't':
					return []byte{'\t'}
				case 'v':
					return []byte{'\v'}
				}
			})
		}

		remain = remain[l:]
		return string(k), string(v), nil
	}
}
<commit_msg>Change some comment<commit_after>package lex

import (
	"fmt"
	"regexp"
	"strings"
)

var (
	// KV format: key[:val]
	// Example:
	//   param:name type:int attr:"hello world"
	kv_re     *regexp.Regexp = regexp.MustCompile(`^([A-z][0-9A-z_]*)(:(("[^"\\]*(?:\\.[^"\\]*)*")|([^\s:"]+)))?\s+`)
	escape_re *regexp.Regexp = regexp.MustCompile(`\\.`)
)

func parseKV(src string) func() (string, string, error) {
	remain := append([]byte(strings.TrimSpace(src)), ' ')
	return func() (string, string, error) {
		// drained
		if len(remain) == 0 {
			return "", "", nil
		}

		m := kv_re.FindSubmatch(remain)
		if m == nil {
			return "", "", fmt.Errorf("Illegal kv format near: %q", string(remain))
		}

		l := len(m[0])
		k := m[1]
		v := m[3]

		// if it is quoted. Unquote it, see: https://golang.org/ref/spec#Rune_literals
		if len(m[4]) != 0 {
			v = escape_re.ReplaceAllFunc(v[1:len(v)-1], func(x []byte) []byte {
				switch c := x[1]; c {
				default:
					return []byte{c}
				case 'a':
					return []byte{'\a'}
				case 'b':
					return []byte{'\b'}
				case 'f':
					return []byte{'\f'}
				case 'n':
					return []byte{'\n'}
				case 'r':
					return []byte{'\r'}
				case 't':
					return []byte{'\t'}
				case 'v':
					return []byte{'\v'}
				}
			})
		}

		remain = remain[l:]
		return string(k), string(v), nil
	}
}
<|endoftext|>


<commit_before>package network

import (
	"encoding/binary"
	"errors"
	"io"
	"math"
)

// --------------
// | len | data |
// --------------
type MsgParser struct {
	lenMsgLen    int
	minMsgLen    uint32
	maxMsgLen    uint32
	littleEndian bool
}

func NewMsgParser() *MsgParser {
	p := new(MsgParser)
	p.lenMsgLen = 2
	p.minMsgLen = 1
	p.maxMsgLen = 4096
	p.littleEndian = false

	return p
}

// It's dangerous to call the method on reading or writing
func (p *MsgParser) SetMsgLen(lenMsgLen int, minMsgLen uint32, maxMsgLen uint32) {
	if lenMsgLen == 1 || lenMsgLen == 2 || lenMsgLen == 4 {
		p.lenMsgLen = lenMsgLen
	}
	if minMsgLen != 0 {
		p.minMsgLen = minMsgLen
	}
	if maxMsgLen != 0 {
		p.maxMsgLen = maxMsgLen
	}

	var max uint32
	switch p.lenMsgLen {
	case 1:
		max = math.MaxUint8
	case 2:
		max = math.MaxUint16
	}
	if p.minMsgLen > max {
		p.minMsgLen = max
	}
	if p.maxMsgLen > max {
		p.maxMsgLen = max
	}
}

// It's dangerous to call the method on reading or writing
func (p *MsgParser) SetByteOrder(littleEndian bool) {
	p.littleEndian = littleEndian
}

func (p *MsgParser) Read(conn *TCPConn) ([]byte, error) {
	var b [4]byte
	bufMsgLen := b[:p.lenMsgLen]

	// read len
	if _, err := io.ReadFull(conn, bufMsgLen); err != nil {
		return nil, err
	}

	// parse len
	var msgLen uint32
	switch p.lenMsgLen {
	case 1:
		msgLen = uint32(bufMsgLen[0])
	case 2:
		if p.littleEndian {
			msgLen = uint32(binary.LittleEndian.Uint16(bufMsgLen))
		} else {
			msgLen = uint32(binary.BigEndian.Uint16(bufMsgLen))
		}
	case 4:
		if p.littleEndian {
			msgLen = binary.LittleEndian.Uint32(bufMsgLen)
		} else {
			msgLen = binary.BigEndian.Uint32(bufMsgLen)
		}
	}

	// check len
	if msgLen > p.maxMsgLen {
		return nil, errors.New("message too long")
	} else if msgLen < p.minMsgLen {
		return nil, errors.New("message too short")
	}

	// data
	msgData := make([]byte, msgLen)
	if _, err := io.ReadFull(conn, msgData); err != nil {
		return nil, err
	}

	return msgData, nil
}

func (p *MsgParser) Write(conn *TCPConn, args ...[]byte) error {
	// get len
	var msgLen uint32
	for i := 0; i < len(args); i++ {
		msgLen += uint32(len(args[i]))
	}

	// check len
	if msgLen > p.maxMsgLen {
		return errors.New("message too long")
	} else if msgLen < p.minMsgLen {
		return errors.New("message too short")
	}

	msg := make([]byte, uint32(p.lenMsgLen)+msgLen)

	// write len
	switch p.lenMsgLen {
	case 1:
		msg[0] = byte(msgLen)
	case 2:
		if p.littleEndian {
			binary.LittleEndian.PutUint16(msg, uint16(msgLen))
		} else {
			binary.BigEndian.PutUint16(msg, uint16(msgLen))
		}
	case 4:
		if p.littleEndian {
			binary.LittleEndian.PutUint32(msg, msgLen)
		} else {
			binary.BigEndian.PutUint32(msg, msgLen)
		}
	}

	// write data
	l := p.lenMsgLen
	for i := 0; i < len(args); i++ {
		copy(msg[l:], args[i])
		l += len(args[i])
	}

	conn.Write(msg)

	return nil
}
<commit_msg>add comment<commit_after>package network

import (
	"encoding/binary"
	"errors"
	"io"
	"math"
)

// --------------
// | len | data |
// --------------
type MsgParser struct {
	lenMsgLen    int
	minMsgLen    uint32
	maxMsgLen    uint32
	littleEndian bool
}

func NewMsgParser() *MsgParser {
	p := new(MsgParser)
	p.lenMsgLen = 2
	p.minMsgLen = 1
	p.maxMsgLen = 4096
	p.littleEndian = false

	return p
}

// It's dangerous to call the method on reading or writing
func (p *MsgParser) SetMsgLen(lenMsgLen int, minMsgLen uint32, maxMsgLen uint32) {
	if lenMsgLen == 1 || lenMsgLen == 2 || lenMsgLen == 4 {
		p.lenMsgLen = lenMsgLen
	}
	if minMsgLen != 0 {
		p.minMsgLen = minMsgLen
	}
	if maxMsgLen != 0 {
		p.maxMsgLen = maxMsgLen
	}

	var max uint32
	switch p.lenMsgLen {
	case 1:
		max = math.MaxUint8
	case 2:
		max = math.MaxUint16
	}
	if p.minMsgLen > max {
		p.minMsgLen = max
	}
	if p.maxMsgLen > max {
		p.maxMsgLen = max
	}
}

// It's dangerous to call the method on reading or writing
func (p *MsgParser) SetByteOrder(littleEndian bool) {
	p.littleEndian = littleEndian
}

// goroutine safe
func (p *MsgParser) Read(conn *TCPConn) ([]byte, error) {
	var b [4]byte
	bufMsgLen := b[:p.lenMsgLen]

	// read len
	if _, err := io.ReadFull(conn, bufMsgLen); err != nil {
		return nil, err
	}

	// parse len
	var msgLen uint32
	switch p.lenMsgLen {
	case 1:
		msgLen = uint32(bufMsgLen[0])
	case 2:
		if p.littleEndian {
			msgLen = uint32(binary.LittleEndian.Uint16(bufMsgLen))
		} else {
			msgLen = uint32(binary.BigEndian.Uint16(bufMsgLen))
		}
	case 4:
		if p.littleEndian {
			msgLen = binary.LittleEndian.Uint32(bufMsgLen)
		} else {
			msgLen = binary.BigEndian.Uint32(bufMsgLen)
		}
	}

	// check len
	if msgLen > p.maxMsgLen {
		return nil, errors.New("message too long")
	} else if msgLen < p.minMsgLen {
		return nil, errors.New("message too short")
	}

	// data
	msgData := make([]byte, msgLen)
	if _, err := io.ReadFull(conn, msgData); err != nil {
		return nil, err
	}

	return msgData, nil
}

// goroutine safe
func (p *MsgParser) Write(conn *TCPConn, args ...[]byte) error {
	// get len
	var msgLen uint32
	for i := 0; i < len(args); i++ {
		msgLen += uint32(len(args[i]))
	}

	// check len
	if msgLen > p.maxMsgLen {
		return errors.New("message too long")
	} else if msgLen < p.minMsgLen {
		return errors.New("message too short")
	}

	msg := make([]byte, uint32(p.lenMsgLen)+msgLen)

	// write len
	switch p.lenMsgLen {
	case 1:
		msg[0] = byte(msgLen)
	case 2:
		if p.littleEndian {
			binary.LittleEndian.PutUint16(msg, uint16(msgLen))
		} else {
			binary.BigEndian.PutUint16(msg, uint16(msgLen))
		}
	case 4:
		if p.littleEndian {
			binary.LittleEndian.PutUint32(msg, msgLen)
		} else {
			binary.BigEndian.PutUint32(msg, msgLen)
		}
	}

	// write data
	l := p.lenMsgLen
	for i := 0; i < len(args); i++ {
		copy(msg[l:], args[i])
		l += len(args[i])
	}

	conn.Write(msg)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import (
	"github.com/garyburd/redigo/redis"
	"strings"
	"time"
)

func NewPool(connectString string, maxIdle int, maxActive int, idleTime time.Duration) *redis.Pool {
	pool := redis.Pool{
		MaxIdle:     15,
		MaxActive:   0,
		IdleTimeout: 240*time.Second,
		Wait:        true,
		Dial: func() (redis.Conn, error) {
			urlParts := strings.Split(connectString, "/")
			c, err := redis.Dial("tcp", urlParts[0])
			if err != nil {
				return nil, err
			}

			if len(urlParts) == 2 {
				selectErr := c.Send("SELECT", urlParts[1])
				if selectErr != nil {
					return nil, selectErr
				}
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
	return &pool
}
<commit_msg>Updated redigo library and added Wait=true to pool<commit_after>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import (
	"github.com/garyburd/redigo/redis"
	"strings"
	"time"
)

func NewPool(connectString string, maxIdle int, maxActive int, idleTime time.Duration) *redis.Pool {
	pool := redis.Pool{
		MaxIdle:     15,
		MaxActive:   0,
		IdleTimeout: 240 * time.Second,
		Wait:        true,
		Dial: func() (redis.Conn, error) {
			urlParts := strings.Split(connectString, "/")
			c, err := redis.Dial("tcp", urlParts[0])
			if err != nil {
				return nil, err
			}

			if len(urlParts) == 2 {
				selectErr := c.Send("SELECT", urlParts[1])
				if selectErr != nil {
					return nil, selectErr
				}
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
	return &pool
}
<|endoftext|>


<commit_before>package janus

import (
	"encoding/json"

	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
)

type OAuthMiddleware struct {
	oauthManager *OAuthManager
	oauthSpec    *OAuthSpec
}

func (m *OAuthMiddleware) ProcessRequest(req *http.Request, c *gin.Context) (error, int) {
	var newSession SessionState
	newSession.OAuthServerID = m.oauthSpec.ID

	log.WithFields(log.Fields{
		"req": req,
	}).Info("Getting body")

	data, exists := c.Get("body")
	if false == exists {
		return nil, http.StatusInternalServerError
	}

	body := data.([]byte)

	if marshalErr := json.Unmarshal(body, &newSession); marshalErr != nil {
		return marshalErr, http.StatusInternalServerError
	}

	m.oauthManager.Set(newSession.AccessToken, newSession, newSession.ExpiresIn)

	return nil, http.StatusOK
}
<commit_msg>Update OAuthMiddleware to remove gin<commit_after>package janus

import (
	"encoding/json"
	"errors"

	"net/http"

	log "github.com/Sirupsen/logrus"
)

// OAuthMiddleware is the after middleware for OAuth routes
type OAuthMiddleware struct {
	oauthManager *OAuthManager
	oauthSpec    *OAuthSpec
}

// ProcessRequest is the middleware method.
func (m *OAuthMiddleware) ProcessRequest(rw http.ResponseWriter, req *http.Request) (int, error) {
	var newSession SessionState
	newSession.OAuthServerID = m.oauthSpec.ID

	log.WithFields(log.Fields{
		"req": req,
	}).Info("Getting body")

	var body []byte
	body = req.Context().Value("body").([]byte)

	if nil == body {
		return http.StatusInternalServerError, errors.New("Request body not present")
	}

	if marshalErr := json.Unmarshal(body, &newSession); marshalErr != nil {
		return http.StatusInternalServerError, marshalErr
	}

	m.oauthManager.Set(newSession.AccessToken, newSession, newSession.ExpiresIn)

	return http.StatusOK, nil
}
<|endoftext|>


<commit_before>package ssdp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
)

// Monitor monitors SSDP's alive and byebye messages.
type Monitor struct {
	Alive  AliveHandler
	Bye    ByeHandler
	Search SearchHandler

	conn *multicastConn
	wg   sync.WaitGroup
}

// Start starts to monitor SSDP messages.
func (m *Monitor) Start() error {
	conn, err := multicastListen(recvAddrIPv4)
	if err != nil {
		return err
	}
	logf("monitoring on %s", conn.LocalAddr().String())
	m.conn = conn
	m.wg.Add(1)
	go func() {
		m.serve()
		m.wg.Done()
	}()
	return nil
}

func (m *Monitor) serve() error {
	err := m.conn.readPackets(0, func(addr net.Addr, data []byte) error {
		msg := make([]byte, len(data))
		copy(msg, data)
		go m.handleRaw(addr, msg)
		return nil
	})
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}

func (m *Monitor) handleRaw(addr net.Addr, raw []byte) error {
	if bytes.HasPrefix(raw, []byte("M-SEARCH ")) {
		return m.handleSearch(addr, raw)
	}
	if bytes.HasPrefix(raw, []byte("NOTIFY ")) {
		return m.handleNotify(addr, raw)
	}
	n := bytes.Index(raw, []byte("\r\n"))
	logf("unexpected method: %q", string(raw[:n]))
	return nil
}

func (m *Monitor) handleNotify(addr net.Addr, raw []byte) error {
	req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(raw)))
	if err != nil {
		return err
	}
	switch nts := req.Header.Get("NTS"); nts {
	case "ssdp:alive":
		if req.Method != "NOTIFY" {
			return fmt.Errorf("unexpected method for %q: %s", "ssdp:alive", req.Method)
		}
		if h := m.Alive; h != nil {
			h(&AliveMessage{
				From:      addr,
				Type:      req.Header.Get("NT"),
				USN:       req.Header.Get("USN"),
				Location:  req.Header.Get("LOCATION"),
				Server:    req.Header.Get("SERVER"),
				rawHeader: req.Header,
			})
		}
	case "ssdp:byebye":
		if req.Method != "NOTIFY" {
			return fmt.Errorf("unexpected method for %q: %s", "ssdp:byebye", req.Method)
		}
		if h := m.Bye; h != nil {
			h(&ByeMessage{
				From:      addr,
				Type:      req.Header.Get("NT"),
				USN:       req.Header.Get("USN"),
				rawHeader: req.Header,
			})
		}
	default:
		return fmt.Errorf("unknown NTS: %s", nts)
	}
	return nil
}

func (m *Monitor) handleSearch(addr net.Addr, raw []byte) error {
	req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(raw)))
	if err != nil {
		return err
	}
	man := req.Header.Get("MAN")
	if man != `"ssdp:discover"` {
		return fmt.Errorf("unexpected MAN: %s", man)
	}
	if h := m.Search; h != nil {
		h(&SearchMessage{
			From:      addr,
			Type:      req.Header.Get("ST"),
			rawHeader: req.Header,
		})
	}
	return nil
}

// Close closes monitoring.
func (m *Monitor) Close() error {
	if m.conn != nil {
		m.conn.Close()
		m.conn = nil
		m.wg.Wait()
	}
	return nil
}

// AliveMessage represents SSDP's ssdp:alive message.
type AliveMessage struct {
	// From is a sender of this message
	From net.Addr

	// Type is a property of "NT"
	Type string

	// USN is a property of "USN"
	USN string

	// Location is a property of "LOCATION"
	Location string

	// Server is a property of "SERVER"
	Server string

	rawHeader http.Header
	maxAge    *int
}

// Header returns all properties in alive message.
func (m *AliveMessage) Header() http.Header {
	return m.rawHeader
}

// MaxAge extracts "max-age" value from "CACHE-CONTROL" property.
func (m *AliveMessage) MaxAge() int {
	if m.maxAge == nil {
		m.maxAge = new(int)
		*m.maxAge = extractMaxAge(m.rawHeader.Get("CACHE-CONTROL"), -1)
	}
	return *m.maxAge
}

// AliveHandler is handler of Alive message.
type AliveHandler func(*AliveMessage)

// ByeMessage represents SSDP's ssdp:byebye message.
type ByeMessage struct {
	// From is a sender of this message
	From net.Addr

	// Type is a property of "NT"
	Type string

	// USN is a property of "USN"
	USN string

	rawHeader http.Header
}

// Header returns all properties in bye message.
func (m *ByeMessage) Header() http.Header {
	return m.rawHeader
}

// ByeHandler is handler of Bye message.
type ByeHandler func(*ByeMessage)

// SearchMessage represents SSDP's ssdp:discover message.
type SearchMessage struct {
	From net.Addr
	Type string

	rawHeader http.Header
}

// Header returns all properties in search message.
func (s *SearchMessage) Header() http.Header {
	return s.rawHeader
}

// SearchHandler is handler of Search message.
type SearchHandler func(*SearchMessage)
<commit_msg>Again handle SSDP messages that don't end in an empty newline<commit_after>package ssdp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"net/http"
	"sync"
)

// Monitor monitors SSDP's alive and byebye messages.
type Monitor struct {
	Alive  AliveHandler
	Bye    ByeHandler
	Search SearchHandler

	conn *multicastConn
	wg   sync.WaitGroup
}

// Start starts to monitor SSDP messages.
func (m *Monitor) Start() error {
	conn, err := multicastListen(recvAddrIPv4)
	if err != nil {
		return err
	}
	logf("monitoring on %s", conn.LocalAddr().String())
	m.conn = conn
	m.wg.Add(1)
	go func() {
		m.serve()
		m.wg.Done()
	}()
	return nil
}

func (m *Monitor) serve() error {
	err := m.conn.readPackets(0, func(addr net.Addr, data []byte) error {
		msg := make([]byte, len(data))
		copy(msg, data)
		go m.handleRaw(addr, msg)
		return nil
	})
	if err != nil && err != io.EOF {
		return err
	}
	return nil
}

// Actually is a double new line
var newLine = []byte{'\r', '\n', '\r', '\n'}

func (m *Monitor) handleRaw(addr net.Addr, raw []byte) error {
	// Add newline to workaround buggy SSDP responses
	if !bytes.HasSuffix(raw, newLine) {
		raw = bytes.Join([][]byte{raw, newLine}, nil)
	}
	if bytes.HasPrefix(raw, []byte("M-SEARCH ")) {
		return m.handleSearch(addr, raw)
	}
	if bytes.HasPrefix(raw, []byte("NOTIFY ")) {
		return m.handleNotify(addr, raw)
	}
	n := bytes.Index(raw, []byte("\r\n"))
	logf("unexpected method: %q", string(raw[:n]))
	return nil
}

func (m *Monitor) handleNotify(addr net.Addr, raw []byte) error {
	req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(raw)))
	if err != nil {
		return err
	}
	switch nts := req.Header.Get("NTS"); nts {
	case "ssdp:alive":
		if req.Method != "NOTIFY" {
			return fmt.Errorf("unexpected method for %q: %s", "ssdp:alive", req.Method)
		}
		if h := m.Alive; h != nil {
			h(&AliveMessage{
				From:      addr,
				Type:      req.Header.Get("NT"),
				USN:       req.Header.Get("USN"),
				Location:  req.Header.Get("LOCATION"),
				Server:    req.Header.Get("SERVER"),
				rawHeader: req.Header,
			})
		}
	case "ssdp:byebye":
		if req.Method != "NOTIFY" {
			return fmt.Errorf("unexpected method for %q: %s", "ssdp:byebye", req.Method)
		}
		if h := m.Bye; h != nil {
			h(&ByeMessage{
				From:      addr,
				Type:      req.Header.Get("NT"),
				USN:       req.Header.Get("USN"),
				rawHeader: req.Header,
			})
		}
	default:
		return fmt.Errorf("unknown NTS: %s", nts)
	}
	return nil
}

func (m *Monitor) handleSearch(addr net.Addr, raw []byte) error {
	req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(raw)))
	if err != nil {
		return err
	}
	man := req.Header.Get("MAN")
	if man != `"ssdp:discover"` {
		return fmt.Errorf("unexpected MAN: %s", man)
	}
	if h := m.Search; h != nil {
		h(&SearchMessage{
			From:      addr,
			Type:      req.Header.Get("ST"),
			rawHeader: req.Header,
		})
	}
	return nil
}

// Close closes monitoring.
func (m *Monitor) Close() error {
	if m.conn != nil {
		m.conn.Close()
		m.conn = nil
		m.wg.Wait()
	}
	return nil
}

// AliveMessage represents SSDP's ssdp:alive message.
type AliveMessage struct {
	// From is a sender of this message
	From net.Addr

	// Type is a property of "NT"
	Type string

	// USN is a property of "USN"
	USN string

	// Location is a property of "LOCATION"
	Location string

	// Server is a property of "SERVER"
	Server string

	rawHeader http.Header
	maxAge    *int
}

// Header returns all properties in alive message.
func (m *AliveMessage) Header() http.Header {
	return m.rawHeader
}

// MaxAge extracts "max-age" value from "CACHE-CONTROL" property.
func (m *AliveMessage) MaxAge() int {
	if m.maxAge == nil {
		m.maxAge = new(int)
		*m.maxAge = extractMaxAge(m.rawHeader.Get("CACHE-CONTROL"), -1)
	}
	return *m.maxAge
}

// AliveHandler is handler of Alive message.
type AliveHandler func(*AliveMessage)

// ByeMessage represents SSDP's ssdp:byebye message.
type ByeMessage struct {
	// From is a sender of this message
	From net.Addr

	// Type is a property of "NT"
	Type string

	// USN is a property of "USN"
	USN string

	rawHeader http.Header
}

// Header returns all properties in bye message.
func (m *ByeMessage) Header() http.Header {
	return m.rawHeader
}

// ByeHandler is handler of Bye message.
type ByeHandler func(*ByeMessage)

// SearchMessage represents SSDP's ssdp:discover message.
type SearchMessage struct {
	From net.Addr
	Type string

	rawHeader http.Header
}

// Header returns all properties in search message.
func (s *SearchMessage) Header() http.Header {
	return s.rawHeader
}

// SearchHandler is handler of Search message.
type SearchHandler func(*SearchMessage)
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

// The data package provides the mlab-ns2 datastore sturcture.
package data

import (
	"fmt"
	"time"
)

//TODO: Data interface and Get,Put,Cache,Rm functions?
//TODO: only index the columns that are needed
//TODO: add json tags

type SliverTool struct {
	ToolID                 string    `datastore:"tool_id"`
	SliceID                string    `datastore:"slice_id"`
	SiteID                 string    `datastore:"site_id"`
	ServerID               string    `datastore:"server_id"`
	ServerPort             string    `datastore:"server_port"`
	HTTPPort               string    `datastore:"http_port"`                // For web-based tools, this is used to build the URL the client is redirected to: http://fqdn[ipv4|ipv6]:http_port
	FQDN                   string    `datastore:"fqdn"`                     // Unannotated fqdn. v4 and v6 versions can be built if necessary.
	SliverIPv4             string    `datastore:"sliver_ipv4"`              // IP addresses. Can be 'off'
	SliverIPv6             string    `datastore:"sliver_ipv6"`              // IP addresses. Can be 'off'
	StatusIPv4             string    `datastore:"status_ipv4"`              // These can have the following values: online and offline.
	StatusIPv6             string    `datastore:"status_ipv6"`              // These can have the following values: online and offline.
	UpdateRequestTimestamp int64     `datastore:"update_request_timestamp"` // To avoid an additional lookup in the datastore
	Latitude               float64   `datastore:"latitude"`                 // To avoid an additional lookup in the datastore
	Longitude              float64   `datastore:"longitude"`                // To avoid an additional lookup in the datastore
	City                   string    `datastore:"city"`                     // To avoid an additional lookup in the datastore
	Country                string    `datastore:"country"`                  // To avoid an additional lookup in the datastore
	When                   time.Time `datastore:"when"`                     // Date representing the last modification time of this entity.
}

type Site struct {
	SiteID                string    `datastore:"site_id"`
	City                  string    `datastore:"city"`
	Country               string    `datastore:"country"`
	Latitude              float64   `datastore:"latitude"`               // Latitude of the airport that uniquely identifies an M-Lab site.
	Longitude             float64   `datastore:"longitude"`              // Longitude of the airport that uniquely identifies an M-Lab site.
	Metro                 []string  `datastore:"metro"`                  // List of sites and metros, e.g., [ath, ath01].
	RegistrationTimestamp int64     `datastore:"registration_timestamp"` // Date representing the registration time (the first time a new site is added to mlab-ns).
	When                  time.Time `datastore:"when"`                   // Date representing the last modification time of this entity.
}

// MMLocation is a format that comes from pre-processed geolocation data.  It is
// intended to be used with the iptrie location map.
//
// The utility takes maximind data as input, pre-processes the data so that it
// has just the amount of detail that we need and produces a binary file that is
// uploaded to the GAE instance.  On upload the handler will copy it into the
// blobstore and update the datastore with this format.
//
//TODO: reference to the location map compression utility.
//TODO: IPv6 addresses are truncated to a /64 before inclusion?
type MMLocation struct {
	RangeStart int64 // first IP address in the block
	RangeEnd   int64 // last IP address in the block
	Latitude   int   // latitude rounded to the nearest integer
	Longitude  int   // longitude rounded to the nearest integer
}

//XXX deprecated
type MaxmindCityLocation struct {
	LocationID string    `datastore:"location_id"`
	Country    string    `datastore:"country"`
	Region     string    `datastore:"region"`
	City       string    `datastore:"city"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlock struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	LocationID string    `datastore:"location_id"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlockv6 struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	Country    string    `datastore:"country"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type CountryCode struct {
	Name        string    `datastore:"name"`
	Alpha2Code  string    `datastore:"alpha2_code"`
	Alpha3Code  string    `datastore:"alpha3_code"`
	NumericCode int64     `datastore:"numeric_code"`
	Latitude    float64   `datastore:"latitude"`
	Longitude   float64   `datastore:"longitude"`
	When        time.Time `datastore:"when"`
}

type EncryptionKey struct {
	KeyID         string `datastore:"key_id"`         // Name of the key (by default is 'admin').
	EncryptionKey string `datastore:"encryption_key"` // 16 bytes encryption key (AES).
}

type Slice struct {
	SliceID string `datastore:"slice_id"`
	ToolID  string `datastore:"tool_id"`
}

type Tool struct {
	SliceID  string `datastore:"slice_id"`
	ToolID   string `datastore:"tool_id"`
	HTTPPort string `datastore:"http_port"`
}

//TODO(gavaletz): generalize this to credentials?
type Nagios struct {
	KeyID    string `datastore:"key_id"`
	Username string `datastore:"username"`
	Password string `datastore:"password"`
	URL      string `datastore:"url"`
}

type Ping struct {
	Latitude      float64 `datastore:"latitude"`
	Longitude     float64 `datastore:"longitude"`
	ToolID        string  `datastore:"tool_id"`
	AddressFamily string  `datastore:"address_family"`
	Time          float64 `datastore:"time"`
}

func GetSliverToolID(toolID, sliceID, serverID, siteID) string {
	return fmt.Sprintf("%s-%s-%s-%s", toolID, sliceID, serverID, siteID)
}
<commit_msg>data: Fix incorrect syntax in GetSliverToolID<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

// The data package provides the mlab-ns2 datastore sturcture.
package data

import (
	"fmt"
	"time"
)

//TODO: Data interface and Get,Put,Cache,Rm functions?
//TODO: only index the columns that are needed
//TODO: add json tags

type SliverTool struct {
	ToolID                 string    `datastore:"tool_id"`
	SliceID                string    `datastore:"slice_id"`
	SiteID                 string    `datastore:"site_id"`
	ServerID               string    `datastore:"server_id"`
	ServerPort             string    `datastore:"server_port"`
	HTTPPort               string    `datastore:"http_port"`                // For web-based tools, this is used to build the URL the client is redirected to: http://fqdn[ipv4|ipv6]:http_port
	FQDN                   string    `datastore:"fqdn"`                     // Unannotated fqdn. v4 and v6 versions can be built if necessary.
	SliverIPv4             string    `datastore:"sliver_ipv4"`              // IP addresses. Can be 'off'
	SliverIPv6             string    `datastore:"sliver_ipv6"`              // IP addresses. Can be 'off'
	StatusIPv4             string    `datastore:"status_ipv4"`              // These can have the following values: online and offline.
	StatusIPv6             string    `datastore:"status_ipv6"`              // These can have the following values: online and offline.
	UpdateRequestTimestamp int64     `datastore:"update_request_timestamp"` // To avoid an additional lookup in the datastore
	Latitude               float64   `datastore:"latitude"`                 // To avoid an additional lookup in the datastore
	Longitude              float64   `datastore:"longitude"`                // To avoid an additional lookup in the datastore
	City                   string    `datastore:"city"`                     // To avoid an additional lookup in the datastore
	Country                string    `datastore:"country"`                  // To avoid an additional lookup in the datastore
	When                   time.Time `datastore:"when"`                     // Date representing the last modification time of this entity.
}

type Site struct {
	SiteID                string    `datastore:"site_id"`
	City                  string    `datastore:"city"`
	Country               string    `datastore:"country"`
	Latitude              float64   `datastore:"latitude"`               // Latitude of the airport that uniquely identifies an M-Lab site.
	Longitude             float64   `datastore:"longitude"`              // Longitude of the airport that uniquely identifies an M-Lab site.
	Metro                 []string  `datastore:"metro"`                  // List of sites and metros, e.g., [ath, ath01].
	RegistrationTimestamp int64     `datastore:"registration_timestamp"` // Date representing the registration time (the first time a new site is added to mlab-ns).
	When                  time.Time `datastore:"when"`                   // Date representing the last modification time of this entity.
}

// MMLocation is a format that comes from pre-processed geolocation data.  It is
// intended to be used with the iptrie location map.
//
// The utility takes maximind data as input, pre-processes the data so that it
// has just the amount of detail that we need and produces a binary file that is
// uploaded to the GAE instance.  On upload the handler will copy it into the
// blobstore and update the datastore with this format.
//
//TODO: reference to the location map compression utility.
//TODO: IPv6 addresses are truncated to a /64 before inclusion?
type MMLocation struct {
	RangeStart int64 // first IP address in the block
	RangeEnd   int64 // last IP address in the block
	Latitude   int   // latitude rounded to the nearest integer
	Longitude  int   // longitude rounded to the nearest integer
}

//XXX deprecated
type MaxmindCityLocation struct {
	LocationID string    `datastore:"location_id"`
	Country    string    `datastore:"country"`
	Region     string    `datastore:"region"`
	City       string    `datastore:"city"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlock struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	LocationID string    `datastore:"location_id"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlockv6 struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	Country    string    `datastore:"country"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type CountryCode struct {
	Name        string    `datastore:"name"`
	Alpha2Code  string    `datastore:"alpha2_code"`
	Alpha3Code  string    `datastore:"alpha3_code"`
	NumericCode int64     `datastore:"numeric_code"`
	Latitude    float64   `datastore:"latitude"`
	Longitude   float64   `datastore:"longitude"`
	When        time.Time `datastore:"when"`
}

type EncryptionKey struct {
	KeyID         string `datastore:"key_id"`         // Name of the key (by default is 'admin').
	EncryptionKey string `datastore:"encryption_key"` // 16 bytes encryption key (AES).
}

type Slice struct {
	SliceID string `datastore:"slice_id"`
	ToolID  string `datastore:"tool_id"`
}

type Tool struct {
	SliceID  string `datastore:"slice_id"`
	ToolID   string `datastore:"tool_id"`
	HTTPPort string `datastore:"http_port"`
}

//TODO(gavaletz): generalize this to credentials?
type Nagios struct {
	KeyID    string `datastore:"key_id"`
	Username string `datastore:"username"`
	Password string `datastore:"password"`
	URL      string `datastore:"url"`
}

type Ping struct {
	Latitude      float64 `datastore:"latitude"`
	Longitude     float64 `datastore:"longitude"`
	ToolID        string  `datastore:"tool_id"`
	AddressFamily string  `datastore:"address_family"`
	Time          float64 `datastore:"time"`
}

func GetSliverToolID(toolID, sliceID, serverID, siteID string) string {
	return fmt.Sprintf("%s-%s-%s-%s", toolID, sliceID, serverID, siteID)
}
<|endoftext|>