<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resource

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/thinkofdeath/steven/resource/internal"
)

const (
	ResourcesVersion = "1.8.4"
	vanillaURL       = "https://s3.amazonaws.com/Minecraft.Download/versions/%[1]s/%[1]s.jar"
)

var (
	lock           sync.RWMutex
	packs          []*pack
	errMissingFile = errors.New("file not found")
)

type pack struct {
	name string

	files map[string]opener
}

type opener func() (io.ReadCloser, error)

// Open searches through all open resource packs for the requested file.
// If a file exists but fails to open that error will be returned instead
// of the standard 'file not found' but only if another file couldn't be
// found to be used in its place.
func Open(plugin, name string) (io.ReadCloser, error) {
	lock.RLock()
	defer lock.RUnlock()
	var lastErr error
	for i := len(packs) - 1; i >= 0; i-- {
		pck := packs[i]
		if f, ok := pck.files[fmt.Sprintf("assets/%s/%s", plugin, name)]; ok {
			r, err := f()
			if err != nil {
				lastErr = err
				continue
			}
			return r, nil
		}
	}
	if lastErr == nil {
		return nil, errMissingFile
	}
	return nil, lastErr
}

// OpenAll searches through all open resource packs for the requested file.
// If a file exists but fails to open that error will be returned instead
// of the standard 'file not found' but only if another file couldn't be
// found to be used in its place. Unlike Open this will return all matching
// files
func OpenAll(plugin, name string) ([]io.ReadCloser, error) {
	lock.RLock()
	defer lock.RUnlock()
	var lastErr error
	var out []io.ReadCloser
	for i := len(packs) - 1; i >= 0; i-- {
		pck := packs[i]
		if f, ok := pck.files[fmt.Sprintf("assets/%s/%s", plugin, name)]; ok {
			r, err := f()
			if err != nil {
				lastErr = err
				continue
			}
			out = append(out, r)
		}
	}
	if lastErr == nil && len(out) == 0 {
		return nil, errMissingFile
	}
	return out, lastErr
}

// Search searches for files that belong to the passed plugin and exist
// the passed path with the passed extension. This searches all open packs.
func Search(plugin, path, ext string) []string {
	lock.RLock()
	defer lock.RUnlock()
	found := map[string]bool{}
	var lst []string
	search := fmt.Sprintf("assets/%s/%s", plugin, path)
	base := fmt.Sprintf("assets/%s/", plugin)
	for _, pck := range packs {
		for k := range pck.files {
			if !found[k] && strings.HasPrefix(k, search) && strings.HasSuffix(k, ext) {
				found[k] = true
				lst = append(lst, k[len(base):])
			}
		}
	}
	return lst
}

func IsActive(name string) bool {
	lock.RLock()
	defer lock.RUnlock()
	for _, pck := range packs {
		if pck.name == name {
			return true
		}
	}
	return false
}

func RemovePack(name string) {
	lock.RLock()
	defer lock.RUnlock()
	for i, pck := range packs {
		if pck.name == name {
			packs = append(packs[:i], packs[i+1:]...)
			return
		}
	}
}

// TODO(Think) Ideally this package has a way to start the download instead of
// being an init thing. Also should have a way to get progress information.

func init() {
	defLocation := fmt.Sprintf("./resources-%s", ResourcesVersion)
	_, err := os.Stat(defLocation)
	if os.IsNotExist(err) {
		downloadDefault(defLocation)
	}
	if err := fromDir(defLocation); err != nil {
		panic(err)
	}

	fromInternal()
}

type dummyCloser struct {
	*bytes.Reader
}

func (dummyCloser) Close() error { return nil }

func fromInternal() {
	lock.Lock()
	defer lock.Unlock()
	p := &pack{
		name:  "$internal",
		files: map[string]opener{},
	}
	for _, name := range internal.AssetNames() {
		name := name
		p.files[name] = func() (io.ReadCloser, error) {
			data, err := internal.Asset(name)
			return dummyCloser{bytes.NewReader(data)}, err
		}
	}
	packs = append(packs, p)
}

func fromDir(d string) error {
	lock.Lock()
	defer lock.Unlock()
	p := &pack{
		name:  d,
		files: map[string]opener{},
	}

	err := filepath.Walk(d, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		rel, err := filepath.Rel(d, path)
		if err != nil {
			return err
		}
		rel = strings.Replace(rel, string(filepath.Separator), "/", -1)
		p.files[rel] = func() (io.ReadCloser, error) {
			return os.Open(path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	packs = append(packs, p)
	return nil
}

func LoadZip(name string) error {
	f, err := os.Open(name)
	if err != nil {
		return err
	}
	return fromFile(f, name)
}

func fromFile(f *os.File, name string) error {
	lock.Lock()
	defer lock.Unlock()
	s, err := f.Stat()
	if err != nil {
		return err
	}
	z, err := zip.NewReader(f, s.Size())
	if err != nil {
		return err
	}
	p := &pack{
		name:  name,
		files: map[string]opener{},
	}
	for _, f := range z.File {
		f := f
		p.files[f.Name] = f.Open
	}
	packs = append(packs, p)
	return nil
}

func downloadDefault(target string) {
	fmt.Printf("Obtaining vanilla resources for %s, please wait...\n", ResourcesVersion)
	resp, err := http.Get(fmt.Sprintf(vanillaURL, ResourcesVersion))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	os.MkdirAll("./", 0777)
	f, err := os.Create(target + ".tmp")
	if err != nil {
		panic(err)
	}
	defer os.Remove(target + ".tmp")
	defer f.Close()
	size, err := io.Copy(f, resp.Body)
	if err != nil {
		panic(err)
	}

	f.Seek(0, 0) // Go back to the start
	fr, err := zip.NewReader(f, size)

	os.MkdirAll(target, 0777)

	// Copy the assets (not the classes) in the new zip
	for _, f := range fr.File {
		if !strings.HasPrefix(f.Name, "assets/") {
			continue
		}
		func() {
			path := filepath.Join(target, f.Name)
			os.MkdirAll(filepath.Dir(path), 0777)
			w, err := os.Create(path)
			if err != nil {
				panic(err)
			}
			defer w.Close()
			r, err := f.Open()
			if err != nil {
				panic(err)
			}
			defer r.Close()
			_, err = io.Copy(w, r)
			if err != nil {
				panic(err)
			}
		}()
	}
}
<commit_msg>resource: update to 1.8.6<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resource

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/thinkofdeath/steven/resource/internal"
)

const (
	ResourcesVersion = "1.8.6"
	vanillaURL       = "https://s3.amazonaws.com/Minecraft.Download/versions/%[1]s/%[1]s.jar"
)

var (
	lock           sync.RWMutex
	packs          []*pack
	errMissingFile = errors.New("file not found")
)

type pack struct {
	name string

	files map[string]opener
}

type opener func() (io.ReadCloser, error)

// Open searches through all open resource packs for the requested file.
// If a file exists but fails to open that error will be returned instead
// of the standard 'file not found' but only if another file couldn't be
// found to be used in its place.
func Open(plugin, name string) (io.ReadCloser, error) {
	lock.RLock()
	defer lock.RUnlock()
	var lastErr error
	for i := len(packs) - 1; i >= 0; i-- {
		pck := packs[i]
		if f, ok := pck.files[fmt.Sprintf("assets/%s/%s", plugin, name)]; ok {
			r, err := f()
			if err != nil {
				lastErr = err
				continue
			}
			return r, nil
		}
	}
	if lastErr == nil {
		return nil, errMissingFile
	}
	return nil, lastErr
}

// OpenAll searches through all open resource packs for the requested file.
// If a file exists but fails to open that error will be returned instead
// of the standard 'file not found' but only if another file couldn't be
// found to be used in its place. Unlike Open this will return all matching
// files
func OpenAll(plugin, name string) ([]io.ReadCloser, error) {
	lock.RLock()
	defer lock.RUnlock()
	var lastErr error
	var out []io.ReadCloser
	for i := len(packs) - 1; i >= 0; i-- {
		pck := packs[i]
		if f, ok := pck.files[fmt.Sprintf("assets/%s/%s", plugin, name)]; ok {
			r, err := f()
			if err != nil {
				lastErr = err
				continue
			}
			out = append(out, r)
		}
	}
	if lastErr == nil && len(out) == 0 {
		return nil, errMissingFile
	}
	return out, lastErr
}

// Search searches for files that belong to the passed plugin and exist
// the passed path with the passed extension. This searches all open packs.
func Search(plugin, path, ext string) []string {
	lock.RLock()
	defer lock.RUnlock()
	found := map[string]bool{}
	var lst []string
	search := fmt.Sprintf("assets/%s/%s", plugin, path)
	base := fmt.Sprintf("assets/%s/", plugin)
	for _, pck := range packs {
		for k := range pck.files {
			if !found[k] && strings.HasPrefix(k, search) && strings.HasSuffix(k, ext) {
				found[k] = true
				lst = append(lst, k[len(base):])
			}
		}
	}
	return lst
}

func IsActive(name string) bool {
	lock.RLock()
	defer lock.RUnlock()
	for _, pck := range packs {
		if pck.name == name {
			return true
		}
	}
	return false
}

func RemovePack(name string) {
	lock.RLock()
	defer lock.RUnlock()
	for i, pck := range packs {
		if pck.name == name {
			packs = append(packs[:i], packs[i+1:]...)
			return
		}
	}
}

// TODO(Think) Ideally this package has a way to start the download instead of
// being an init thing. Also should have a way to get progress information.

func init() {
	defLocation := fmt.Sprintf("./resources-%s", ResourcesVersion)
	_, err := os.Stat(defLocation)
	if os.IsNotExist(err) {
		downloadDefault(defLocation)
	}
	if err := fromDir(defLocation); err != nil {
		panic(err)
	}

	fromInternal()
}

type dummyCloser struct {
	*bytes.Reader
}

func (dummyCloser) Close() error { return nil }

func fromInternal() {
	lock.Lock()
	defer lock.Unlock()
	p := &pack{
		name:  "$internal",
		files: map[string]opener{},
	}
	for _, name := range internal.AssetNames() {
		name := name
		p.files[name] = func() (io.ReadCloser, error) {
			data, err := internal.Asset(name)
			return dummyCloser{bytes.NewReader(data)}, err
		}
	}
	packs = append(packs, p)
}

func fromDir(d string) error {
	lock.Lock()
	defer lock.Unlock()
	p := &pack{
		name:  d,
		files: map[string]opener{},
	}

	err := filepath.Walk(d, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		rel, err := filepath.Rel(d, path)
		if err != nil {
			return err
		}
		rel = strings.Replace(rel, string(filepath.Separator), "/", -1)
		p.files[rel] = func() (io.ReadCloser, error) {
			return os.Open(path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	packs = append(packs, p)
	return nil
}

func LoadZip(name string) error {
	f, err := os.Open(name)
	if err != nil {
		return err
	}
	return fromFile(f, name)
}

func fromFile(f *os.File, name string) error {
	lock.Lock()
	defer lock.Unlock()
	s, err := f.Stat()
	if err != nil {
		return err
	}
	z, err := zip.NewReader(f, s.Size())
	if err != nil {
		return err
	}
	p := &pack{
		name:  name,
		files: map[string]opener{},
	}
	for _, f := range z.File {
		f := f
		p.files[f.Name] = f.Open
	}
	packs = append(packs, p)
	return nil
}

func downloadDefault(target string) {
	fmt.Printf("Obtaining vanilla resources for %s, please wait...\n", ResourcesVersion)
	resp, err := http.Get(fmt.Sprintf(vanillaURL, ResourcesVersion))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	os.MkdirAll("./", 0777)
	f, err := os.Create(target + ".tmp")
	if err != nil {
		panic(err)
	}
	defer os.Remove(target + ".tmp")
	defer f.Close()
	size, err := io.Copy(f, resp.Body)
	if err != nil {
		panic(err)
	}

	f.Seek(0, 0) // Go back to the start
	fr, err := zip.NewReader(f, size)

	os.MkdirAll(target, 0777)

	// Copy the assets (not the classes) in the new zip
	for _, f := range fr.File {
		if !strings.HasPrefix(f.Name, "assets/") {
			continue
		}
		func() {
			path := filepath.Join(target, f.Name)
			os.MkdirAll(filepath.Dir(path), 0777)
			w, err := os.Create(path)
			if err != nil {
				panic(err)
			}
			defer w.Close()
			r, err := f.Open()
			if err != nil {
				panic(err)
			}
			defer r.Close()
			_, err = io.Copy(w, r)
			if err != nil {
				panic(err)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package teardown

import (
	"bufio"
	"fmt"
	"log"
	"os"
)

type RequestStatus int
type ResponseStatus int

const (
	Ack RequestStatus = iota
	Unknown
	Fail

	Yes ResponseStatus = iota
	No
	Maybe
)

type Request struct {
	Status   RequestStatus
	Response ResponseStatus
	Key      string
	Value    string
}

type TestRunner struct {
	requests []Request
	client   Client
	count    int
}

func NewTestRunner(client Client) *TestRunner {
	t := TestRunner{
		requests: []Request{},
		client:   client,
	}

	return &t
}

func (t *TestRunner) Step() error {
	request := Request{
		Key:   fmt.Sprintf("/%d", t.count),
		Value: "hi!",
	}

	err := t.client.Put(request.Key, request.Value)

	if err != nil {
		request.Status = Fail
	} else {
		request.Status = Ack
	}

	value, err := t.client.Get(request.Key)

	if err != nil {
		request.Response = No
	} else {
		request.Response = Yes
	}

	log.Println("Error", err)
	log.Println("Get response", value)

	t.requests = append(t.requests, request)
	t.count++

	return nil
}

func (t *TestRunner) Finalize() {
	successfulWrites := 0
	correctFailures := 0
	missingWrites := 0
	extraWrites := 0

	for _, request := range t.requests {
		if request.Status == Ack && request.Response == Yes {
			successfulWrites++
		}
		if request.Status == Fail && request.Response == Yes {
			extraWrites++
		}
		if request.Status == Fail && request.Response == No {
			correctFailures++
		}
		if request.Status == Ack && request.Response == No {
			missingWrites++
		}
	}

	log.Println("Successful writes:", successfulWrites)
	log.Println("Correct failures:", correctFailures)
	log.Println("Missing writes:", missingWrites)
	log.Println("Extra writes", extraWrites)
}

func RunTests(cluster Cluster, client Client) {
	cluster.Setup()
	tests := NewTestRunner(client)

	bio := bufio.NewReader(os.Stdin)
	for i := 0; i < 100; i++ {
		bio.ReadLine()
		log.Println("Next")
		tests.Step()
	}

	tests.Finalize()

	cluster.Teardown()
}
<commit_msg>Refactor test runner<commit_after>package teardown

import (
	"bufio"
	"fmt"
	"log"
	"os"
)

type RequestStatus int
type ResponseStatus int

const (
	Ack RequestStatus = iota
	Unknown
	Fail

	Yes ResponseStatus = iota
	No
	Maybe
)

type Request struct {
	Status   RequestStatus
	Response ResponseStatus
	Key      string
	Value    string
}

type TestRunner struct {
	requests []Request
	client   Client
	cluster  Cluster
	count    int
}

func (t *TestRunner) Step() error {
	request := Request{
		Key:   fmt.Sprintf("/%d", t.count),
		Value: "hi!",
	}

	err := t.client.Put(request.Key, request.Value)

	if err != nil {
		request.Status = Fail
	} else {
		request.Status = Ack
	}

	value, err := t.client.Get(request.Key)

	if err != nil {
		request.Response = No
	} else {
		request.Response = Yes
	}

	log.Println("Error", err)
	log.Println("Get response", value)

	t.requests = append(t.requests, request)
	t.count++

	return nil
}

func (t *TestRunner) Report() {
	successfulWrites := 0
	correctFailures := 0
	missingWrites := 0
	extraWrites := 0

	for _, request := range t.requests {
		if request.Status == Ack && request.Response == Yes {
			successfulWrites++
		}
		if request.Status == Fail && request.Response == Yes {
			extraWrites++
		}
		if request.Status == Fail && request.Response == No {
			correctFailures++
		}
		if request.Status == Ack && request.Response == No {
			missingWrites++
		}
	}

	log.Println("Successful writes:", successfulWrites)
	log.Println("Correct failures:", correctFailures)
	log.Println("Missing writes:", missingWrites)
	log.Println("Extra writes", extraWrites)
}

func (t *TestRunner) Run() {
	t.cluster.Setup()

	bio := bufio.NewReader(os.Stdin)
	for i := 0; i < 100; i++ {
		bio.ReadLine()
		log.Println("Next")
		t.Step()
	}

	t.Report()
	t.cluster.Teardown()
}

func NewTestRunner(cluster Cluster, client Client) *TestRunner {
	t := TestRunner{
		requests: []Request{},
		client:   client,
		cluster:  cluster,
	}

	return &t
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

var offset = flag.Int64("s", -1, "The offset to read from the pack file")
var verbose = flag.Bool("t", false, "Output verbose information")

func main() {
	flag.Parse()
	f, err := GetArgInputFile()
	if err != nil {
		panic(err)
	}
	p, err := ReadPackedObjectAtOffset(*offset, f)
	if err != nil {
		panic(err)
	}
	if *verbose {
		fmt.Fprintf(os.Stdout, "Object at [%d] => Type: %s, Size: %d\n", *offset, p.objectType, p.size)
		fmt.Fprintf(os.Stdout, "  ObjRef: %s, NegativeOffset: %d\n", p.hashOfRef, p.negOffsetOfRef)
		fmt.Fprintf(os.Stdout, "  Data(starts below):\n")
	}
	fmt.Fprintf(os.Stdout, "%s", p.data)
}
<commit_msg>Test commit<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
)

var offset = flag.Int64("s", -1, "The offset to read from the pack file")
var verbose = flag.Bool("t", false, "Output verbose information")

func main() {
	flag.Parse()
	f, err := GetArgInputFile()
	if err != nil {
		panic(err)
	}
	p, err := ReadPackedObjectAtOffset(*offset, f)
	if err != nil {
		panic(err)
	}
	if *verbose {
		fmt.Fprintf(os.Stdout, "Object at [%d] => Type: %s, Size: %d\n", *offset, p.objectType, p.size)
		fmt.Fprintf(os.Stdout, "  ObjRef: %s, NegativeOffset: %d\n", p.hashOfRef, p.negOffsetOfRef)
		fmt.Fprintf(os.Stdout, "  Data(starts below):\n")
	}
	fmt.Fprintf(os.Stdout, "%s", p.data)
}

<|endoftext|>


<commit_before>package brocadevtm

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/sky-uk/go-brocade-vtm/api"
	"github.com/sky-uk/go-brocade-vtm/api/model/3.8/location"
	"github.com/sky-uk/terraform-provider-brocadevtm/brocadevtm/util"
	"net/http"
)

func resourceLocation() *schema.Resource {
	return &schema.Resource{
		Create: resourceLocationCreate,
		Read:   resourceLocationRead,
		Update: resourceLocationUpdate,
		Delete: resourceLocationDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "Unique name of the location",
				ForceNew:    true,
			},
			"location_id": {
				Type:         schema.TypeInt,
				Required:     true,
				Description:  "The location identifier",
				ValidateFunc: util.ValidateUnsignedInteger,
			},
			"latitude": {
				Type:         schema.TypeFloat,
				Optional:     true,
				Default:      0.0,
				Description:  "The latitude of the location",
				ValidateFunc: checkLatitudeWithinRange,
			},
			"longitude": {
				Type:         schema.TypeFloat,
				Optional:     true,
				Default:      0.0,
				Description:  "The longitude of the location",
				ValidateFunc: checkLongitudeWithinRange,
			},
			"note": {
				Type:        schema.TypeString,
				Optional:    true,
				Computed:    true,
				Description: "A note regarding the location",
			},
			"type": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				Description:  "Is the location used by traffic managers or GLBs?",
				ValidateFunc: checkValidLocationType,
			},
		},
	}
}

func checkValidLocationType(v interface{}, k string) (ws []string, errors []error) {
	locationType := v.(string)
	if locationType != "config" && locationType != "glb" {
		errors = append(errors, fmt.Errorf("%q must be one of config or glb", k))
	}
	return
}

func checkLatitudeWithinRange(v interface{}, k string) (ws []string, errors []error) {
	latitude := v.(float64)
	if latitude < -90 || latitude > 90 {
		errors = append(errors, fmt.Errorf("%q must be between -90 and 90 degrees inclusive", k))
	}
	return
}

func checkLongitudeWithinRange(v interface{}, k string) (ws []string, errors []error) {
	longitude := v.(float64)
	if longitude < -180 || longitude > 180 {
		errors = append(errors, fmt.Errorf("%q must be between -180 and 180 degrees inclusive", k))
	}
	return
}

func resourceLocationCreate(d *schema.ResourceData, m interface{}) error {

	var createLocation location.Location
	var name string
	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)

	if v, ok := d.GetOk("name"); ok && v != "" {
		name = v.(string)
	}
	if v, ok := d.GetOk("location_id"); ok {
		locationID := v.(int)
		createLocation.Properties.Basic.ID = uint(locationID)
	}
	if v, ok := d.GetOk("latitude"); ok {
		createLocation.Properties.Basic.Latitude = v.(float64)
	}
	if v, ok := d.GetOk("longitude"); ok {
		createLocation.Properties.Basic.Longitude = v.(float64)
	}
	if v, ok := d.GetOk("note"); ok && v != "" {
		createLocation.Properties.Basic.Note = v.(string)
	}
	if v, ok := d.GetOk("type"); ok && v != "" {
		createLocation.Properties.Basic.Type = v.(string)
	}

	err := client.Set("locations", name, createLocation, nil)
	if err != nil {
		return fmt.Errorf("BrocadeVTM Location error whilst creating %s: %v", name, err)
	}

	d.SetId(name)
	return resourceLocationRead(d, m)
}

func resourceLocationRead(d *schema.ResourceData, m interface{}) error {

	var readLocation location.Location
	var name string
	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)

	if v, ok := d.GetOk("name"); ok {
		name = v.(string)
	}

	client.WorkWithConfigurationResources()
	err := client.GetByName("locations", name, &readLocation)
	if client.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}
	if err != nil {
		d.SetId("")
		return fmt.Errorf("BrocadeVTM location error whilst retrieving %s: %v", name, err)
	}

	d.Set("name", name)
	d.Set("location_id", readLocation.Properties.Basic.ID)
	d.Set("latitude", readLocation.Properties.Basic.Latitude)
	d.Set("longitude", readLocation.Properties.Basic.Longitude)
	d.Set("note", readLocation.Properties.Basic.Note)
	d.Set("type", readLocation.Properties.Basic.Type)

	return nil
}

func resourceLocationUpdate(d *schema.ResourceData, m interface{}) error {

	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)
	var updateLocation location.Location
	var name string
	hasChanges := false

	if v, ok := d.GetOk("name"); ok {
		name = v.(string)
	}

	if d.HasChange("location_id") {
		if v, ok := d.GetOk("location_id"); ok {
			locationID := v.(int)
			updateLocation.Properties.Basic.ID = uint(locationID)
		}
		hasChanges = true
	}
	if d.HasChange("latitude") {
		if v, ok := d.GetOk("latitude"); ok {
			updateLocation.Properties.Basic.Latitude = v.(float64)
		}
		hasChanges = true
	}
	if d.HasChange("longitude") {
		if v, ok := d.GetOk("longitude"); ok {
			updateLocation.Properties.Basic.Longitude = v.(float64)
		}
		hasChanges = true
	}
	if d.HasChange("note") {
		if v, ok := d.GetOk("note"); ok && v != "" {
			updateLocation.Properties.Basic.Note = v.(string)
		}
		hasChanges = true
	}
	if d.HasChange("type") {
		if v, ok := d.GetOk("type"); ok && v != "" {
			updateLocation.Properties.Basic.Type = v.(string)
		}
		hasChanges = true
	}

	if hasChanges {

		err := client.Set("locations", name, updateLocation, nil)
		if err != nil {
			return fmt.Errorf("BrocadeVTM locations error whilst updating %s: %v", name, err)
		}
	}
	return resourceLocationRead(d, m)

}

func resourceLocationDelete(d *schema.ResourceData, m interface{}) error {
	return DeleteResource("locations", d, m)
}
<commit_msg>Refactored location resource to use maps<commit_after>package brocadevtm

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/sky-uk/go-brocade-vtm/api"
	"github.com/sky-uk/terraform-provider-brocadevtm/brocadevtm/util"
	"net/http"
)

func resourceLocation() *schema.Resource {
	return &schema.Resource{
		Create: resourceLocationCreate,
		Read:   resourceLocationRead,
		Update: resourceLocationUpdate,
		Delete: resourceLocationDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "Unique name of the location",
				ForceNew:    true,
			},
			"location_id": {
				Type:         schema.TypeInt,
				Required:     true,
				Description:  "The location identifier",
				ValidateFunc: util.ValidateUnsignedInteger,
			},
			"latitude": {
				Type:         schema.TypeFloat,
				Optional:     true,
				Default:      0.0,
				Description:  "The latitude of the location",
				ValidateFunc: checkLatitudeWithinRange,
			},
			"longitude": {
				Type:         schema.TypeFloat,
				Optional:     true,
				Default:      0.0,
				Description:  "The longitude of the location",
				ValidateFunc: checkLongitudeWithinRange,
			},
			"note": {
				Type:        schema.TypeString,
				Optional:    true,
				Computed:    true,
				Description: "A note regarding the location",
			},
			"type": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				Description:  "Is the location used by traffic managers or GLBs?",
				ValidateFunc: checkValidLocationType,
			},
		},
	}
}

func checkValidLocationType(v interface{}, k string) (ws []string, errors []error) {
	locationType := v.(string)
	if locationType != "config" && locationType != "glb" {
		errors = append(errors, fmt.Errorf("%q must be one of config or glb", k))
	}
	return
}

func checkLatitudeWithinRange(v interface{}, k string) (ws []string, errors []error) {
	latitude := v.(float64)
	if latitude < -90 || latitude > 90 {
		errors = append(errors, fmt.Errorf("%q must be between -90 and 90 degrees inclusive", k))
	}
	return
}

func checkLongitudeWithinRange(v interface{}, k string) (ws []string, errors []error) {
	longitude := v.(float64)
	if longitude < -180 || longitude > 180 {
		errors = append(errors, fmt.Errorf("%q must be between -180 and 180 degrees inclusive", k))
	}
	return
}

func resourceLocationCreate(d *schema.ResourceData, m interface{}) error {

	var name string
	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)

	locationBasicConfiguration := make(map[string]interface{})
	locationPropertiesConfiguration := make(map[string]interface{})
	locationConfiguration := make(map[string]interface{})

	if v, ok := d.GetOk("name"); ok && v != "" {
		name = v.(string)
	}
	if v, ok := d.GetOk("location_id"); ok {
		locationBasicConfiguration["id"] = uint(v.(int))
	}
	if v, ok := d.GetOk("latitude"); ok {
		locationBasicConfiguration["latitude"] = v.(float64)
	}
	if v, ok := d.GetOk("longitude"); ok {
		locationBasicConfiguration["longitude"] = v.(float64)
	}
	if v, ok := d.GetOk("note"); ok && v != "" {
		locationBasicConfiguration["note"] = v.(string)
	}
	if v, ok := d.GetOk("type"); ok && v != "" {
		locationBasicConfiguration["type"] = v.(string)
	}

	locationPropertiesConfiguration["basic"] = locationBasicConfiguration
	locationConfiguration["properties"] = locationPropertiesConfiguration

	err := client.Set("locations", name, locationConfiguration, nil)
	if err != nil {
		return fmt.Errorf("BrocadeVTM Location error whilst creating %s: %v", name, err)
	}

	d.SetId(name)
	return resourceLocationRead(d, m)
}

func resourceLocationRead(d *schema.ResourceData, m interface{}) error {

	name := d.Id()
	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)

	locationBasicConfiguration := make(map[string]interface{})
	locationPropertiesConfiguration := make(map[string]interface{})
	locationConfiguration := make(map[string]interface{})

	client.WorkWithConfigurationResources()
	err := client.GetByName("locations", name, &locationConfiguration)
	if client.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}
	if err != nil {
		d.SetId("")
		return fmt.Errorf("BrocadeVTM location error whilst retrieving %s: %v", name, err)
	}
	locationPropertiesConfiguration = locationConfiguration["properties"].(map[string]interface{})
	locationBasicConfiguration = locationPropertiesConfiguration["basic"].(map[string]interface{})

	d.Set("location_id", locationBasicConfiguration["id"])
	d.Set("latitude", locationBasicConfiguration["latitude"])
	d.Set("longitude", locationBasicConfiguration["longitude"])
	d.Set("note", locationBasicConfiguration["note"])
	d.Set("type", locationBasicConfiguration["type"])

	return nil
}

func resourceLocationUpdate(d *schema.ResourceData, m interface{}) error {

	config := m.(map[string]interface{})
	client := config["jsonClient"].(*api.Client)
	var name string

	locationBasicConfiguration := make(map[string]interface{})
	locationPropertiesConfiguration := make(map[string]interface{})
	locationConfiguration := make(map[string]interface{})

	if v, ok := d.GetOk("name"); ok {
		name = v.(string)
	}
	if d.HasChange("location_id") {
		locationBasicConfiguration["id"] = uint(d.Get("location_id").(int))
	}
	if d.HasChange("latitude") {
		locationBasicConfiguration["latitude"] = d.Get("latitude").(float64)
	}
	if d.HasChange("longitude") {
		locationBasicConfiguration["longitude"] = d.Get("longitude").(float64)
	}
	if d.HasChange("note") {
		locationBasicConfiguration["note"] = d.Get("note").(string)
	}
	if d.HasChange("type") {
		locationBasicConfiguration["type"] = d.Get("type").(string)
	}

	locationPropertiesConfiguration["basic"] = locationBasicConfiguration
	locationConfiguration["properties"] = locationPropertiesConfiguration

	err := client.Set("locations", name, locationConfiguration, nil)
	if err != nil {
		return fmt.Errorf("BrocadeVTM locations error whilst updating %s: %v", name, err)
	}

	return resourceLocationRead(d, m)
}

func resourceLocationDelete(d *schema.ResourceData, m interface{}) error {
	return DeleteResource("locations", d, m)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcetasks

import (
	"fmt"
	"github.com/golang/glog"
	compute "google.golang.org/api/compute/v0.beta"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)

//go:generate fitask -type=Address
type Address struct {
	Name      *string
	Lifecycle *fi.Lifecycle

	IPAddress *string
}

func (e *Address) Find(c *fi.Context) (*Address, error) {
	actual, err := e.find(c.Cloud.(gce.GCECloud))
	if actual != nil && err == nil {
		if e.IPAddress == nil {
			e.IPAddress = actual.IPAddress
		}
	}
	return actual, err
}

func findAddressByIP(cloud gce.GCECloud, ip string) (*Address, error) {
	// Technically this is a regex, but it doesn't matter...
	r, err := cloud.Compute().Addresses.List(cloud.Project(), cloud.Region()).Filter("address eq " + ip).Do()
	if err != nil {
		return nil, fmt.Errorf("error listing IPAddresss: %v", err)
	}

	if len(r.Items) == 0 {
		return nil, nil
	}
	if len(r.Items) > 1 {
		return nil, fmt.Errorf("found multiple Addresses matching %q", ip)
	}

	actual := &Address{}
	actual.IPAddress = &r.Items[0].Address
	actual.Name = &r.Items[0].Name

	return actual, nil
}

func (e *Address) find(cloud gce.GCECloud) (*Address, error) {
	r, err := cloud.Compute().Addresses.Get(cloud.Project(), cloud.Region(), *e.Name).Do()
	if err != nil {
		if gce.IsNotFound(err) {
			return nil, nil
		}

		return nil, fmt.Errorf("error listing IPAddresss: %v", err)
	}

	actual := &Address{}
	actual.IPAddress = &r.Address
	actual.Name = &r.Name

	return actual, nil
}

var _ fi.HasAddress = &Address{}

func (e *Address) FindIPAddress(context *fi.Context) (*string, error) {
	actual, err := e.find(context.Cloud.(gce.GCECloud))
	if err != nil {
		return nil, fmt.Errorf("error querying for IPAddress: %v", err)
	}
	if actual == nil {
		return nil, nil
	}
	return actual.IPAddress, nil
}

func (e *Address) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (_ *Address) CheckChanges(a, e, changes *Address) error {
	if a != nil {
		if changes.Name != nil {
			return fi.CannotChangeField("Name")
		}
		if changes.IPAddress != nil {
			return fi.CannotChangeField("Address")
		}
	}
	return nil
}

func (_ *Address) RenderGCE(t gce.GCEAPITarget, a, e, changes *Address) error {
	cloud := t.Cloud
	addr := &compute.Address{
		Name:    *e.Name,
		Address: fi.StringValue(e.IPAddress),
		Region:  cloud.Region(),
	}

	if a == nil {
		glog.Infof("GCE creating address: %q", addr.Name)

		op, err := cloud.Compute().Addresses.Insert(cloud.Project(), cloud.Region(), addr).Do()
		if err != nil {
			return fmt.Errorf("error creating IPAddress: %v", err)
		}

		if err := cloud.WaitForOp(op); err != nil {
			return fmt.Errorf("error waiting for IPAddress: %v", err)
		}
	} else {
		return fmt.Errorf("Cannot apply changes to IPAddress: %v", changes)
	}

	return nil
}

type terraformAddress struct {
	Name *string `json:"name,omitempty"`
}

func (_ *Address) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Address) error {
	tf := &terraformAddress{
		Name: e.Name,
	}
	return t.RenderResource("google_compute_address", *e.Name, tf)
}

func (e *Address) TerraformAddress() *terraform.Literal {
	name := fi.StringValue(e.Name)

	return terraform.LiteralProperty("google_compute_address", name, "address")
}
<commit_msg>Fix RenderGCE issue on Address<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcetasks

import (
	"fmt"
	"github.com/golang/glog"
	compute "google.golang.org/api/compute/v0.beta"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)

//go:generate fitask -type=Address
type Address struct {
	Name      *string
	Lifecycle *fi.Lifecycle

	IPAddress *string
}

func (e *Address) Find(c *fi.Context) (*Address, error) {
	actual, err := e.find(c.Cloud.(gce.GCECloud))
	if actual != nil && err == nil {
		if e.IPAddress == nil {
			e.IPAddress = actual.IPAddress
		}
	}
	return actual, err
}

func findAddressByIP(cloud gce.GCECloud, ip string) (*Address, error) {
	// Technically this is a regex, but it doesn't matter...
	r, err := cloud.Compute().Addresses.List(cloud.Project(), cloud.Region()).Filter("address eq " + ip).Do()
	if err != nil {
		return nil, fmt.Errorf("error listing IPAddresss: %v", err)
	}

	if len(r.Items) == 0 {
		return nil, nil
	}
	if len(r.Items) > 1 {
		return nil, fmt.Errorf("found multiple Addresses matching %q", ip)
	}

	actual := &Address{}
	actual.IPAddress = &r.Items[0].Address
	actual.Name = &r.Items[0].Name

	return actual, nil
}

func (e *Address) find(cloud gce.GCECloud) (*Address, error) {
	r, err := cloud.Compute().Addresses.Get(cloud.Project(), cloud.Region(), *e.Name).Do()
	if err != nil {
		if gce.IsNotFound(err) {
			return nil, nil
		}

		return nil, fmt.Errorf("error listing IPAddresss: %v", err)
	}

	actual := &Address{}
	actual.IPAddress = &r.Address
	actual.Name = &r.Name

	return actual, nil
}

var _ fi.HasAddress = &Address{}

func (e *Address) FindIPAddress(context *fi.Context) (*string, error) {
	actual, err := e.find(context.Cloud.(gce.GCECloud))
	if err != nil {
		return nil, fmt.Errorf("error querying for IPAddress: %v", err)
	}
	if actual == nil {
		return nil, nil
	}
	return actual.IPAddress, nil
}

func (e *Address) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (_ *Address) CheckChanges(a, e, changes *Address) error {
	if a != nil {
		if changes.Name != nil {
			return fi.CannotChangeField("Name")
		}
		if changes.IPAddress != nil {
			return fi.CannotChangeField("Address")
		}
	}
	return nil
}

func (_ *Address) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Address) error {
	cloud := t.Cloud
	addr := &compute.Address{
		Name:    *e.Name,
		Address: fi.StringValue(e.IPAddress),
		Region:  cloud.Region(),
	}

	if a == nil {
		glog.Infof("GCE creating address: %q", addr.Name)

		op, err := cloud.Compute().Addresses.Insert(cloud.Project(), cloud.Region(), addr).Do()
		if err != nil {
			return fmt.Errorf("error creating IPAddress: %v", err)
		}

		if err := cloud.WaitForOp(op); err != nil {
			return fmt.Errorf("error waiting for IPAddress: %v", err)
		}
	} else {
		return fmt.Errorf("Cannot apply changes to IPAddress: %v", changes)
	}

	return nil
}

type terraformAddress struct {
	Name *string `json:"name,omitempty"`
}

func (_ *Address) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Address) error {
	tf := &terraformAddress{
		Name: e.Name,
	}
	return t.RenderResource("google_compute_address", *e.Name, tf)
}

func (e *Address) TerraformAddress() *terraform.Literal {
	name := fi.StringValue(e.Name)

	return terraform.LiteralProperty("google_compute_address", name, "address")
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/api/v1/models"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedKafkaPolicyTest", func() {

	var microscopeErr error
	var microscopeCancel func() error
	var kubectl *helpers.Kubectl
	var ciliumPod string

	var (
		logger      = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		l7Policy    = helpers.ManifestGet("kafka-sw-security-policy.yaml")
		demoPath    = helpers.ManifestGet("kafka-sw-app.yaml")
		kafkaApp    = "kafka"
		zookApp     = "zook"
		backupApp   = "empire-backup"
		empireHqApp = "empire-hq"
		outpostApp  = "empire-outpost"
		apps        = []string{kafkaApp, zookApp, backupApp, empireHqApp, outpostApp}
		appPods     = map[string]string{}

		prodHqAnnounce    = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		conOutpostAnnoune = `-c "./kafka-consume.sh --topic empire-announce --from-beginning --max-messages 1"`
		prodHqDeathStar   = `-c "echo 'deathstar reactor design v3' | ./kafka-produce.sh --topic deathstar-plans"`
		conOutDeathStar   = `-c "./kafka-consume.sh --topic deathstar-plans --from-beginning --max-messages 1"`
		prodBackAnnounce  = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		prodOutAnnounce   = `-c "echo 'Vader Booed at Empire Karaoke Party' | ./kafka-produce.sh --topic empire-announce"`
	)

	BeforeAll(func() {
		logger = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		kubectl.Apply(helpers.ManifestGet("cilium_ds.yaml"))
		status, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 300)
		Expect(status).Should(BeTrue(), "Cilium is not ready after timeout")
		Expect(err).Should(BeNil(), "Cilium is not ready after timeout")

		err = kubectl.WaitKubeDNS()
		Expect(err).Should(BeNil(), "DNS is not ready after timeout")

		kubectl.Apply(demoPath)
		_, err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=kafkaTestApp", 300)
		Expect(err).Should(BeNil(), "Kafka Pods are not ready after timeout")

		appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "app")

		ciliumPod, err = kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s2)
		Expect(err).To(BeNil(), "Cannot get cilium Pod")

	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium service list",
			"cilium policy get",
			"cilium endpoint list")
	})

	JustBeforeEach(func() {
		microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
		Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
	})

	AfterEach(func() {
		// On aftereach don't make assertions to delete all.
		_ = kubectl.Delete(demoPath)
		_ = kubectl.Delete(l7Policy)

		err := kubectl.WaitCleanAllTerminatingPods()
		Expect(err).To(BeNil(), "Terminating containers are not deleted after timeout")

	})

	It("KafkaPolicies", func() {

		By("Testing basic Kafka Produce and Consume")
		// We need to produce first, since consumer script waits for
		// some messages to be already there by the producer.

		err := kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to backup on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to outpost on topic empire-announce")

		By(fmt.Sprintf("Waiting for CEP to exist for %s", appPods[kafkaApp]))
		err = kubectl.WaitForCEPToExist(appPods[kafkaApp], helpers.DefaultNamespace)
		Expect(err).To(BeNil(), "CEP did not get created for %s", appPods[kafkaApp])

		By("Getting policy revision number for each endpoint")
		cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[kafkaApp])
		Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", kafkaApp, appPods[kafkaApp])
		kafkaRevBeforeUpdate := cep.Status.Policy.Realized.PolicyRevision

		By("Apply L7 kafka policy and wait")

		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7Policy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "L7 policy cannot be imported correctly")

		By("validate that the pods have the correct policy")

		err = kubectl.WaitCEPRevisionIncrease(appPods[kafkaApp], helpers.DefaultNamespace, kafkaRevBeforeUpdate)

		desiredPolicyStatus := map[string]models.EndpointPolicyEnabled{
			backupApp:   models.EndpointPolicyEnabledNone,
			empireHqApp: models.EndpointPolicyEnabledNone,
			kafkaApp:    models.EndpointPolicyEnabledIngress,
			outpostApp:  models.EndpointPolicyEnabledNone,
			zookApp:     models.EndpointPolicyEnabledNone,
		}

		for app, policy := range desiredPolicyStatus {
			cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[app])
			Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", app, appPods[app])
			Expect(cep.Status.Policy.Spec.PolicyEnabled).To(Equal(policy), "Policy for %q mismatch", app)
		}

		By("Validating Policy trace")
		trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[backupApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: DENIED")

		By("Testing Kafka L7 policy enforcement status")
		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce from empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(HaveOccurred(), " Produce to backup on topic empire-announce should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(HaveOccurred(), " Consume from outpost on topic deathstar-plans should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(HaveOccurred(), "Produce to outpost on topic empire-announce should have been denied")
	})
})
<commit_msg>test/k8sT: have KafkaPolicies test use assertion helpers<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/api/v1/models"
	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedKafkaPolicyTest", func() {

	var microscopeErr error
	var microscopeCancel func() error
	var kubectl *helpers.Kubectl
	var ciliumPod string

	var (
		logger      = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		l7Policy    = helpers.ManifestGet("kafka-sw-security-policy.yaml")
		demoPath    = helpers.ManifestGet("kafka-sw-app.yaml")
		kafkaApp    = "kafka"
		zookApp     = "zook"
		backupApp   = "empire-backup"
		empireHqApp = "empire-hq"
		outpostApp  = "empire-outpost"
		apps        = []string{kafkaApp, zookApp, backupApp, empireHqApp, outpostApp}
		appPods     = map[string]string{}

		prodHqAnnounce    = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		conOutpostAnnoune = `-c "./kafka-consume.sh --topic empire-announce --from-beginning --max-messages 1"`
		prodHqDeathStar   = `-c "echo 'deathstar reactor design v3' | ./kafka-produce.sh --topic deathstar-plans"`
		conOutDeathStar   = `-c "./kafka-consume.sh --topic deathstar-plans --from-beginning --max-messages 1"`
		prodBackAnnounce  = `-c "echo 'Happy 40th Birthday to General Tagge' | ./kafka-produce.sh --topic empire-announce"`
		prodOutAnnounce   = `-c "echo 'Vader Booed at Empire Karaoke Party' | ./kafka-produce.sh --topic empire-announce"`
	)

	BeforeAll(func() {
		logger = log.WithFields(logrus.Fields{"testName": "K8sValidatedKafkaPolicyTest"})
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		kubectl.Apply(helpers.ManifestGet("cilium_ds.yaml"))
		ExpectCiliumReady(kubectl)
		ExpectKubeDNSReady(kubectl)

		kubectl.Apply(demoPath)
		_, err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=kafkaTestApp", 300)
		Expect(err).Should(BeNil(), "Kafka Pods are not ready after timeout")

		appPods = helpers.GetAppPods(apps, helpers.DefaultNamespace, kubectl, "app")

		ciliumPod, err = kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s2)
		Expect(err).To(BeNil(), "Cannot get cilium Pod")

	})

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace,
			"cilium service list",
			"cilium policy get",
			"cilium endpoint list")
	})

	JustBeforeEach(func() {
		microscopeErr, microscopeCancel = kubectl.MicroscopeStart()
		Expect(microscopeErr).To(BeNil(), "Microscope cannot be started")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(microscopeCancel()).To(BeNil(), "cannot stop microscope")
	})

	AfterEach(func() {
		// On aftereach don't make assertions to delete all.
		_ = kubectl.Delete(demoPath)
		_ = kubectl.Delete(l7Policy)

		err := kubectl.WaitCleanAllTerminatingPods()
		Expect(err).To(BeNil(), "Terminating containers are not deleted after timeout")

	})

	It("KafkaPolicies", func() {

		By("Testing basic Kafka Produce and Consume")
		// We need to produce first, since consumer script waits for
		// some messages to be already there by the producer.

		err := kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to backup on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to outpost on topic empire-announce")

		By(fmt.Sprintf("Waiting for CEP to exist for %s", appPods[kafkaApp]))
		err = kubectl.WaitForCEPToExist(appPods[kafkaApp], helpers.DefaultNamespace)
		Expect(err).To(BeNil(), "CEP did not get created for %s", appPods[kafkaApp])

		By("Getting policy revision number for each endpoint")
		cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[kafkaApp])
		Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", kafkaApp, appPods[kafkaApp])
		kafkaRevBeforeUpdate := cep.Status.Policy.Realized.PolicyRevision

		By("Apply L7 kafka policy and wait")

		_, err = kubectl.CiliumPolicyAction(
			helpers.KubeSystemNamespace, l7Policy,
			helpers.KubectlApply, helpers.HelperTimeout)
		Expect(err).To(BeNil(), "L7 policy cannot be imported correctly")

		By("validate that the pods have the correct policy")

		err = kubectl.WaitCEPRevisionIncrease(appPods[kafkaApp], helpers.DefaultNamespace, kafkaRevBeforeUpdate)

		desiredPolicyStatus := map[string]models.EndpointPolicyEnabled{
			backupApp:   models.EndpointPolicyEnabledNone,
			empireHqApp: models.EndpointPolicyEnabledNone,
			kafkaApp:    models.EndpointPolicyEnabledIngress,
			outpostApp:  models.EndpointPolicyEnabledNone,
			zookApp:     models.EndpointPolicyEnabledNone,
		}

		for app, policy := range desiredPolicyStatus {
			cep := kubectl.CepGet(helpers.DefaultNamespace, appPods[app])
			Expect(cep).ToNot(BeNil(), "cannot get cep for app %q and pod %s", app, appPods[app])
			Expect(cep.Status.Policy.Spec.PolicyEnabled).To(Equal(policy), "Policy for %q mismatch", app)
		}

		By("Validating Policy trace")
		trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 9092",
			appPods[backupApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: ALLOWED")

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
			appPods[empireHqApp], appPods[kafkaApp]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		trace.ExpectContains("Final verdict: DENIED")

		By("Testing Kafka L7 policy enforcement status")
		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqAnnounce))
		Expect(err).Should(BeNil(), "Failed to produce to empire-hq on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[empireHqApp], fmt.Sprintf(prodHqDeathStar))
		Expect(err).Should(BeNil(), "Failed to produce from empire-hq on topic deathstar-plans")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutpostAnnoune))
		Expect(err).Should(BeNil(), "Failed to consume from outpost on topic empire-announce")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[backupApp], fmt.Sprintf(prodBackAnnounce))
		Expect(err).Should(HaveOccurred(), " Produce to backup on topic empire-announce should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(conOutDeathStar))
		Expect(err).Should(HaveOccurred(), " Consume from outpost on topic deathstar-plans should have been denied")

		err = kubectl.ExecKafkaPodCmd(
			helpers.DefaultNamespace, appPods[outpostApp], fmt.Sprintf(prodOutAnnounce))
		Expect(err).Should(HaveOccurred(), "Produce to outpost on topic empire-announce should have been denied")
	})
})
<|endoftext|>


<commit_before>package upcloud

import (
	"log"
	"strconv"

	"github.com/UpCloudLtd/upcloud-go-api/upcloud"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/request"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/service"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceUpCloudFirewallRule() *schema.Resource {
	return &schema.Resource{
		Create: resourceUpCloudFirewallRuleCreate,
		Read:   resourceUpCloudFirewallRuleRead,
		Delete: resourceUpCloudFirewallRuleDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"server_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"direction": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"action": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"position": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"family": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"protocol": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "tcp",
			},
			"icmp_type": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_address_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_address_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_port_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_port_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_address_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_address_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_port_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_port_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"comment": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceUpCloudFirewallRuleCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)

	createFirewallRuleRequest := &request.CreateFirewallRuleRequest{
		ServerUUID: d.Get("server_id").(string),
		FirewallRule: upcloud.FirewallRule{
			Direction: d.Get("direction").(string),
			Action:    d.Get("action").(string),
			Family:    d.Get("family").(string),
		},
	}

	if position, ok := d.GetOk("position"); ok {
		position, err := strconv.Atoi(position.(string))
		if err != nil {
			return err
		}
		createFirewallRuleRequest.Position = position
	}

	if protocol, ok := d.GetOk("protocol"); ok {
		createFirewallRuleRequest.Protocol = protocol.(string)
	}

	if icmpType, ok := d.GetOk("icmp_type"); ok {
		createFirewallRuleRequest.ICMPType = icmpType.(string)
	}

	if destinationAddressStart, ok := d.GetOk("destination_address_start"); ok {
		log.Printf("[DEBUG] Destination address start: %+v", destinationAddressStart)
		createFirewallRuleRequest.DestinationAddressStart = destinationAddressStart.(string)
	}

	if destinationAddressEnd, ok := d.GetOk("destination_address_end"); ok {
		log.Printf("[DEBUG] Destination address end: %+v", destinationAddressEnd)
		createFirewallRuleRequest.DestinationAddressEnd = destinationAddressEnd.(string)
	}

	if destinationPortStart, ok := d.GetOk("destination_port_start"); ok {
		createFirewallRuleRequest.DestinationPortStart = destinationPortStart.(string)
	}

	if destinationPortEnd, ok := d.GetOk("destination_port_end"); ok {
		createFirewallRuleRequest.DestinationPortEnd = destinationPortEnd.(string)
	}

	if sourceAddressStart, ok := d.GetOk("source_address_start"); ok {
		createFirewallRuleRequest.SourceAddressStart = sourceAddressStart.(string)
	}

	if sourceAddressEnd, ok := d.GetOk("source_address_end"); ok {
		createFirewallRuleRequest.SourceAddressEnd = sourceAddressEnd.(string)
	}

	if sourcePortStart, ok := d.GetOk("source_port_start"); ok {
		createFirewallRuleRequest.SourcePortStart = sourcePortStart.(string)
	}

	if sourcePortEnd, ok := d.GetOk("source_port_end"); ok {
		createFirewallRuleRequest.SourcePortEnd = sourcePortEnd.(string)
	}

	if comment, ok := d.GetOk("comment"); ok {
		createFirewallRuleRequest.Comment = comment.(string)
	}

	log.Printf("[DEBUG] Firewall rule: %v", createFirewallRuleRequest)

	firewallRule, err := client.CreateFirewallRule(createFirewallRuleRequest)

	if err != nil {
		return err
	}

	d.SetId(createFirewallRuleRequest.ServerUUID + strconv.Itoa(firewallRule.Position))

	return resourceUpCloudFirewallRuleRead(d, meta)
}

func resourceUpCloudFirewallRuleRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)
	position, err := strconv.Atoi(d.Get("position").(string))

	if err != nil {
		return err
	}

	r := &request.GetFirewallRuleDetailsRequest{
		ServerUUID: d.Get("server_id").(string),
		Position:   position,
	}

	firewallRule, err := client.GetFirewallRuleDetails(r)

	if err != nil {
		return err
	}

	d.SetId(d.Get("server_id").(string) + strconv.Itoa(firewallRule.Position))
	d.Set("action", firewallRule.Action)
	d.Set("comment", firewallRule.Comment)
	d.Set("destination_address_end", firewallRule.DestinationAddressEnd)
	d.Set("destination_address_start", firewallRule.DestinationAddressStart)
	d.Set("destination_port_end", firewallRule.DestinationPortEnd)
	d.Set("destination_port_start", firewallRule.DestinationPortStart)
	d.Set("direction", firewallRule.Direction)
	d.Set("family", firewallRule.Family)
	d.Set("icmp_type", firewallRule.ICMPType)
	d.Set("position", firewallRule.Position)
	d.Set("protocol", firewallRule.Protocol)
	d.Set("source_address_end", firewallRule.SourceAddressEnd)
	d.Set("source_address_start", firewallRule.SourceAddressStart)
	d.Set("source_port_end", firewallRule.SourcePortEnd)
	d.Set("source_port_start", firewallRule.SourcePortStart)

	return nil
}

func resourceUpCloudFirewallRuleDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)

	position, err := strconv.Atoi(d.Id())

	if err != nil {
		return err
	}

	deleteFirewallRuleRequest := &request.DeleteFirewallRuleRequest{
		ServerUUID: d.Get("server_id").(string),
		Position:   position,
	}

	err = client.DeleteFirewallRule(deleteFirewallRuleRequest)

	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix firewall rule deletion<commit_after>package upcloud

import (
	"log"
	"strconv"

	"github.com/UpCloudLtd/upcloud-go-api/upcloud"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/request"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/service"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceUpCloudFirewallRule() *schema.Resource {
	return &schema.Resource{
		Create: resourceUpCloudFirewallRuleCreate,
		Read:   resourceUpCloudFirewallRuleRead,
		Delete: resourceUpCloudFirewallRuleDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"server_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"direction": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"action": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"position": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"family": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"protocol": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "tcp",
			},
			"icmp_type": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_address_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_address_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_port_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"source_port_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_address_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_address_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_port_start": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"destination_port_end": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"comment": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceUpCloudFirewallRuleCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)

	createFirewallRuleRequest := &request.CreateFirewallRuleRequest{
		ServerUUID: d.Get("server_id").(string),
		FirewallRule: upcloud.FirewallRule{
			Direction: d.Get("direction").(string),
			Action:    d.Get("action").(string),
			Family:    d.Get("family").(string),
		},
	}

	if position, ok := d.GetOk("position"); ok {
		position, err := strconv.Atoi(position.(string))
		if err != nil {
			return err
		}
		createFirewallRuleRequest.Position = position
	}

	if protocol, ok := d.GetOk("protocol"); ok {
		createFirewallRuleRequest.Protocol = protocol.(string)
	}

	if icmpType, ok := d.GetOk("icmp_type"); ok {
		createFirewallRuleRequest.ICMPType = icmpType.(string)
	}

	if destinationAddressStart, ok := d.GetOk("destination_address_start"); ok {
		log.Printf("[DEBUG] Destination address start: %+v", destinationAddressStart)
		createFirewallRuleRequest.DestinationAddressStart = destinationAddressStart.(string)
	}

	if destinationAddressEnd, ok := d.GetOk("destination_address_end"); ok {
		log.Printf("[DEBUG] Destination address end: %+v", destinationAddressEnd)
		createFirewallRuleRequest.DestinationAddressEnd = destinationAddressEnd.(string)
	}

	if destinationPortStart, ok := d.GetOk("destination_port_start"); ok {
		createFirewallRuleRequest.DestinationPortStart = destinationPortStart.(string)
	}

	if destinationPortEnd, ok := d.GetOk("destination_port_end"); ok {
		createFirewallRuleRequest.DestinationPortEnd = destinationPortEnd.(string)
	}

	if sourceAddressStart, ok := d.GetOk("source_address_start"); ok {
		createFirewallRuleRequest.SourceAddressStart = sourceAddressStart.(string)
	}

	if sourceAddressEnd, ok := d.GetOk("source_address_end"); ok {
		createFirewallRuleRequest.SourceAddressEnd = sourceAddressEnd.(string)
	}

	if sourcePortStart, ok := d.GetOk("source_port_start"); ok {
		createFirewallRuleRequest.SourcePortStart = sourcePortStart.(string)
	}

	if sourcePortEnd, ok := d.GetOk("source_port_end"); ok {
		createFirewallRuleRequest.SourcePortEnd = sourcePortEnd.(string)
	}

	if comment, ok := d.GetOk("comment"); ok {
		createFirewallRuleRequest.Comment = comment.(string)
	}

	log.Printf("[DEBUG] Firewall rule: %v", createFirewallRuleRequest)

	firewallRule, err := client.CreateFirewallRule(createFirewallRuleRequest)

	if err != nil {
		return err
	}

	d.SetId(createFirewallRuleRequest.ServerUUID + strconv.Itoa(firewallRule.Position))

	return resourceUpCloudFirewallRuleRead(d, meta)
}

func resourceUpCloudFirewallRuleRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)
	position, err := strconv.Atoi(d.Get("position").(string))

	if err != nil {
		return err
	}

	r := &request.GetFirewallRuleDetailsRequest{
		ServerUUID: d.Get("server_id").(string),
		Position:   position,
	}

	firewallRule, err := client.GetFirewallRuleDetails(r)

	if err != nil {
		return err
	}

	d.SetId(d.Get("server_id").(string) + strconv.Itoa(firewallRule.Position))
	d.Set("action", firewallRule.Action)
	d.Set("comment", firewallRule.Comment)
	d.Set("destination_address_end", firewallRule.DestinationAddressEnd)
	d.Set("destination_address_start", firewallRule.DestinationAddressStart)
	d.Set("destination_port_end", firewallRule.DestinationPortEnd)
	d.Set("destination_port_start", firewallRule.DestinationPortStart)
	d.Set("direction", firewallRule.Direction)
	d.Set("family", firewallRule.Family)
	d.Set("icmp_type", firewallRule.ICMPType)
	d.Set("position", firewallRule.Position)
	d.Set("protocol", firewallRule.Protocol)
	d.Set("source_address_end", firewallRule.SourceAddressEnd)
	d.Set("source_address_start", firewallRule.SourceAddressStart)
	d.Set("source_port_end", firewallRule.SourcePortEnd)
	d.Set("source_port_start", firewallRule.SourcePortStart)

	return nil
}

func resourceUpCloudFirewallRuleDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*service.Service)

	position, err := strconv.Atoi(d.Get("position").(string))

	if err != nil {
		return err
	}

	deleteFirewallRuleRequest := &request.DeleteFirewallRuleRequest{
		ServerUUID: d.Get("server_id").(string),
		Position:   position,
	}

	err = client.DeleteFirewallRule(deleteFirewallRuleRequest)

	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package expensivequery

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	rpprof "runtime/pprof"
	"sort"
	"strings"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/disk"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/memory"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

type memoryUsageAlarm struct {
	err                    error
	isServerMemoryQuotaSet bool
	serverMemoryQuota      uint64
	lastCheckTime          time.Time

	tmpDir              string
	lastLogFileName     []string
	lastProfileFileName [][]string // heap, goroutine
}

func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) {
	record = &memoryUsageAlarm{}
	if alert := config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio; alert == 0 || alert == 1 {
		record.err = errors.New("close memory usage alarm recorder")
		return
	}
	if quota := config.GetGlobalConfig().Performance.ServerMemoryQuota; quota != 0 {
		record.serverMemoryQuota = quota
		record.isServerMemoryQuotaSet = true
	} else {
		// TODO: Get the memory info in container directly.
		record.serverMemoryQuota, record.err = memory.MemTotal()
		if record.err != nil {
			logutil.BgLogger().Error("get system total memory fail", zap.Error(record.err))
			return
		}
		record.isServerMemoryQuotaSet = false
	}
	record.lastCheckTime = time.Time{}
	record.tmpDir = filepath.Join(config.GetGlobalConfig().TempStoragePath, "record")
	record.lastProfileFileName = make([][]string, 2)
	// Read last records
	files, err := ioutil.ReadDir(record.tmpDir)
	if err != nil {
		return record
	}
	for _, f := range files {
		name := filepath.Join(record.tmpDir, f.Name())
		if strings.Contains(f.Name(), "running_sql") {
			record.lastLogFileName = append(record.lastLogFileName, name)
		}
		if strings.Contains(f.Name(), "heap") {
			record.lastProfileFileName[0] = append(record.lastProfileFileName[0], name)
		}
		if strings.Contains(f.Name(), "goroutine") {
			record.lastProfileFileName[1] = append(record.lastProfileFileName[1], name)
		}
	}

	return record
}

// If Performance.ServerMemoryQuota is set, use `ServerMemoryQuota * MemoryUsageAlarmRatio` to check oom risk.
// If Performance.ServerMemoryQuota is not set, use `system total memory size * MemoryUsageAlarmRatio` to check oom risk.
func (record *memoryUsageAlarm) alarm4ExcessiveMemUsage(sm util.SessionManager) {
	var memoryUsage uint64
	instanceStats := &runtime.MemStats{}
	if record.isServerMemoryQuotaSet {
		runtime.ReadMemStats(instanceStats)
		memoryUsage = instanceStats.HeapAlloc
	} else {
		memoryUsage, record.err = memory.MemUsed()
		if record.err != nil {
			logutil.BgLogger().Error("get system memory usage fail", zap.Error(record.err))
			return
		}
	}

	// TODO: Consider NextGC to record SQLs.
	if float64(memoryUsage) > float64(record.serverMemoryQuota)*config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio {
		// At least ten seconds between two recordings that memory usage is less than threshold (default 80% system memory).
		// If the memory is still exceeded, only records once.
		interval := time.Since(record.lastCheckTime)
		record.lastCheckTime = time.Now()
		if interval > 10*time.Second {
			record.doRecord(memoryUsage, sm)
		}
	}
}

func (record *memoryUsageAlarm) doRecord(memUsage uint64, sm util.SessionManager) {
	logutil.BgLogger().Warn("the TiDB instance now takes a lot of memory, has the risk of OOM",
		zap.Bool("is server-momory-quota set", record.isServerMemoryQuotaSet),
		zap.Any("memory size", record.serverMemoryQuota),
		zap.Any("memory usage", memUsage),
		zap.Any("memory-usage-alarm-ratio", config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio),
	)

	if record.err = disk.CheckAndInitTempDir(); record.err != nil {
		return
	}
	record.recordSQLAndSummaryTable(sm)
	record.recordProfile()

	tryRemove := func(filename *[]string) {
		// Keep the last 5 files
		for len(*filename) > 5 {
			err := os.Remove((*filename)[0])
			if err != nil {
				logutil.BgLogger().Error("remove temp files failed", zap.Error(err))
				return
			}
			*filename = (*filename)[1:]
		}
	}
	tryRemove(&record.lastLogFileName)
	for i := range record.lastProfileFileName {
		tryRemove(&record.lastProfileFileName[i])
	}
}

func (record *memoryUsageAlarm) recordSQLAndSummaryTable(sm util.SessionManager) {
	processInfo := sm.ShowProcessList()
	pinfo := make([]*util.ProcessInfo, 0, len(processInfo))
	for _, info := range processInfo {
		if len(info.Info) != 0 {
			pinfo = append(pinfo, info)
		}
	}

	fileName := filepath.Join(record.tmpDir, "running_sql"+record.lastCheckTime.Format(time.RFC3339))
	record.lastLogFileName = append(record.lastLogFileName, fileName)
	f, err := os.Create(fileName)
	if err != nil {
		logutil.BgLogger().Error("create oom record file fail", zap.Error(err))
		return
	}
	defer func() {
		err := f.Close()
		if err != nil {
			logutil.BgLogger().Error("close oom record file fail", zap.Error(err))
		}
	}()
	printTop10 := func(cmp func(i, j int) bool) {
		sort.Slice(pinfo, cmp)
		list := pinfo
		if len(list) > 10 {
			list = list[:10]
		}
		var buf strings.Builder
		for i, info := range list {
			buf.WriteString(fmt.Sprintf("SQL %v: \n", i))
			fields := genLogFields(record.lastCheckTime.Sub(info.Time), info)
			for _, field := range fields {
				switch field.Type {
				case zapcore.StringType:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, field.String))
				case zapcore.Uint8Type, zapcore.Uint16Type, zapcore.Uint32Type, zapcore.Uint64Type:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, uint64(field.Integer)))
				case zapcore.Int8Type, zapcore.Int16Type, zapcore.Int32Type, zapcore.Int64Type:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, field.Integer))
				}
				buf.WriteString("\n")
			}
		}
		buf.WriteString("\n")
		_, err = f.WriteString(buf.String())
	}

	_, err = f.WriteString("The 10 SQLs with the most memory usage for OOM analysis\n")
	printTop10(func(i, j int) bool {
		return pinfo[i].StmtCtx.MemTracker.MaxConsumed() > pinfo[j].StmtCtx.MemTracker.MaxConsumed()
	})

	_, err = f.WriteString("The 10 SQLs with the most time usage for OOM analysis\n")
	printTop10(func(i, j int) bool {
		return pinfo[i].Time.Before(pinfo[j].Time)
	})

	logutil.BgLogger().Info("record SQLs with the most memory usage or time usage", zap.Any("SQLs file path", fileName))
}

func (record *memoryUsageAlarm) recordProfile() {
	items := []struct {
		name  string
		debug int
	}{
		{name: "heap"},
		{name: "goroutine", debug: 2},
	}
	for i, item := range items {
		fileName := filepath.Join(record.tmpDir, item.name+record.lastCheckTime.Format(time.RFC3339))
		record.lastProfileFileName[i] = append(record.lastProfileFileName[i], fileName)
		f, err := os.Create(fileName)
		if err != nil {
			logutil.BgLogger().Error(fmt.Sprintf("create %v profile file fail", item.name), zap.Error(err))
			return
		}
		defer func() {
			err := f.Close()
			if err != nil {
				logutil.BgLogger().Error(fmt.Sprintf("close %v profile file fail", item.name), zap.Error(err))
			}
		}()
		p := rpprof.Lookup(item.name)
		err = p.WriteTo(f, item.debug)
		if err != nil {
			logutil.BgLogger().Error(fmt.Sprintf("write %v profile file fail", item.name), zap.Error(err))
			return
		}
		logutil.BgLogger().Info(fmt.Sprintf("record %v profile successfully", item.name), zap.Any("Profile file path", fileName))
	}
}
<commit_msg>util: reduce memoryUsageAlarm log (#20562)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package expensivequery

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	rpprof "runtime/pprof"
	"sort"
	"strings"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/disk"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/memory"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

type memoryUsageAlarm struct {
	err                    error
	isServerMemoryQuotaSet bool
	serverMemoryQuota      uint64
	lastCheckTime          time.Time

	tmpDir              string
	lastLogFileName     []string
	lastProfileFileName [][]string // heap, goroutine
}

func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) {
	record = &memoryUsageAlarm{}
	if alert := config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio; alert == 0 || alert == 1 {
		record.err = errors.New("close memory usage alarm recorder")
		return
	}
	if quota := config.GetGlobalConfig().Performance.ServerMemoryQuota; quota != 0 {
		record.serverMemoryQuota = quota
		record.isServerMemoryQuotaSet = true
	} else {
		// TODO: Get the memory info in container directly.
		record.serverMemoryQuota, record.err = memory.MemTotal()
		if record.err != nil {
			logutil.BgLogger().Error("get system total memory fail", zap.Error(record.err))
			return
		}
		record.isServerMemoryQuotaSet = false
	}
	record.lastCheckTime = time.Time{}
	record.tmpDir = filepath.Join(config.GetGlobalConfig().TempStoragePath, "record")
	record.lastProfileFileName = make([][]string, 2)
	// Read last records
	files, err := ioutil.ReadDir(record.tmpDir)
	if err != nil {
		return record
	}
	for _, f := range files {
		name := filepath.Join(record.tmpDir, f.Name())
		if strings.Contains(f.Name(), "running_sql") {
			record.lastLogFileName = append(record.lastLogFileName, name)
		}
		if strings.Contains(f.Name(), "heap") {
			record.lastProfileFileName[0] = append(record.lastProfileFileName[0], name)
		}
		if strings.Contains(f.Name(), "goroutine") {
			record.lastProfileFileName[1] = append(record.lastProfileFileName[1], name)
		}
	}

	return record
}

// If Performance.ServerMemoryQuota is set, use `ServerMemoryQuota * MemoryUsageAlarmRatio` to check oom risk.
// If Performance.ServerMemoryQuota is not set, use `system total memory size * MemoryUsageAlarmRatio` to check oom risk.
func (record *memoryUsageAlarm) alarm4ExcessiveMemUsage(sm util.SessionManager) {
	var memoryUsage uint64
	instanceStats := &runtime.MemStats{}
	runtime.ReadMemStats(instanceStats)
	if record.isServerMemoryQuotaSet {
		memoryUsage = instanceStats.HeapAlloc
	} else {
		memoryUsage, record.err = memory.MemUsed()
		if record.err != nil {
			logutil.BgLogger().Error("get system memory usage fail", zap.Error(record.err))
			return
		}
	}

	// TODO: Consider NextGC to record SQLs.
	if float64(memoryUsage) > float64(record.serverMemoryQuota)*config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio {
		// At least ten seconds between two recordings that memory usage is less than threshold (default 80% system memory).
		// If the memory is still exceeded, only records once.
		interval := time.Since(record.lastCheckTime)
		record.lastCheckTime = time.Now()
		if interval > 10*time.Second {
			record.doRecord(memoryUsage, instanceStats.HeapAlloc, sm)
		}
	}
}

func (record *memoryUsageAlarm) doRecord(memUsage uint64, instanceMemoryUsage uint64, sm util.SessionManager) {
	fields := make([]zap.Field, 0, 6)
	fields = append(fields, zap.Bool("is server-memory-quota set", record.isServerMemoryQuotaSet))
	if record.isServerMemoryQuotaSet {
		fields = append(fields, zap.Any("server-memory-quota", record.serverMemoryQuota))
		fields = append(fields, zap.Any("tidb-server memory usage", memUsage))
	} else {
		fields = append(fields, zap.Any("system memory total", record.serverMemoryQuota))
		fields = append(fields, zap.Any("system memory usage", memUsage))
		fields = append(fields, zap.Any("tidb-server memory usage", instanceMemoryUsage))
	}
	fields = append(fields, zap.Any("memory-usage-alarm-ratio", config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio))
	fields = append(fields, zap.Any("record path", record.tmpDir))

	logutil.BgLogger().Warn("tidb-server has the risk of OOM. Running SQLs and heap profile will be recorded in record path", fields...)

	if record.err = disk.CheckAndInitTempDir(); record.err != nil {
		return
	}
	record.recordSQL(sm)
	record.recordProfile()

	tryRemove := func(filename *[]string) {
		// Keep the last 5 files
		for len(*filename) > 5 {
			err := os.Remove((*filename)[0])
			if err != nil {
				logutil.BgLogger().Error("remove temp files failed", zap.Error(err))
				return
			}
			*filename = (*filename)[1:]
		}
	}
	tryRemove(&record.lastLogFileName)
	for i := range record.lastProfileFileName {
		tryRemove(&record.lastProfileFileName[i])
	}
}

func (record *memoryUsageAlarm) recordSQL(sm util.SessionManager) {
	processInfo := sm.ShowProcessList()
	pinfo := make([]*util.ProcessInfo, 0, len(processInfo))
	for _, info := range processInfo {
		if len(info.Info) != 0 {
			pinfo = append(pinfo, info)
		}
	}

	fileName := filepath.Join(record.tmpDir, "running_sql"+record.lastCheckTime.Format(time.RFC3339))
	record.lastLogFileName = append(record.lastLogFileName, fileName)
	f, err := os.Create(fileName)
	if err != nil {
		logutil.BgLogger().Error("create oom record file fail", zap.Error(err))
		return
	}
	defer func() {
		err := f.Close()
		if err != nil {
			logutil.BgLogger().Error("close oom record file fail", zap.Error(err))
		}
	}()
	printTop10 := func(cmp func(i, j int) bool) {
		sort.Slice(pinfo, cmp)
		list := pinfo
		if len(list) > 10 {
			list = list[:10]
		}
		var buf strings.Builder
		for i, info := range list {
			buf.WriteString(fmt.Sprintf("SQL %v: \n", i))
			fields := genLogFields(record.lastCheckTime.Sub(info.Time), info)
			for _, field := range fields {
				switch field.Type {
				case zapcore.StringType:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, field.String))
				case zapcore.Uint8Type, zapcore.Uint16Type, zapcore.Uint32Type, zapcore.Uint64Type:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, uint64(field.Integer)))
				case zapcore.Int8Type, zapcore.Int16Type, zapcore.Int32Type, zapcore.Int64Type:
					buf.WriteString(fmt.Sprintf("%v: %v", field.Key, field.Integer))
				}
				buf.WriteString("\n")
			}
		}
		buf.WriteString("\n")
		_, err = f.WriteString(buf.String())
	}

	_, err = f.WriteString("The 10 SQLs with the most memory usage for OOM analysis\n")
	printTop10(func(i, j int) bool {
		return pinfo[i].StmtCtx.MemTracker.MaxConsumed() > pinfo[j].StmtCtx.MemTracker.MaxConsumed()
	})

	_, err = f.WriteString("The 10 SQLs with the most time usage for OOM analysis\n")
	printTop10(func(i, j int) bool {
		return pinfo[i].Time.Before(pinfo[j].Time)
	})
}

func (record *memoryUsageAlarm) recordProfile() {
	items := []struct {
		name  string
		debug int
	}{
		{name: "heap"},
		{name: "goroutine", debug: 2},
	}
	for i, item := range items {
		fileName := filepath.Join(record.tmpDir, item.name+record.lastCheckTime.Format(time.RFC3339))
		record.lastProfileFileName[i] = append(record.lastProfileFileName[i], fileName)
		f, err := os.Create(fileName)
		if err != nil {
			logutil.BgLogger().Error(fmt.Sprintf("create %v profile file fail", item.name), zap.Error(err))
			return
		}
		defer func() {
			err := f.Close()
			if err != nil {
				logutil.BgLogger().Error(fmt.Sprintf("close %v profile file fail", item.name), zap.Error(err))
			}
		}()
		p := rpprof.Lookup(item.name)
		err = p.WriteTo(f, item.debug)
		if err != nil {
			logutil.BgLogger().Error(fmt.Sprintf("write %v profile file fail", item.name), zap.Error(err))
			return
		}
	}
}
<|endoftext|>