<commit_before>package deploy

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/Unknwon/com"
	"github.com/codegangsta/cli"
	"gopkg.in/inconshreveable/log15.v2"
)

// Git is deployment of git repository
type Git struct {
	Repo    string
	Message string
	Local   string
	Branch  string
}

// Command return git deploy command
func (g *Git) Command() cli.Command {
	return cli.Command{
		Name:  "git",
		Usage: "deploy via git push",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "local", Value: "public", Usage: "local website directory"},
			cli.StringFlag{Name: "repo", Usage: "local repository directory"},
			cli.StringFlag{Name: "message", Usage: "pushing commit message"},
			cli.StringFlag{Name: "branch", Value: "master", Usage: "the remote branch that git push to"},
		},
		Action: func(ctx *cli.Context) {
			g2, err := g.Create(ctx)
			if err != nil {
				log15.Error("Deploy|Git|Fail|%s", err.Error())
				return
			}
			if err = g2.Do(); err != nil {
				log15.Error("Deploy|Git|Fail|%s", err.Error())
				return
			}
			log15.Info("Deploy|Git|Finish")
		},
	}
}

// String return git deployment typename
func (g *Git) String() string {
	return "Git"
}

// Create creates git deploy settings in Context
func (g *Git) Create(ctx *cli.Context) (Method, error) {
	g2 := &Git{
		Repo:    ctx.String("repo"),
		Message: ctx.String("message"),
		Local:   ctx.String("local"),
		Branch:  ctx.String("branch"),
	}
	if !com.IsDir(g2.Local) {
		return nil, fmt.Errorf("directory '%s' is not existed", g2.Local)
	}
	if !com.IsDir(g2.Repo) || !com.IsDir(filepath.Join(g2.Repo, ".git")) {
		return nil, fmt.Errorf("directory '%s' is not a git repository", g2.Repo)
	}
	if g2.Message == "" {
		g2.Message = "PUGO BUILD UPDATE - {t}"
	}
	return g2, nil
}

// Do do git deploy action with built Context
func (g *Git) Do() error {
	err := filepath.Walk(g.Local, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		rel, _ := filepath.Rel(g.Local, path)
		rel = filepath.ToSlash(rel)

		log15.Debug(rel)

		return nil
	})
	return err
	/*
		// git add -A
		log15.Debug("Deploy|Git|git add -A")
		_, errOut, err := com.ExecCmdDir(g.gitRepo, "git", "add", "-A")
		if err = returnGetError(errOut, err); err != nil {
			return err
		}

		// git commit -m "message"
		log15.Debug("Deploy|Git|git commit -m")
		message := strings.Replace(g.gitMessage, "{t}", time.Now().Format(time.RFC1123), 1)
		_, errOut, err = com.ExecCmdDir(g.gitRepo, "git", "commit", "-m", message)
		if err = returnGetError(errOut, err); err != nil {
			return err
		}

		// git push
		log15.Debug("Deploy|Git|git push -f")
		_, errOut, err = com.ExecCmdDir(g.gitRepo, "git", "push", "-f")
		if err = returnGetError(errOut, err); err != nil {
			return err
		}
		return nil
	*/
}

func returnGetError(errOut string, err error) error {
	if errOut != "" && strings.Contains(errOut, "fatal:") {
		return fmt.Errorf(errOut)
	}
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>[deploy] git method done<commit_after>package deploy

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
	"github.com/codegangsta/cli"
	"gopkg.in/inconshreveable/log15.v2"
)

// Git is deployment of git repository
type Git struct {
	Repo    string
	Message string
	Local   string
	Branch  string
}

// Command return git deploy command
func (g *Git) Command() cli.Command {
	return cli.Command{
		Name:  "git",
		Usage: "deploy via git push",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "local", Value: "public", Usage: "local website directory"},
			cli.StringFlag{Name: "repo", Usage: "local repository directory"},
			cli.StringFlag{Name: "message", Usage: "pushing commit message"},
			cli.StringFlag{Name: "branch", Value: "master", Usage: "the remote branch that git push to"},
		},
		Action: func(ctx *cli.Context) {
			g2, err := g.Create(ctx)
			if err != nil {
				log15.Error("Deploy|Git|Fail|%s", err.Error())
				return
			}
			if err = g2.Do(); err != nil {
				log15.Error("Deploy|Git|Fail|%s", err.Error())
				return
			}
			log15.Info("Deploy|Git|Finish")
		},
	}
}

// String return git deployment typename
func (g *Git) String() string {
	return "Git"
}

// Create creates git deploy settings in Context
func (g *Git) Create(ctx *cli.Context) (Method, error) {
	g2 := &Git{
		Repo:    ctx.String("repo"),
		Message: ctx.String("message"),
		Local:   ctx.String("local"),
		Branch:  ctx.String("branch"),
	}
	if !com.IsDir(g2.Local) {
		return nil, fmt.Errorf("directory '%s' is not existed", g2.Local)
	}
	if !com.IsDir(g2.Repo) || !com.IsDir(filepath.Join(g2.Repo, ".git")) {
		return nil, fmt.Errorf("directory '%s' is not a git repository", g2.Repo)
	}
	if g2.Message == "" {
		g2.Message = "PUGO BUILD UPDATE - {t}"
	}
	return g2, nil
}

// Do do git deploy action with built Context
func (g *Git) Do() error {
	log15.Debug("Deploy|Git|Overwrite")
	err := filepath.Walk(g.Local, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		rel, _ := filepath.Rel(g.Local, path)
		rel = filepath.ToSlash(rel)

		toFile := filepath.Join(g.Repo, rel)

		// overwrite file in repo
		os.MkdirAll(filepath.Dir(toFile), os.ModePerm)
		fileBytes, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		if err = ioutil.WriteFile(toFile, fileBytes, os.ModePerm); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	log15.Debug("Deploy|Git|git add -A")
	if err = runGitExec(g.Repo, []string{"add", "-A"}); err != nil {
		return err
	}
	log15.Debug("Deploy|Git|git commit -m")
	message := strings.Replace(g.Message, "{t}", time.Now().Format(time.RFC1123), 1)
	if err = runGitExec(g.Repo, []string{"commit", "-m", message}); err != nil {
		return err
	}
	log15.Debug("Deploy|Git|git push -f")
	if err = runGitExec(g.Repo, []string{"push", "-f"}); err != nil {
		return err
	}
	return nil
	/*
		// git add -A
		log15.Debug("Deploy|Git|git add -A")
		_, errOut, err := com.ExecCmdDir(g.gitRepo, "git", "add", "-A")
		if err = returnGetError(errOut, err); err != nil {
			return err
		}

		// git commit -m "message"
		log15.Debug("Deploy|Git|git commit -m")
		message := strings.Replace(g.gitMessage, "{t}", time.Now().Format(time.RFC1123), 1)
		_, errOut, err = com.ExecCmdDir(g.gitRepo, "git", "commit", "-m", message)
		if err = returnGetError(errOut, err); err != nil {
			return err
		}

		// git push
		log15.Debug("Deploy|Git|git push -f")
		_, errOut, err = com.ExecCmdDir(g.gitRepo, "git", "push", "-f")
		if err = returnGetError(errOut, err); err != nil {
			return err
		}
		return nil
	*/
}

func runGitExec(dir string, commands []string) error {
	_, errOut, err := com.ExecCmdDir(dir, "git", commands...)
	return returnGetError(errOut, err)
}

func returnGetError(errOut string, err error) error {
	if errOut != "" && strings.Contains(errOut, "fatal:") {
		return fmt.Errorf(errOut)
	}
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package bench

import (
	"time"

	"github.com/nats-io/nats"
)

// NATSRequester implements Requester by publishing a message to NATS and
// waiting to receive it.
type NATSRequester struct {
	URL         string
	PayloadSize int
	Subject     string
	conn        *nats.Conn
	sub         *nats.Subscription
	msg         []byte
}

// Setup prepares the Requester for benchmarking.
func (n *NATSRequester) Setup() error {
	conn, err := nats.Connect(n.URL)
	if err != nil {
		return err
	}
	sub, err := conn.SubscribeSync(n.Subject)
	if err != nil {
		return err
	}
	n.conn = conn
	n.sub = sub
	n.msg = make([]byte, n.PayloadSize)
	return nil
}

// Request performs a synchronous request to the system under test.
func (n *NATSRequester) Request() error {
	if err := n.conn.Publish(n.Subject, n.msg); err != nil {
		return err
	}
	_, err := n.sub.NextMsg(30 * time.Second)
	return err
}

// Teardown is called upon benchmark completion.
func (n *NATSRequester) Teardown() error {
	err := n.sub.Unsubscribe()
	if err != nil {
		return err
	}
	n.sub = nil
	n.conn.Close()
	n.conn = nil
	return nil
}
<commit_msg>Close NATS connection on setup failure<commit_after>package bench

import (
	"time"

	"github.com/nats-io/nats"
)

// NATSRequester implements Requester by publishing a message to NATS and
// waiting to receive it.
type NATSRequester struct {
	URL         string
	PayloadSize int
	Subject     string
	conn        *nats.Conn
	sub         *nats.Subscription
	msg         []byte
}

// Setup prepares the Requester for benchmarking.
func (n *NATSRequester) Setup() error {
	conn, err := nats.Connect(n.URL)
	if err != nil {
		return err
	}
	sub, err := conn.SubscribeSync(n.Subject)
	if err != nil {
		conn.Close()
		return err
	}
	n.conn = conn
	n.sub = sub
	n.msg = make([]byte, n.PayloadSize)
	return nil
}

// Request performs a synchronous request to the system under test.
func (n *NATSRequester) Request() error {
	if err := n.conn.Publish(n.Subject, n.msg); err != nil {
		return err
	}
	_, err := n.sub.NextMsg(30 * time.Second)
	return err
}

// Teardown is called upon benchmark completion.
func (n *NATSRequester) Teardown() error {
	if err := n.sub.Unsubscribe(); err != nil {
		return err
	}
	n.sub = nil
	n.conn.Close()
	n.conn = nil
	return nil
}
<|endoftext|>


<commit_before>// Package battlenet implements the OAuth2 protocol for authenticating users through Battle.net.
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package battlenet

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"fmt"
	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

const (
	authURL      string = "https://us.battle.net/oauth/authorize"
	tokenURL     string = "https://us.battle.net/oauth/token"
	endpointUser string = "https://us.api.battle.net/account/user"
)

// Provider is the implementation of `goth.Provider` for accessing Battle.net.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// New creates a new Battle.net provider and sets up important connection details.
// You should always call `battlenet.New` to get a new provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "battlenet",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the battlenet package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks Battle.net for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	return &Session{
		AuthURL: p.config.AuthCodeURL(state),
	}, nil
}

// FetchUser will go to Battle.net and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken:  sess.AccessToken,
		Provider:     p.Name(),
		RefreshToken: sess.RefreshToken,
		ExpiresAt:    sess.ExpiresAt,
	}

	if user.AccessToken == "" {
		// data is not yet retrieved since accessToken is still empty
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	// Get the userID, battlenet needs userID in order to get user profile info
	response, err := p.Client().Get(endpointUser + "?access_token=" + url.QueryEscape(sess.AccessToken))
	if err != nil {
		if response != nil {
			response.Body.Close()
		}
		return user, err
	}

	if response.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, response.StatusCode)
	}

	bits, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return user, err
	}

	u := struct {
		ID        int64  `json:"id"`
		Battletag string `json:"battletag"`
	}{}

	if err = json.NewDecoder(bytes.NewReader(bits)).Decode(&u); err != nil {
		return user, err
	}

	user.NickName = u.Battletag
	user.UserID = fmt.Sprintf("%d", u.ID)
	return user, err
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{},
	}

	if len(scopes) > 0 {
		for _, scope := range scopes {
			c.Scopes = append(c.Scopes, scope)
		}
	}
	return c
}

//RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return false
}

//RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	return nil, nil
}
<commit_msg>updated battle.net provioder to use the new oauth serivces<commit_after>// Package battlenet implements the OAuth2 protocol for authenticating users through Battle.net.
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package battlenet

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"

	"fmt"

	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

const (
	authURL      string = "https://us.battle.net/oauth/authorize"
	tokenURL     string = "https://us.battle.net/oauth/token"
	endpointUser string = "https://us.battle.net/oauth/userinfo"
)

// Provider is the implementation of `goth.Provider` for accessing Battle.net.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// New creates a new Battle.net provider and sets up important connection details.
// You should always call `battlenet.New` to get a new provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "battlenet",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the battlenet package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks Battle.net for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	return &Session{
		AuthURL: p.config.AuthCodeURL(state),
	}, nil
}

// FetchUser will go to Battle.net and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken:  sess.AccessToken,
		Provider:     p.Name(),
		RefreshToken: sess.RefreshToken,
		ExpiresAt:    sess.ExpiresAt,
	}

	if user.AccessToken == "" {
		// data is not yet retrieved since accessToken is still empty
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	// Get the userID, battlenet needs userID in order to get user profile info
	c := p.Client()
	req, _ := http.NewRequest("GET", endpointUser, nil)
	req.Header.Add("Authorization", "Bearer "+sess.AccessToken)

	response, err := c.Do(req)
	if err != nil {
		if response != nil {
			response.Body.Close()
		}
		return user, err
	}

	if response.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, response.StatusCode)
	}

	bits, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return user, err
	}

	u := struct {
		ID        int64  `json:"id"`
		Battletag string `json:"battletag"`
	}{}

	if err = json.NewDecoder(bytes.NewReader(bits)).Decode(&u); err != nil {
		return user, err
	}

	user.NickName = u.Battletag
	user.UserID = fmt.Sprintf("%d", u.ID)
	return user, err
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{},
	}

	if len(scopes) > 0 {
		for _, scope := range scopes {
			c.Scopes = append(c.Scopes, scope)
		}
	}
	return c
}

//RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return false
}

//RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"os"
	"path/filepath"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("plan", func() {
	var (
		bbl      actors.BBL
		stateDir string
	)

	BeforeEach(func() {
		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		stateDir = configuration.StateFileDir

		bbl = actors.NewBBL(stateDir, pathToBBL, configuration, "plan-env")
	})

	AfterEach(func() {
		session := bbl.Down()
		Eventually(session, 10*time.Minute).Should(gexec.Exit())
	})

	It("sets up the bbl state directory", func() {
		session := bbl.Plan("--name", bbl.PredefinedEnvID())
		Eventually(session, 40*time.Minute).Should(gexec.Exit(0))

		By("verifying that artifacts are created in state dir", func() {
			checkExists := func(dir string, filenames []string) {
				for _, f := range filenames {
					_, err := os.Stat(filepath.Join(dir, f))
					Expect(err).NotTo(HaveOccurred())
				}
			}

			checkExists(stateDir, []string{"bbl-state.json"})
			checkExists(stateDir, []string{"create-jumpbox.sh"})
			checkExists(stateDir, []string{"create-director.sh"})
			checkExists(stateDir, []string{"delete-jumpbox.sh"})
			checkExists(stateDir, []string{"delete-director.sh"})
			checkExists(filepath.Join(stateDir, ".bbl", "cloudconfig"), []string{
				"cloud-config.yml",
				"ops.yml",
			})
			checkExists(filepath.Join(stateDir, ".bbl"), []string{
				"previous-user-ops-file.yml",
			})
			checkExists(filepath.Join(stateDir, "bosh-deployment"), []string{
				"bosh.yml",
				"cpi.yml",
				"credhub.yml",
				"jumpbox-user.yml",
				"uaa.yml",
			})
			checkExists(filepath.Join(stateDir, "jumpbox-deployment"), []string{
				"cpi.yml",
				"jumpbox.yml",
			})
			checkExists(filepath.Join(stateDir, "terraform"), []string{
				"template.tf",
			})
			checkExists(filepath.Join(stateDir, "vars"), []string{
				// "bosh-state.json",
				"director-variables.yml",
				// "jumpbox-state.json",
				"jumpbox-variables.yml",
				// "terraform.tfstate",
				"user-ops-file.yml",
			})
		})
	})
})
<commit_msg>plan acceptance test is sensitive to RUN_TEST env<commit_after>package acceptance_test

import (
	"os"
	"path/filepath"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("plan", func() {
	var (
		bbl      actors.BBL
		stateDir string
	)

	BeforeEach(func() {
		acceptance.SkipUnless("plan")

		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		stateDir = configuration.StateFileDir

		bbl = actors.NewBBL(stateDir, pathToBBL, configuration, "plan-env")
	})

	AfterEach(func() {
		session := bbl.Down()
		Eventually(session, 10*time.Minute).Should(gexec.Exit())
	})

	It("sets up the bbl state directory", func() {
		session := bbl.Plan("--name", bbl.PredefinedEnvID())
		Eventually(session, 40*time.Minute).Should(gexec.Exit(0))

		By("verifying that artifacts are created in state dir", func() {
			checkExists := func(dir string, filenames []string) {
				for _, f := range filenames {
					_, err := os.Stat(filepath.Join(dir, f))
					Expect(err).NotTo(HaveOccurred())
				}
			}

			checkExists(stateDir, []string{"bbl-state.json"})
			checkExists(stateDir, []string{"create-jumpbox.sh"})
			checkExists(stateDir, []string{"create-director.sh"})
			checkExists(stateDir, []string{"delete-jumpbox.sh"})
			checkExists(stateDir, []string{"delete-director.sh"})
			checkExists(filepath.Join(stateDir, ".bbl", "cloudconfig"), []string{
				"cloud-config.yml",
				"ops.yml",
			})
			checkExists(filepath.Join(stateDir, ".bbl"), []string{
				"previous-user-ops-file.yml",
			})
			checkExists(filepath.Join(stateDir, "bosh-deployment"), []string{
				"bosh.yml",
				"cpi.yml",
				"credhub.yml",
				"jumpbox-user.yml",
				"uaa.yml",
			})
			checkExists(filepath.Join(stateDir, "jumpbox-deployment"), []string{
				"cpi.yml",
				"jumpbox.yml",
			})
			checkExists(filepath.Join(stateDir, "terraform"), []string{
				"template.tf",
			})
			checkExists(filepath.Join(stateDir, "vars"), []string{
				// "bosh-state.json",
				"director-variables.yml",
				// "jumpbox-state.json",
				"jumpbox-variables.yml",
				// "terraform.tfstate",
				"user-ops-file.yml",
			})
		})
	})
})
<|endoftext|>


<commit_before>package server

// transfer2go agent server implementation
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/vkuznet/transfer2go/common"
	"github.com/vkuznet/transfer2go/model"
	"github.com/vkuznet/transfer2go/utils"
)

// GET methods

// FilesHandler provides information about files in catalog
func FilesHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	pattern := r.FormValue("pattern")
	files := model.TFC.Files(pattern)
	data, err := json.Marshal(files)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// StatusHandler provides information about the agent
func StatusHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	addrs := utils.HostIP()
	astats := common.AgentStatus{Addrs: addrs, Catalog: model.TFC.Type, Name: _alias, Url: _myself, Protocol: _protocol, Backend: _backend, Tool: _tool, TransferCounter: model.TransferCounter, Agents: _agents, TimeStamp: time.Now().Unix()}
	data, err := json.Marshal(astats)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// AgentsHandler serves list of known agents
func AgentsHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	data, err := json.Marshal(_agents)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// DefaultHandler provides information about the agent
func DefaultHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	w.WriteHeader(http.StatusOK)
	// TODO: implement here default page for data-service
	// should be done via templates
	msg := fmt.Sprintf("Default page: %v\nagents: %v\n", time.Now(), _agents)
	w.Write([]byte(msg))
}

// ResetHandler resets current agent with default protocol and null backend/tool attributes
func ResetHandler(w http.ResponseWriter, r *http.Request) {
	_protocol = "http"
	_backend = ""
	_tool = ""
	log.Printf("ResetHandler switched to protocol=%s backend=%s tool=%s\n", _protocol, _backend, _tool)
	w.WriteHeader(http.StatusOK)
}

// POST methods

// TFCHandler registers given record in local TFC
func TFCHandler(w http.ResponseWriter, r *http.Request) {

	if !(r.Method == "POST" || r.Method == "GET") {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	if r.Method == "GET" {
		data := model.TFC.Dump()
		w.WriteHeader(http.StatusOK)
		w.Write(data)
		return
	}
	var entry model.CatalogEntry
	err := json.NewDecoder(r.Body).Decode(&entry)
	if err != nil {
		log.Println("ERROR TFCHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	err = model.TFC.Add(entry)
	if err != nil {
		log.Println("ERROR TFCHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

// RegisterAgentHandler registers current agent with another one
func RegisterAgentHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	var agentParams AgentInfo
	err := json.NewDecoder(r.Body).Decode(&agentParams)
	if err != nil {
		log.Println("ERROR RegisterAgentHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// register another agent
	agent := agentParams.Agent
	alias := agentParams.Alias
	if aurl, ok := _agents[alias]; ok {
		if aurl != agent {
			msg := fmt.Sprintf("Agent %s (%s) already exists in agents map, %v\n", alias, aurl, _agents)
			w.WriteHeader(http.StatusConflict)
			w.Write([]byte(msg))
			return
		}
		w.WriteHeader(http.StatusOK)
	} else {
		_agents[alias] = agent // register given agent/alias pair internally
		w.WriteHeader(http.StatusOK)
	}
}

// RegisterProtocolHandler registers current agent with another one
func RegisterProtocolHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	var protocolParams AgentProtocol
	err := json.NewDecoder(r.Body).Decode(&protocolParams)
	if err != nil {
		log.Println("ERROR RegisterProtocolHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// register another protocol for myself
	if stat, err := os.Stat(protocolParams.Tool); err == nil && !stat.IsDir() {
		_protocol = protocolParams.Protocol
		_backend = protocolParams.Backend
		_tool = protocolParams.Tool
		log.Printf("INFO RegisterProtocolHandler switched to protocol=%s backend=%s tool=%s\n", _protocol, _backend, _tool)
		w.WriteHeader(http.StatusOK)
		return
	}
}

// RequestHandler initiate transfer work for given request
func RequestHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	if utils.VERBOSE > 0 {
		log.Println("RequestHandler received request", r)
	}

	// Read the body into a string for json decoding
	var content = &model.TransferCollection{}
	err := json.NewDecoder(r.Body).Decode(&content)
	if err != nil {
		log.Println("ERROR RequestHandler unable to decode TransferCollection", err)
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// go through each request and queue items individually to run job over the given request
	for _, rdoc := range content.Requests {

		// let's create a job with the payload
		work := model.Job{TransferRequest: rdoc}

		// Push the work onto the queue.
		model.JobQueue <- work
	}

	w.WriteHeader(http.StatusOK)
}

// TransferDataHandler handles TransferData type received over HTTP
func TransferDataHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	var td model.TransferData
	err := json.NewDecoder(r.Body).Decode(&td)
	if err != nil {
		log.Println("ERROR TransferDataHandler unable to unmarshal incoming data", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// parse request
	arr := strings.Split(td.File, "/")
	fname := arr[len(arr)-1]
	// TODO: I need to revisit how to construct pfn on an agent during upload
	pfn := fmt.Sprintf("%s/%s", _backend, fname)
	err = ioutil.WriteFile(fname, td.Data, 0666)
	if err != nil {
		log.Println("ERROR, TransferHandler unable to write file", fname)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// verify hash, bytes of transferred data
	hash, bytes := utils.Hash(td.Data)
	if hash != td.Hash {
		log.Println("ERROR, TransferHandler written file has different hash", hash, td.Hash)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	if bytes != td.Bytes {
		log.Println("ERROR, TransferHandler written file has different number of bytes", bytes, td.Bytes)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Printf("wrote %s:%s to %s:%s\n", td.SrcAlias, fname, td.DstAlias, pfn)
	if model.TFC.Type != "filesystem" {
		entry := model.CatalogEntry{Lfn: td.File, Pfn: pfn, Dataset: td.Dataset, Block: td.Block, Bytes: bytes, Hash: hash}
		model.TFC.Add(entry)
	}
	w.WriteHeader(http.StatusOK)
}
<commit_msg>Change TransferDataHandler to UploadDataHandler<commit_after>package server

// transfer2go agent server implementation
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/vkuznet/transfer2go/common"
	"github.com/vkuznet/transfer2go/model"
	"github.com/vkuznet/transfer2go/utils"
)

// GET methods

// FilesHandler provides information about files in catalog
func FilesHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	pattern := r.FormValue("pattern")
	files := model.TFC.Files(pattern)
	data, err := json.Marshal(files)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// StatusHandler provides information about the agent
func StatusHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	addrs := utils.HostIP()
	astats := common.AgentStatus{Addrs: addrs, Catalog: model.TFC.Type, Name: _alias, Url: _myself, Protocol: _protocol, Backend: _backend, Tool: _tool, TransferCounter: model.TransferCounter, Agents: _agents, TimeStamp: time.Now().Unix()}
	data, err := json.Marshal(astats)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// AgentsHandler serves list of known agents
func AgentsHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	data, err := json.Marshal(_agents)
	if err != nil {
		log.Println("ERROR AgentsHandler", err)
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusOK)
	}
	w.Write(data)
}

// DefaultHandler provides information about the agent
func DefaultHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	w.WriteHeader(http.StatusOK)
	// TODO: implement here default page for data-service
	// should be done via templates
	msg := fmt.Sprintf("Default page: %v\nagents: %v\n", time.Now(), _agents)
	w.Write([]byte(msg))
}

// ResetHandler resets current agent with default protocol and null backend/tool attributes
func ResetHandler(w http.ResponseWriter, r *http.Request) {
	_protocol = "http"
	_backend = ""
	_tool = ""
	log.Printf("ResetHandler switched to protocol=%s backend=%s tool=%s\n", _protocol, _backend, _tool)
	w.WriteHeader(http.StatusOK)
}

// POST methods

// TFCHandler registers given record in local TFC
func TFCHandler(w http.ResponseWriter, r *http.Request) {

	if !(r.Method == "POST" || r.Method == "GET") {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	if r.Method == "GET" {
		data := model.TFC.Dump()
		w.WriteHeader(http.StatusOK)
		w.Write(data)
		return
	}
	var entry model.CatalogEntry
	err := json.NewDecoder(r.Body).Decode(&entry)
	if err != nil {
		log.Println("ERROR TFCHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	err = model.TFC.Add(entry)
	if err != nil {
		log.Println("ERROR TFCHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

// RegisterAgentHandler registers current agent with another one
func RegisterAgentHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	var agentParams AgentInfo
	err := json.NewDecoder(r.Body).Decode(&agentParams)
	if err != nil {
		log.Println("ERROR RegisterAgentHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// register another agent
	agent := agentParams.Agent
	alias := agentParams.Alias
	if aurl, ok := _agents[alias]; ok {
		if aurl != agent {
			msg := fmt.Sprintf("Agent %s (%s) already exists in agents map, %v\n", alias, aurl, _agents)
			w.WriteHeader(http.StatusConflict)
			w.Write([]byte(msg))
			return
		}
		w.WriteHeader(http.StatusOK)
	} else {
		_agents[alias] = agent // register given agent/alias pair internally
		w.WriteHeader(http.StatusOK)
	}
}

// RegisterProtocolHandler registers current agent with another one
func RegisterProtocolHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()

	var protocolParams AgentProtocol
	err := json.NewDecoder(r.Body).Decode(&protocolParams)
	if err != nil {
		log.Println("ERROR RegisterProtocolHandler unable to decode", r.Body, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// register another protocol for myself
	if stat, err := os.Stat(protocolParams.Tool); err == nil && !stat.IsDir() {
		_protocol = protocolParams.Protocol
		_backend = protocolParams.Backend
		_tool = protocolParams.Tool
		log.Printf("INFO RegisterProtocolHandler switched to protocol=%s backend=%s tool=%s\n", _protocol, _backend, _tool)
		w.WriteHeader(http.StatusOK)
		return
	}
}

// RequestHandler initiate transfer work for given request
func RequestHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	if utils.VERBOSE > 0 {
		log.Println("RequestHandler received request", r)
	}

	// Read the body into a string for json decoding
	var content = &model.TransferCollection{}
	err := json.NewDecoder(r.Body).Decode(&content)
	if err != nil {
		log.Println("ERROR RequestHandler unable to decode TransferCollection", err)
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// go through each request and queue items individually to run job over the given request
	for _, rdoc := range content.Requests {

		// let's create a job with the payload
		work := model.Job{TransferRequest: rdoc}

		// Push the work onto the queue.
		model.JobQueue <- work
	}

	w.WriteHeader(http.StatusOK)
}

// UploadDataHandler upload TransferData HTTP
func UploadDataHandler(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	defer r.Body.Close()
	var td model.TransferData
	err := json.NewDecoder(r.Body).Decode(&td)
	if err != nil {
		log.Println("ERROR UploadDataHandler unable to unmarshal incoming data", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// parse request
	arr := strings.Split(td.File, "/")
	fname := arr[len(arr)-1]
	// TODO: I need to revisit how to construct pfn on agent during upload
	pfn := fmt.Sprintf("%s/%s", _backend, fname)
	err = ioutil.WriteFile(fname, td.Data, 0666)
	if err != nil {
		log.Println("ERROR, UploadDataHandler unable to write file", fname)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	// verify hash, bytes of transferred data
	hash, bytes := utils.Hash(td.Data)
	if hash != td.Hash {
		log.Println("ERROR, UploadDataHandler written file has different hash", hash, td.Hash)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	if bytes != td.Bytes {
		log.Println("ERROR, UploadDataHandler written file has different number of bytes", bytes, td.Bytes)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Printf("wrote %s:%s to %s:%s\n", td.SrcAlias, fname, td.DstAlias, pfn)
	if model.TFC.Type != "filesystem" {
		entry := model.CatalogEntry{Lfn: td.File, Pfn: pfn, Dataset: td.Dataset, Block: td.Block, Bytes: bytes, Hash: hash}
		model.TFC.Add(entry)
	}
	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package amazonec2

import (
	"errors"
)

type region struct {
	AmiId string
}

// Ubuntu 16.04 LTS 20170619.1 hvm:ebs-ssd (amd64)
// See https://cloud-images.ubuntu.com/locator/ec2/
var regionDetails map[string]*region = map[string]*region{
	"ap-northeast-1":  {"ami-785c491f"},
	"ap-northeast-2":  {"ami-94d20dfa"},
	"ap-southeast-1":  {"ami-2378f540"},
	"ap-southeast-2":  {"ami-e94e5e8a"},
	"ap-south-1":      {"ami-49e59a26"},
	"ca-central-1":    {"ami-7ed56a1a"},
	"cn-north-1":      {"ami-a163b4cc"}, // Note: this is 20170303
	"eu-central-1":    {"ami-1c45e273"},
	"eu-west-1":       {"ami-6d48500b"},
	"eu-west-2":       {"ami-cc7066a8"},
	"eu-west-3":       {"ami-c1cf79bc"}, // Note: this is 20180126
	"sa-east-1":       {"ami-34afc458"},
	"us-east-1":       {"ami-d15a75c7"},
	"us-east-2":       {"ami-8b92b4ee"},
	"us-west-1":       {"ami-73f7da13"},
	"us-west-2":       {"ami-835b4efa"},
	"us-gov-west-1":   {"ami-939412f2"},
	"custom-endpoint": {""},
}

func awsRegionsList() []string {
	var list []string

	for k := range regionDetails {
		list = append(list, k)
	}

	return list
}

func validateAwsRegion(region string) (string, error) {
	for _, v := range awsRegionsList() {
		if v == region {
			return region, nil
		}
	}

	return "", errors.New("Invalid region specified")
}
<commit_msg>EC2: Update AMIs to mitigate Meltdown and Spectre<commit_after>package amazonec2

import (
	"errors"
)

type region struct {
	AmiId string
}

// Ubuntu 16.04 LTS 20180228.1 hvm:ebs-ssd (amd64)
// See https://cloud-images.ubuntu.com/locator/ec2/
var regionDetails map[string]*region = map[string]*region{
	"ap-northeast-1":  {"ami-bcb7f6da"},
	"ap-northeast-2":  {"ami-5073de3e"},
	"ap-southeast-1":  {"ami-41e4af3d"},
	"ap-southeast-2":  {"ami-c1498fa3"},
	"ap-south-1":      {"ami-1083dc7f"},
	"ca-central-1":    {"ami-8d9e19e9"},
	"cn-north-1":      {"ami-cc4499a1"}, // Note: this is 20180126
	"cn-northwest-1":  {"ami-fd0e1a9f"}, // Note: this is 20180126
	"eu-central-1":    {"ami-bc4925d3"},
	"eu-west-1":       {"ami-0b541372"},
	"eu-west-2":       {"ami-ff46a298"},
	"eu-west-3":       {"ami-9465d3e9"},
	"sa-east-1":       {"ami-b5501bd9"},
	"us-east-1":       {"ami-927185ef"},
	"us-east-2":       {"ami-b9daeddc"},
	"us-west-1":       {"ami-264c4646"},
	"us-west-2":       {"ami-78a22900"},
	"us-gov-west-1":   {"ami-2561ea44"},
	"custom-endpoint": {""},
}

func awsRegionsList() []string {
	var list []string

	for k := range regionDetails {
		list = append(list, k)
	}

	return list
}

func validateAwsRegion(region string) (string, error) {
	for _, v := range awsRegionsList() {
		if v == region {
			return region, nil
		}
	}

	return "", errors.New("Invalid region specified")
}
<|endoftext|>


<commit_before>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test is going to create volume on the fresh testbed very first time.
// After installing vmdk volume plugin/driver, volume creation should not be
// failed very first time.

// This test is going to cover the issue reported at #656
// TODO: as of now we are running the test against photon vm it should be run
// against various/applicable linux distros.

package e2e_test

import (
	volumeNameUtil "github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils"
	TestUtil "github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/test_util"
	"os"
	"testing"
)

// Test objective: create volume on fresh setup very first time and verifies the volume
// is created successfully.

/*

The issue was observed on photon so steps are mentioned for photon OS only in fact
test should be OS agnostic.

1. create docker volume using following command
	docker volume create --driver=vsphere --name=testVol -o size=10gb
2. verify volume is created correctly or not
3. delete created volume

expectation: volume should be created correctly

*/

func TestVolumeCreationFirstTime(t *testing.T) {
	var err error
	var out []byte
	volumeName := volumeNameUtil.GetVolumeNameWithTimeStamp("abc")

	// create volume
	out, err = TestUtil.CreateDefaultVolume(os.Getenv("VM1"), volumeName)

	if err != nil {
		t.Fatalf("\nError has occurred [%s] \n\twhile creating volume [%s] very first time: err -> %v", out, volumeName, err)
	} else {
		t.Logf("\nTestcase passed: successfully able to create volume [%s]\n", out)
		// delete volume
		TestUtil.DeleteVolume(volumeName, os.Getenv("VM1"))
	}
}
<commit_msg>Run the test against both of the test VMs we use now. (#1018)<commit_after>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test is going to create volume on the fresh testbed very first time.
// After installing vmdk volume plugin/driver, volume creation should not be
// failed very first time.

// This test is going to cover the issue reported at #656
// TODO: as of now we are running the test against photon vm it should be run
// against various/applicable linux distros.

package e2e_test

import (
	volumeNameUtil "github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils"
	TestUtil "github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/test_util"
	"os"
	"testing"
)

// Test objective: create volume on fresh setup very first time and verifies the volume
// is created successfully.

/*

The issue was observed on photon so steps are mentioned for photon OS only in fact
test should be OS agnostic.

1. create docker volume using following command
	docker volume create --driver=vsphere --name=testVol -o size=10gb
2. verify volume is created correctly or not
3. delete created volume

expectation: volume should be created correctly

*/

var dockerHosts = []string{os.Getenv("VM2"), os.Getenv("VM1")}

func TestVolumeCreationFirstTime(t *testing.T) {
	var err error
	var out []byte
	for _, host := range dockerHosts {
		volumeName := volumeNameUtil.GetVolumeNameWithTimeStamp("abc")

		// create volume
		out, err = TestUtil.CreateDefaultVolume(host, volumeName)

		if err != nil {
			t.Errorf("\nError has occurred [%s] \n\twhile creating volume [%s] very first time: err -> %v", out, volumeName, err)
		} else {
			t.Logf("\nTestcase passed: successfully able to create volume [%s]\n", out)
			// delete volume
			TestUtil.DeleteVolume(volumeName, host)
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brnstz/bus/common"
	"github.com/jmoiron/sqlx"
)

const maxStops = 3

type Trip struct {
	Id          string
	Headsign    string
	DirectionId int
}

type Service struct {
	Id      string
	RouteId string
}

type ScheduledStopTime struct {
	RouteId      string `db:"route_id"`
	StopId       string `db:"stop_id"`
	ServiceId    string `db:"service_id"`
	DepartureSec int    `db:"departure_sec"`
}

func df(t time.Time) string {
	return t.Format("2006-01-02")
}

type timeSlice []time.Time

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return p[i].Before(p[j])
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewScheduledStopTime(routeId, stopId, serviceId, timeStr string) (sst ScheduledStopTime, err error) {
	dsec := common.TimeStrToSecs(timeStr)

	sst = ScheduledStopTime{
		RouteId:      routeId,
		StopId:       stopId,
		ServiceId:    serviceId,
		DepartureSec: dsec,
	}

	return
}
func (s ScheduledStopTime) String() string {
	return fmt.Sprintf("{%v %v %v @ %v (%v)}",
		s.RouteId, s.ServiceId, s.StopId,
		common.SecsToTimeStr(s.DepartureSec), s.DepartureSec,
	)
}

type Stop struct {
	Id          string `json:"stop_id" db:"stop_id"`
	Name        string `json:"stop_name" db:"stop_name"`
	RouteId     string `json:"route_id" db:"route_id"`
	StationType string `json:"station_type" db:"stype"`

	DirectionId int    `json:"direction_id" db:"direction_id"`
	Headsign    string `json:"headsign" db:"headsign"`

	Lat float64 `json:"lat" db:"lat"`
	Lon float64 `json:"lon" db:"lon"`

	Dist float64 `json:"dist" db:"dist"`

	Scheduled []*Departure `json:"scheduled"`
	Live      []*Departure `json:"live"`
}

func (s *Stop) AppendLive() {
	calls, err := GetCallsByRouteStop(
		s.RouteId, strconv.Itoa(s.DirectionId),
		s.Id,
	)
	if err != nil {
		log.Println("can't append live schedules")
		return
	}

	sort.Sort(calls)
	for i := 0; i < len(calls) && i < maxStops; i++ {
		s.Live = append(s.Live, &Departure{
			Desc: calls[i].Extensions.Distances.PresentableDistance,
		})
	}
}

func (s Stop) String() string {
	return fmt.Sprintf("{%v %v %v %v @ (%v,%v)}",
		s.Id, s.Name, s.RouteId, s.Headsign, s.Lat, s.Lon,
	)
}

func (s Stop) Key() string {
	return fmt.Sprintf("%v%v", s.Id, s.RouteId)
}

func getServiceIdByDay(db sqlx.Ext, routeId, day string, now *time.Time) (serviceId string, err error) {
	row := db.QueryRowx(`
		SELECT service_id, route_id, max(start_date)
		FROM   service_route_day
		WHERE  day         = $1 AND
		       end_date    > $2 AND
			   route_id    = $3
		GROUP BY service_id, route_id
		LIMIT 1
	`, day, now, routeId,
	)

	var dummy1 string
	var dummy2 time.Time

	err = row.Scan(&serviceId, &dummy1, &dummy2)
	if err != nil {
		log.Println("can't scan service id", err, day, now, routeId)
		return
	}

	return
}

func GetStopsByLoc(db sqlx.Ext, lat, lon, meters float64, filter string) (stops []*Stop, err error) {

	stops = []*Stop{}
	params := []interface{}{lat, lon, lat, lon, meters}

	q := `
		SELECT * FROM (
			SELECT
				DISTINCT ON (route_id, direction_id)
				stop_id,
				stop_name,
				direction_id,
				headsign,
				route_id,
				stype,
				latitude(location) AS lat,
				longitude(location) AS lon,
				earth_distance(location, ll_to_earth($1, $2)) AS dist
			FROM stop
			WHERE earth_box(ll_to_earth($3, $4), $5) @> location
	`

	if len(filter) > 0 {
		q = q + ` AND stype = $6 `
		params = append(params, filter)
	}

	q = q + `
			ORDER BY route_id, direction_id
		) unique_routes

		ORDER BY dist ASC
	`

	err = sqlx.Select(db, &stops, q, params...)
	if err != nil {
		log.Println("can't get stop", err)
		return
	}

	now := time.Now()
	for _, stop := range stops {
		ydaysecs := []int64{}
		todaysecs := []int64{}

		allTimes := timeSlice{}

		yesterday := now.Add(-time.Hour * 12)
		yesterdayName := strings.ToLower(yesterday.Format("Monday"))
		todayName := strings.ToLower(now.Format("Monday"))

		if yesterdayName != todayName {
			var yesterdayId string
			// Looks for trips starting yesterday that arrive here
			// after midnight
			yesterdayId, err = getServiceIdByDay(
				db, stop.RouteId, yesterdayName, &now,
			)

			if err == sql.ErrNoRows {
				err = nil
				log.Println("no rows, ok, moving on")
				break
			}
			if err != nil {
				log.Println("can't get yesterday id", err)
				return
			}

			qYesterday := `
			SELECT scheduled_stop_time.departure_sec
			FROM   scheduled_stop_time
			WHERE  route_id   = $1 AND
			       stop_id    = $2 AND
				   service_id = $3 AND
				   departure_sec >= 86400 AND
				   departure_sec > $4
			ORDER BY departure_sec LIMIT $5
		`
			nowSecs :=
				now.Hour()*3600 + now.Minute()*60 + now.Second() + 86400

			err = sqlx.Select(db, &ydaysecs,
				qYesterday, stop.RouteId, stop.Id,
				yesterdayId, nowSecs, maxStops)

			if err != nil {
				log.Println("can't scan yesterday values", err)
				return
			}

			yesterday = yesterday.Add(
				-time.Hour * time.Duration(yesterday.Hour()))
			yesterday = yesterday.Add(
				-time.Minute * time.Duration(yesterday.Minute()))
			yesterday = yesterday.Add(
				-time.Second * time.Duration(yesterday.Second()))
			yesterday = yesterday.Add(
				-time.Nanosecond * time.Duration(yesterday.Nanosecond()))

			for _, ydaysec := range ydaysecs {
				thisTime := yesterday.Add(time.Second * time.Duration(ydaysec))
				allTimes = append(allTimes, thisTime)
			}
		}

		if true {
			var todayId string
			todayId, err = getServiceIdByDay(db, stop.RouteId, todayName, &now)
			if err == sql.ErrNoRows {
				err = nil
				log.Println("no rows there", err)
				break
			}
			if err != nil {
				log.Println("can't get today id", err)
				return
			}

			qToday := `
			SELECT scheduled_stop_time.departure_sec
			FROM   scheduled_stop_time
			WHERE  route_id   = $1 AND
			       stop_id    = $2 AND
				   service_id = $3 AND
				   departure_sec > $4
			ORDER BY departure_sec LIMIT $5
		`

			nowSecs := now.Hour()*3600 + now.Minute()*60 + now.Second()
			err = sqlx.Select(db, &todaysecs, qToday, stop.RouteId, stop.Id,
				todayId, nowSecs, maxStops)

			today := now
			today = today.Add(
				-time.Hour * time.Duration(today.Hour()))
			today = today.Add(
				-time.Minute * time.Duration(today.Minute()))
			today = today.Add(
				-time.Second * time.Duration(today.Second()))
			today = today.Add(
				-time.Nanosecond * time.Duration(today.Nanosecond()))

			for _, todaysec := range todaysecs {
				thisTime := today.Add(time.Second * time.Duration(todaysec))
				allTimes = append(allTimes, thisTime)
			}

			if err != nil {
				log.Println("can't scan today values", err)
				return
			}
		}

		sort.Sort(allTimes)

		for i, thisTime := range allTimes {
			if i > maxStops {
				break
			}
			stop.Scheduled = append(
				stop.Scheduled, &Departure{Time: thisTime},
			)
		}
		stop.AppendLive()
	}

	return stops, err
}

type ServiceRouteDay struct {
	ServiceId string
	RouteId   string
	Day       string

	StartDate time.Time
	EndDate   time.Time
}

func (s ServiceRouteDay) String() string {
	return fmt.Sprintf("{%v %v %v %v %v}",
		s.ServiceId, s.RouteId, s.Day, df(s.StartDate), df(s.EndDate),
	)
}

type ServiceRouteException struct {
	ServiceId     string
	RouteId       string
	ExceptionDate time.Time
}

type Departure struct {
	Time time.Time `json:"time" db:"time"`
	Desc string    `json:"desc"`

	// FIXME: stops away? miles away?
}
<commit_msg>fixme comment<commit_after>package models

import (
	"database/sql"
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brnstz/bus/common"
	"github.com/jmoiron/sqlx"
)

const maxStops = 3

type Trip struct {
	Id          string
	Headsign    string
	DirectionId int
}

type Service struct {
	Id      string
	RouteId string
}

type ScheduledStopTime struct {
	RouteId      string `db:"route_id"`
	StopId       string `db:"stop_id"`
	ServiceId    string `db:"service_id"`
	DepartureSec int    `db:"departure_sec"`
}

func df(t time.Time) string {
	return t.Format("2006-01-02")
}

type timeSlice []time.Time

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return p[i].Before(p[j])
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewScheduledStopTime(routeId, stopId, serviceId, timeStr string) (sst ScheduledStopTime, err error) {
	dsec := common.TimeStrToSecs(timeStr)

	sst = ScheduledStopTime{
		RouteId:      routeId,
		StopId:       stopId,
		ServiceId:    serviceId,
		DepartureSec: dsec,
	}

	return
}
func (s ScheduledStopTime) String() string {
	return fmt.Sprintf("{%v %v %v @ %v (%v)}",
		s.RouteId, s.ServiceId, s.StopId,
		common.SecsToTimeStr(s.DepartureSec), s.DepartureSec,
	)
}

type Stop struct {
	Id          string `json:"stop_id" db:"stop_id"`
	Name        string `json:"stop_name" db:"stop_name"`
	RouteId     string `json:"route_id" db:"route_id"`
	StationType string `json:"station_type" db:"stype"`

	DirectionId int    `json:"direction_id" db:"direction_id"`
	Headsign    string `json:"headsign" db:"headsign"`

	Lat float64 `json:"lat" db:"lat"`
	Lon float64 `json:"lon" db:"lon"`

	Dist float64 `json:"dist" db:"dist"`

	Scheduled []*Departure `json:"scheduled"`
	Live      []*Departure `json:"live"`
}

func (s *Stop) AppendLive() {
	calls, err := GetCallsByRouteStop(
		s.RouteId, strconv.Itoa(s.DirectionId),
		s.Id,
	)
	if err != nil {
		log.Println("can't append live schedules")
		return
	}

	sort.Sort(calls)
	for i := 0; i < len(calls) && i < maxStops; i++ {
		s.Live = append(s.Live, &Departure{
			Desc: calls[i].Extensions.Distances.PresentableDistance,
		})
	}
}

func (s Stop) String() string {
	return fmt.Sprintf("{%v %v %v %v @ (%v,%v)}",
		s.Id, s.Name, s.RouteId, s.Headsign, s.Lat, s.Lon,
	)
}

func (s Stop) Key() string {
	return fmt.Sprintf("%v%v", s.Id, s.RouteId)
}

func getServiceIdByDay(db sqlx.Ext, routeId, day string, now *time.Time) (serviceId string, err error) {
	row := db.QueryRowx(`
		SELECT service_id, route_id, max(start_date)
		FROM   service_route_day
		WHERE  day         = $1 AND
		       end_date    > $2 AND
			   route_id    = $3
		GROUP BY service_id, route_id
		LIMIT 1
	`, day, now, routeId,
	)

	var dummy1 string
	var dummy2 time.Time

	err = row.Scan(&serviceId, &dummy1, &dummy2)
	if err != nil {
		log.Println("can't scan service id", err, day, now, routeId)
		return
	}

	return
}

func GetStopsByLoc(db sqlx.Ext, lat, lon, meters float64, filter string) (stops []*Stop, err error) {

	stops = []*Stop{}
	params := []interface{}{lat, lon, lat, lon, meters}

	// FIXME: this is not returning the closet stop
	q := `
		SELECT * FROM (
			SELECT
				DISTINCT ON (route_id, direction_id)
				stop_id,
				stop_name,
				direction_id,
				headsign,
				route_id,
				stype,
				latitude(location) AS lat,
				longitude(location) AS lon,
				earth_distance(location, ll_to_earth($1, $2)) AS dist
			FROM stop
			WHERE earth_box(ll_to_earth($3, $4), $5) @> location
	`

	if len(filter) > 0 {
		q = q + ` AND stype = $6 `
		params = append(params, filter)
	}

	q = q + `
			ORDER BY route_id, direction_id
		) unique_routes

		ORDER BY dist ASC
	`

	err = sqlx.Select(db, &stops, q, params...)
	if err != nil {
		log.Println("can't get stop", err)
		return
	}

	now := time.Now()
	for _, stop := range stops {
		ydaysecs := []int64{}
		todaysecs := []int64{}

		allTimes := timeSlice{}

		yesterday := now.Add(-time.Hour * 12)
		yesterdayName := strings.ToLower(yesterday.Format("Monday"))
		todayName := strings.ToLower(now.Format("Monday"))

		if yesterdayName != todayName {
			var yesterdayId string
			// Looks for trips starting yesterday that arrive here
			// after midnight
			yesterdayId, err = getServiceIdByDay(
				db, stop.RouteId, yesterdayName, &now,
			)

			if err == sql.ErrNoRows {
				err = nil
				log.Println("no rows, ok, moving on")
				break
			}
			if err != nil {
				log.Println("can't get yesterday id", err)
				return
			}

			qYesterday := `
			SELECT scheduled_stop_time.departure_sec
			FROM   scheduled_stop_time
			WHERE  route_id   = $1 AND
			       stop_id    = $2 AND
				   service_id = $3 AND
				   departure_sec >= 86400 AND
				   departure_sec > $4
			ORDER BY departure_sec LIMIT $5
		`
			nowSecs :=
				now.Hour()*3600 + now.Minute()*60 + now.Second() + 86400

			err = sqlx.Select(db, &ydaysecs,
				qYesterday, stop.RouteId, stop.Id,
				yesterdayId, nowSecs, maxStops)

			if err != nil {
				log.Println("can't scan yesterday values", err)
				return
			}

			yesterday = yesterday.Add(
				-time.Hour * time.Duration(yesterday.Hour()))
			yesterday = yesterday.Add(
				-time.Minute * time.Duration(yesterday.Minute()))
			yesterday = yesterday.Add(
				-time.Second * time.Duration(yesterday.Second()))
			yesterday = yesterday.Add(
				-time.Nanosecond * time.Duration(yesterday.Nanosecond()))

			for _, ydaysec := range ydaysecs {
				thisTime := yesterday.Add(time.Second * time.Duration(ydaysec))
				allTimes = append(allTimes, thisTime)
			}
		}

		if true {
			var todayId string
			todayId, err = getServiceIdByDay(db, stop.RouteId, todayName, &now)
			if err == sql.ErrNoRows {
				err = nil
				log.Println("no rows there", err)
				break
			}
			if err != nil {
				log.Println("can't get today id", err)
				return
			}

			qToday := `
			SELECT scheduled_stop_time.departure_sec
			FROM   scheduled_stop_time
			WHERE  route_id   = $1 AND
			       stop_id    = $2 AND
				   service_id = $3 AND
				   departure_sec > $4
			ORDER BY departure_sec LIMIT $5
		`

			nowSecs := now.Hour()*3600 + now.Minute()*60 + now.Second()
			err = sqlx.Select(db, &todaysecs, qToday, stop.RouteId, stop.Id,
				todayId, nowSecs, maxStops)

			today := now
			today = today.Add(
				-time.Hour * time.Duration(today.Hour()))
			today = today.Add(
				-time.Minute * time.Duration(today.Minute()))
			today = today.Add(
				-time.Second * time.Duration(today.Second()))
			today = today.Add(
				-time.Nanosecond * time.Duration(today.Nanosecond()))

			for _, todaysec := range todaysecs {
				thisTime := today.Add(time.Second * time.Duration(todaysec))
				allTimes = append(allTimes, thisTime)
			}

			if err != nil {
				log.Println("can't scan today values", err)
				return
			}
		}

		sort.Sort(allTimes)

		for i, thisTime := range allTimes {
			if i > maxStops {
				break
			}
			stop.Scheduled = append(
				stop.Scheduled, &Departure{Time: thisTime},
			)
		}
		stop.AppendLive()
	}

	return stops, err
}

type ServiceRouteDay struct {
	ServiceId string
	RouteId   string
	Day       string

	StartDate time.Time
	EndDate   time.Time
}

func (s ServiceRouteDay) String() string {
	return fmt.Sprintf("{%v %v %v %v %v}",
		s.ServiceId, s.RouteId, s.Day, df(s.StartDate), df(s.EndDate),
	)
}

type ServiceRouteException struct {
	ServiceId     string
	RouteId       string
	ExceptionDate time.Time
}

type Departure struct {
	Time time.Time `json:"time" db:"time"`
	Desc string    `json:"desc"`

	// FIXME: stops away? miles away?
}
<|endoftext|>


<commit_before>package needle

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ParsedUpload struct {
	FileName    string
	Data        []byte
	bytesBuffer *bytes.Buffer
	MimeType    string
	PairMap     map[string]string
	IsGzipped   bool
	// IsZstd           bool
	OriginalDataSize int
	ModifiedTime     uint64
	Ttl              *TTL
	IsChunkedFile    bool
	UncompressedData []byte
	ContentMd5       string
}

func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (pu *ParsedUpload, e error) {
	bytesBuffer.Reset()
	pu = &ParsedUpload{bytesBuffer: bytesBuffer}
	pu.PairMap = make(map[string]string)
	for k, v := range r.Header {
		if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
			pu.PairMap[k] = v[0]
		}
	}

	if r.Method == "POST" {
		e = parseMultipart(r, sizeLimit, pu)
	} else {
		e = parsePut(r, sizeLimit, pu)
	}
	if e != nil {
		return
	}

	pu.ModifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
	pu.Ttl, _ = ReadTTL(r.FormValue("ttl"))

	pu.OriginalDataSize = len(pu.Data)
	pu.UncompressedData = pu.Data
	// println("received data", len(pu.Data), "isGzipped", pu.IsGzipped, "mime", pu.MimeType, "name", pu.FileName)
	if pu.IsGzipped {
		if unzipped, e := util.DecompressData(pu.Data); e == nil {
			pu.OriginalDataSize = len(unzipped)
			pu.UncompressedData = unzipped
			// println("ungzipped data size", len(unzipped))
		}
	} else {
		ext := filepath.Base(pu.FileName)
		mimeType := pu.MimeType
		if mimeType == "" {
			mimeType = http.DetectContentType(pu.Data)
		}
		// println("detected mimetype to", pu.MimeType)
		if mimeType == "application/octet-stream" {
			mimeType = ""
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); mimeType == "" && !iAmSure || shouldBeCompressed && iAmSure {
			// println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType)
			if compressedData, err := util.GzipData(pu.Data); err == nil {
				if len(compressedData)*10 < len(pu.Data)*9 {
					pu.Data = compressedData
					pu.IsGzipped = true
				}
				// println("gzipped data size", len(compressedData))
			}
		}
	}

	// md5
	h := md5.New()
	h.Write(pu.UncompressedData)
	pu.ContentMd5 = base64.StdEncoding.EncodeToString(h.Sum(nil))
	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
		if expectedChecksum != pu.ContentMd5 {
			e = fmt.Errorf("Content-MD5 did not match md5 of file data expected [%s] received [%s] size %d", expectedChecksum, pu.ContentMd5, len(pu.UncompressedData))
			return
		}
	}

	return
}

func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
	pu.MimeType = r.Header.Get("Content-Type")
	pu.FileName = ""
	dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
	if err == io.EOF || dataSize == sizeLimit+1 {
		io.Copy(ioutil.Discard, r.Body)
	}
	pu.Data = pu.bytesBuffer.Bytes()
	r.Body.Close()
	return nil
}

func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
	defer func() {
		if e != nil && r.Body != nil {
			io.Copy(ioutil.Discard, r.Body)
			r.Body.Close()
		}
	}()
	form, fe := r.MultipartReader()
	if fe != nil {
		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
		e = fe
		return
	}

	// first multi-part item
	part, fe := form.NextPart()
	if fe != nil {
		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
		e = fe
		return
	}

	pu.FileName = part.FileName()
	if pu.FileName != "" {
		pu.FileName = path.Base(pu.FileName)
	}

	var dataSize int64
	dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
	if e != nil {
		glog.V(0).Infoln("Reading Content [ERROR]", e)
		return
	}
	if dataSize == sizeLimit+1 {
		e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
		return
	}
	pu.Data = pu.bytesBuffer.Bytes()

	// if the filename is empty string, do a search on the other multi-part items
	for pu.FileName == "" {
		part2, fe := form.NextPart()
		if fe != nil {
			break // no more or on error, just safely break
		}

		fName := part2.FileName()

		// found the first <file type> multi-part has filename
		if fName != "" {
			pu.bytesBuffer.Reset()
			dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
			if fe2 != nil {
				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
				e = fe2
				return
			}
			if dataSize2 == sizeLimit+1 {
				e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
				return
			}

			// update
			pu.Data = pu.bytesBuffer.Bytes()
			pu.FileName = path.Base(fName)
			break
		}
	}

	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))

	if !pu.IsChunkedFile {

		dotIndex := strings.LastIndex(pu.FileName, ".")
		ext, mtype := "", ""
		if dotIndex > 0 {
			ext = strings.ToLower(pu.FileName[dotIndex:])
			mtype = mime.TypeByExtension(ext)
		}
		contentType := part.Header.Get("Content-Type")
		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
			pu.MimeType = contentType // only return mime type if not deductable
			mtype = contentType
		}

	}
	pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
	// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"

	return
}
<commit_msg>do not try to compress if not sure about the file content<commit_after>package needle

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ParsedUpload struct {
	FileName    string
	Data        []byte
	bytesBuffer *bytes.Buffer
	MimeType    string
	PairMap     map[string]string
	IsGzipped   bool
	// IsZstd           bool
	OriginalDataSize int
	ModifiedTime     uint64
	Ttl              *TTL
	IsChunkedFile    bool
	UncompressedData []byte
	ContentMd5       string
}

func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (pu *ParsedUpload, e error) {
	bytesBuffer.Reset()
	pu = &ParsedUpload{bytesBuffer: bytesBuffer}
	pu.PairMap = make(map[string]string)
	for k, v := range r.Header {
		if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
			pu.PairMap[k] = v[0]
		}
	}

	if r.Method == "POST" {
		e = parseMultipart(r, sizeLimit, pu)
	} else {
		e = parsePut(r, sizeLimit, pu)
	}
	if e != nil {
		return
	}

	pu.ModifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
	pu.Ttl, _ = ReadTTL(r.FormValue("ttl"))

	pu.OriginalDataSize = len(pu.Data)
	pu.UncompressedData = pu.Data
	// println("received data", len(pu.Data), "isGzipped", pu.IsGzipped, "mime", pu.MimeType, "name", pu.FileName)
	if pu.IsGzipped {
		if unzipped, e := util.DecompressData(pu.Data); e == nil {
			pu.OriginalDataSize = len(unzipped)
			pu.UncompressedData = unzipped
			// println("ungzipped data size", len(unzipped))
		}
	} else {
		ext := filepath.Base(pu.FileName)
		mimeType := pu.MimeType
		if mimeType == "" {
			mimeType = http.DetectContentType(pu.Data)
		}
		// println("detected mimetype to", pu.MimeType)
		if mimeType == "application/octet-stream" {
			mimeType = ""
		}
		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); shouldBeCompressed && iAmSure {
			// println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType)
			if compressedData, err := util.GzipData(pu.Data); err == nil {
				if len(compressedData)*10 < len(pu.Data)*9 {
					pu.Data = compressedData
					pu.IsGzipped = true
				}
				// println("gzipped data size", len(compressedData))
			}
		}
	}

	// md5
	h := md5.New()
	h.Write(pu.UncompressedData)
	pu.ContentMd5 = base64.StdEncoding.EncodeToString(h.Sum(nil))
	if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
		if expectedChecksum != pu.ContentMd5 {
			e = fmt.Errorf("Content-MD5 did not match md5 of file data expected [%s] received [%s] size %d", expectedChecksum, pu.ContentMd5, len(pu.UncompressedData))
			return
		}
	}

	return
}

func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
	pu.MimeType = r.Header.Get("Content-Type")
	pu.FileName = ""
	dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
	if err == io.EOF || dataSize == sizeLimit+1 {
		io.Copy(ioutil.Discard, r.Body)
	}
	pu.Data = pu.bytesBuffer.Bytes()
	r.Body.Close()
	return nil
}

func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
	defer func() {
		if e != nil && r.Body != nil {
			io.Copy(ioutil.Discard, r.Body)
			r.Body.Close()
		}
	}()
	form, fe := r.MultipartReader()
	if fe != nil {
		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
		e = fe
		return
	}

	// first multi-part item
	part, fe := form.NextPart()
	if fe != nil {
		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
		e = fe
		return
	}

	pu.FileName = part.FileName()
	if pu.FileName != "" {
		pu.FileName = path.Base(pu.FileName)
	}

	var dataSize int64
	dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
	if e != nil {
		glog.V(0).Infoln("Reading Content [ERROR]", e)
		return
	}
	if dataSize == sizeLimit+1 {
		e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
		return
	}
	pu.Data = pu.bytesBuffer.Bytes()

	// if the filename is empty string, do a search on the other multi-part items
	for pu.FileName == "" {
		part2, fe := form.NextPart()
		if fe != nil {
			break // no more or on error, just safely break
		}

		fName := part2.FileName()

		// found the first <file type> multi-part has filename
		if fName != "" {
			pu.bytesBuffer.Reset()
			dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
			if fe2 != nil {
				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
				e = fe2
				return
			}
			if dataSize2 == sizeLimit+1 {
				e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
				return
			}

			// update
			pu.Data = pu.bytesBuffer.Bytes()
			pu.FileName = path.Base(fName)
			break
		}
	}

	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))

	if !pu.IsChunkedFile {

		dotIndex := strings.LastIndex(pu.FileName, ".")
		ext, mtype := "", ""
		if dotIndex > 0 {
			ext = strings.ToLower(pu.FileName[dotIndex:])
			mtype = mime.TypeByExtension(ext)
		}
		contentType := part.Header.Get("Content-Type")
		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
			pu.MimeType = contentType // only return mime type if not deductable
			mtype = contentType
		}

	}
	pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
	// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"

	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"github.com/brunetto/goutils/debug"
	"github.com/capnm/sysinfo" // TODO: can be thrown using syscall!!
	"bitbucket.org/brunetto/sltools/slt"
)

func main () () {
	defer debug.TimeMe(time.Now())

	if len(os.Args) < 3 {
		log.Fatal("Provide the ICs and how many timesteps you want to integrate")
	}

	var (
		timeLimit string
		err error
		kiraString string
		kiraArgs []string
		kiraWrappedCmd *exec.Cmd
		pathName, icsName, outName, errName, ext string
		icsFile, outFile, errFile *os.File
		regRes map[string]string
		done = make(chan string, 1)
		randomSeed string = ""
		u  *user.User
		host, wd string
	)

	pathName = filepath.Dir(os.Args[1])
	icsName = filepath.Base(os.Args[1])
	timeLimit = os.Args[2]

	if len(os.Args) == 4 {
		randomSeed = os.Args[3]
	}

	log.Println("###################################################")

	// Extract fileNameBody, round and ext
	log.Println("Extract files names")
	ext = filepath.Ext(icsName)
	regRes, err = slt.Reg(icsName)
	if err != nil {
		log.Println("Can't derive standard names from STDOUT => wrap it!!")
		errName = "err-" + icsName + ext
		outName = "out-" + icsName + ext
	} else {
		if regRes["prefix"] != "ics" {
			log.Fatalf("Please specify a STDIN file, found %v prefix", regRes["prefix"])
		}

		// Creating new filenames
		errName = "err-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
		outName = "out-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
	}

	if icsName, err = filepath.Abs(filepath.Join(pathName, icsName)); err != nil {
		log.Fatal("Error composing icsName: ", err)
	}
	if errName, err = filepath.Abs(filepath.Join(pathName, errName)); err != nil {
		log.Fatal("Error composing errName: ", err)
	}
	if outName, err = filepath.Abs(filepath.Join(pathName, outName)); err != nil {
		log.Fatal("Error composing outName: ", err)
	}

	if icsFile, err = os.Open(icsName); err != nil {log.Fatal(err)}
	defer icsFile.Close()
	if outFile, err = os.Create(outName); err != nil {log.Fatal(err)}
	defer outFile.Close()
	if errFile, err = os.Create(errName); err != nil {log.Fatal(err)}
	defer errFile.Close()

	errFile.WriteString("\n==============================\n")
	errFile.WriteString("\nStart with kiraWrap (" + time.Now().Format(time.RFC850) + ").\n")
	errFile.WriteString("\n==============================\n")

	log.Println("Assuming kira is in $HOME/bin/kira, if not, please copy it there... for sake of simplicity!:P")

	kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira")
	kiraArgs =  []string{"-t", timeLimit,// +  // number of timesteps to compute
				"-d", "1",// +  // log output interval
				"-D", "1",// +  // snapshot interval
				"-b", "1",// +  // frequency of full binary output
				"-f", "0",// +  // dynamical friction (0 = no friction, 1 = friction)
				"-n", "10",// +  // terminate if teh cluster remains with only 10 particles
				"-e", "0.000",// + // softening
				"-B",// // switch on binary evolution
				//"-s 36543" // random seed
	}

	// Add the random seed if specified
	if randomSeed != "" {
		kiraArgs = append(kiraArgs, "-s", randomSeed)
	}

	kiraWrappedCmd = exec.Command(kiraString, kiraArgs...)
	if kiraWrappedCmd.Stdin = icsFile; err != nil {log.Fatal("Error connecting ICs to kira STDIN: ", err)}
	if kiraWrappedCmd.Stdout = outFile; err != nil {log.Fatal("Error connecting ICs to kira STDOUT: ", err)}
	if kiraWrappedCmd.Stderr = errFile; err != nil {log.Fatal("Error connecting ICs to kira STDERR: ", err)}

	if u, err = user.Current(); err != nil {
		log.Fatal("Can't retrieve username: ", err)
	}
	if host, err = os.Hostname(); err != nil {
		log.Fatal("Can't retrieve hostname: ", err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve working dir: ", err)
	}

	log.Println("Run summary:")

	log.Printf("Username: %v (%v)\n", u.Username, u.Name)
	log.Println("Hostname: ", host)
	log.Println("Working dir: ", wd)
	log.Println("LD_LIBRARY_PATH: ", os.Getenv("LD_LIBRARY_PATH"))

	log.Println("Command: ", kiraString, kiraArgs)
	log.Println("STDIN = ", icsName)
	log.Println("STDOUT = ", outName)
	log.Println("STDERR = ", errName)

	log.Println("Ready... steady... Go!")

	if err = kiraWrappedCmd.Start(); err != nil {
		log.Fatal("Error starting kiraWrappedCmd: ", err)
	}

	log.Println("Waiting kira to finish while checking for problems...")
	// Wait for the process to end normally
	go waitProcess(kiraWrappedCmd, done)
	// Check for pp3-stalling situations
	go killTrigger(errName, kiraWrappedCmd, done)

	if err = errors.New(<-done); err.Error() != "" {
		errFile.WriteString("\n"+err.Error()+"\n")
	}

	errFile.WriteString("\n==============================\n")
	errFile.WriteString("\nDone with kiraWrap (" + time.Now().Format(time.RFC850) + ").\n")
	errFile.WriteString("\n==============================\n")
	fmt.Print("\x07") // Beep when finish!!:D
}

func killTrigger(errName string, kiraWrappedCmd *exec.Cmd, done chan string) () {
	const toGB = float64(1. / (1024*1024*1024))
	const maxStderrGB = float64(2)
	const minDiskGB = float64(5)
	const maxMemPerCent = float64(98)

	var (
		fileInfo os.FileInfo
		sysInfo *sysinfo.SI
		memAvail float64
		diskAvailGB float64
		wd string
		reason string
		err error
	)

	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve local dir: ", err)
	}

	fsInfo := syscall.Statfs_t{}

	for {
		// Check STDERR file size
		if fileInfo, err = os.Stat(errName); err != nil {
			log.Fatal("Error checking STDERR file size, err")
		}
		// STDERR exceesing aloowed dimension of 2GB
		// probably the simulation is stalling because
		// of pp3 locked on a binary
		if float64(fileInfo.Size()) * toGB > maxStderrGB {
			reason = "probable pp3 stalling"
			break
		}

		// Check memory availability
		sysInfo = sysinfo.Get()
		memAvail = float64(sysInfo.FreeRam) / float64(sysInfo.TotalRam) * 100
		if memAvail > maxMemPerCent {
			reason = fmt.Sprintf("memory less than 95%% on the system: %2.2f",  memAvail)
			break
		}

		// Check HDD space availability
		if err = syscall.Statfs(wd, &fsInfo); err!= nil{
			log.Fatal("Cant't retrieve file system information: ", err)
		}
		if diskAvailGB = float64(fsInfo.Bavail) * float64(fsInfo.Frsize) * toGB; diskAvailGB < minDiskGB {
			reason = " available disk space less than 5 GB on the system"
			break
		}

		// Wait some time
		time.Sleep(time.Duration(2) * time.Minute)
	}
	log.Println("Kill kira because ", reason)
	if err := kiraWrappedCmd.Process.Kill(); err != nil {
		log.Fatal("failed to kill: ", err)
	}
	done <- "killed because " + reason
}

func waitProcess(kiraWrappedCmd *exec.Cmd, done chan string) () {
	err := kiraWrappedCmd.Wait()
	log.Println("Process exited with error ", err)
	if err == nil {
		log.Println("Maybe the simulation is COMPLETE or the STD* are mixed and wrong.")
	}
	done <- ""
}

<commit_msg>update kiraWrap info on STD*<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"github.com/brunetto/goutils/debug"
	"github.com/capnm/sysinfo" // TODO: can be thrown using syscall!!
	"bitbucket.org/brunetto/sltools/slt"
)

func main () () {
	defer debug.TimeMe(time.Now())

	if len(os.Args) < 3 {
		log.Fatal("Provide the ICs and how many timesteps you want to integrate")
	}

	var (
		timeLimit string
		err error
		kiraString string
		kiraArgs []string
		kiraWrappedCmd *exec.Cmd
		pathName, icsName, outName, errName, ext string
		icsFile, outFile, errFile *os.File
		regRes map[string]string
		done = make(chan string, 1)
		randomSeed string = ""
		u  *user.User
		host, wd string
	)

	pathName = filepath.Dir(os.Args[1])
	icsName = filepath.Base(os.Args[1])
	timeLimit = os.Args[2]

	if len(os.Args) == 4 {
		randomSeed = os.Args[3]
	}

	log.Println("###################################################")

	// Extract fileNameBody, round and ext
	log.Println("Extract files names")
	ext = filepath.Ext(icsName)
	regRes, err = slt.Reg(icsName)
	if err != nil {
		log.Println("Can't derive standard names from STDOUT => wrap it!!")
		errName = "err-" + icsName + ext
		outName = "out-" + icsName + ext
	} else {
		if regRes["prefix"] != "ics" {
			log.Fatalf("Please specify a STDIN file, found %v prefix", regRes["prefix"])
		}

		// Creating new filenames
		errName = "err-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
		outName = "out-" + regRes["baseName"] + "-run" + regRes["run"] + "-rnd" + regRes["rnd"] + ext
	}

	if icsName, err = filepath.Abs(filepath.Join(pathName, icsName)); err != nil {
		log.Fatal("Error composing icsName: ", err)
	}
	if errName, err = filepath.Abs(filepath.Join(pathName, errName)); err != nil {
		log.Fatal("Error composing errName: ", err)
	}
	if outName, err = filepath.Abs(filepath.Join(pathName, outName)); err != nil {
		log.Fatal("Error composing outName: ", err)
	}

	if icsFile, err = os.Open(icsName); err != nil {log.Fatal(err)}
	defer icsFile.Close()
	if outFile, err = os.Create(outName); err != nil {log.Fatal(err)}
	defer outFile.Close()
	if errFile, err = os.Create(errName); err != nil {log.Fatal(err)}
	defer errFile.Close()

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString("\n#==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Start with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	log.Println("Assuming kira is in $HOME/bin/kira, if not, please copy it there... for sake of simplicity!:P")

	kiraString = filepath.Join(os.Getenv("HOME"), "/bin/", "kira")
	kiraArgs =  []string{"-t", timeLimit,// +  // number of timesteps to compute
				"-d", "1",// +  // log output interval
				"-D", "1",// +  // snapshot interval
				"-b", "1",// +  // frequency of full binary output
				"-f", "0",// +  // dynamical friction (0 = no friction, 1 = friction)
				"-n", "10",// +  // terminate if teh cluster remains with only 10 particles
				"-e", "0.000",// + // softening
				"-B",// // switch on binary evolution
				//"-s 36543" // random seed
	}

	// Add the random seed if specified
	if randomSeed != "" {
		kiraArgs = append(kiraArgs, "-s", randomSeed)
	}

	kiraWrappedCmd = exec.Command(kiraString, kiraArgs...)
	if kiraWrappedCmd.Stdin = icsFile; err != nil {log.Fatal("Error connecting ICs to kira STDIN: ", err)}
	if kiraWrappedCmd.Stdout = outFile; err != nil {log.Fatal("Error connecting ICs to kira STDOUT: ", err)}
	if kiraWrappedCmd.Stderr = errFile; err != nil {log.Fatal("Error connecting ICs to kira STDERR: ", err)}

	if u, err = user.Current(); err != nil {
		log.Fatal("Can't retrieve username: ", err)
	}
	if host, err = os.Hostname(); err != nil {
		log.Fatal("Can't retrieve hostname: ", err)
	}
	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve working dir: ", err)
	}

	log.Println("Run summary:")

	log.Printf("Username: %v (%v)\n", u.Username, u.Name)
	log.Println("Hostname: ", host)
	log.Println("Working dir: ", wd)
	log.Println("LD_LIBRARY_PATH: ", os.Getenv("LD_LIBRARY_PATH"))

	log.Println("Command: ", kiraString, kiraArgs)
	log.Println("STDIN = ", icsName)
	log.Println("STDOUT = ", outName)
	log.Println("STDERR = ", errName)

	log.Println("Ready... steady... Go!")

	if err = kiraWrappedCmd.Start(); err != nil {
		log.Fatal("Error starting kiraWrappedCmd: ", err)
	}

	log.Println("Waiting kira to finish while checking for problems...")
	// Wait for the process to end normally
	go waitProcess(kiraWrappedCmd, done)
	// Check for pp3-stalling situations
	go killTrigger(errName, kiraWrappedCmd, done)

	if err = errors.New(<-done); err.Error() != "" {
		errFile.WriteString("\n"+err.Error()+"\n")
	}

	errFile.WriteString("\n#==============================\n")
	errFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	errFile.WriteString(fmt.Sprintf("\n#   Username: %v (%V)\n", u.Username, u.Name))
	errFile.WriteString(fmt.Sprintf("\n#   Hostname: %v\n", host))
	errFile.WriteString("\n==============================\n")

	outFile.WriteString("\n#==============================\n")
	outFile.WriteString(fmt.Sprintf("\n#   %v Done with kiraWrap.\n", time.Now().Format(time.RFC850)))
	outFile.WriteString("\n#==============================\n")

	fmt.Print("\x07") // Beep when finish!!:D
}

func killTrigger(errName string, kiraWrappedCmd *exec.Cmd, done chan string) () {
	const toGB = float64(1. / (1024*1024*1024))
	const maxStderrGB = float64(2)
	const minDiskGB = float64(5)
	const maxMemPerCent = float64(98)

	var (
		fileInfo os.FileInfo
		sysInfo *sysinfo.SI
		memAvail float64
		diskAvailGB float64
		wd string
		reason string
		err error
	)

	if wd, err = os.Getwd(); err != nil {
		log.Fatal("Can't retrieve local dir: ", err)
	}

	fsInfo := syscall.Statfs_t{}

	for {
		// Check STDERR file size
		if fileInfo, err = os.Stat(errName); err != nil {
			log.Fatal("Error checking STDERR file size, err")
		}
		// STDERR exceesing aloowed dimension of 2GB
		// probably the simulation is stalling because
		// of pp3 locked on a binary
		if float64(fileInfo.Size()) * toGB > maxStderrGB {
			reason = "probable pp3 stalling"
			break
		}

		// Check memory availability
		sysInfo = sysinfo.Get()
		memAvail = float64(sysInfo.FreeRam) / float64(sysInfo.TotalRam) * 100
		if memAvail > maxMemPerCent {
			reason = fmt.Sprintf("memory less than 95%% on the system: %2.2f",  memAvail)
			break
		}

		// Check HDD space availability
		if err = syscall.Statfs(wd, &fsInfo); err!= nil{
			log.Fatal("Cant't retrieve file system information: ", err)
		}
		if diskAvailGB = float64(fsInfo.Bavail) * float64(fsInfo.Frsize) * toGB; diskAvailGB < minDiskGB {
			reason = " available disk space less than 5 GB on the system"
			break
		}

		// Wait some time
		time.Sleep(time.Duration(2) * time.Minute)
	}
	log.Println("Kill kira because ", reason)
	if err := kiraWrappedCmd.Process.Kill(); err != nil {
		log.Fatal("failed to kill: ", err)
	}
	done <- "killed because " + reason
}

func waitProcess(kiraWrappedCmd *exec.Cmd, done chan string) () {
	err := kiraWrappedCmd.Wait()
	log.Println("Process exited with error ", err)
	if err == nil {
		log.Println("Maybe the simulation is COMPLETE or the STD* are mixed and wrong.")
	}
	done <- ""
}

<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */
package controller

import (
	"testing"

	v12 "k8s.io/api/core/v1"

	v1 "kubevirt.io/client-go/api/v1"
)

func TestAddPodCondition(t *testing.T) {

	vmi := v1.NewMinimalVMI("test")

	pc1 := &v12.PodCondition{
		Type:   v12.PodScheduled,
		Status: v12.ConditionFalse,
	}
	pc2 := &v12.PodCondition{
		Type:   v12.PodScheduled,
		Status: v12.ConditionTrue,
	}

	cm := NewVirtualMachineInstanceConditionManager()

	cm.AddPodCondition(vmi, pc1)
	cm.AddPodCondition(vmi, pc2)

	if len(vmi.Status.Conditions) != 1 {
		t.Errorf("There should be exactly 1 condition when muliple conditions of the same type were added")
	}
}
<commit_msg>Improved VMI ConditionManager test<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */
package controller

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	v12 "k8s.io/api/core/v1"

	v1 "kubevirt.io/client-go/api/v1"
)

var _ = Describe("VirtualMachineInstance ConditionManager", func() {

	var vmi *v1.VirtualMachineInstance
	var cm *VirtualMachineConditionManager
	var pc1 *v12.PodCondition
	var pc2 *v12.PodCondition

	BeforeEach(func() {
		vmi = v1.NewMinimalVMI("test")

		pc1 = &v12.PodCondition{
			Type:   v12.PodScheduled,
			Status: v12.ConditionFalse,
		}
		pc2 = &v12.PodCondition{
			Type:   v12.PodScheduled,
			Status: v12.ConditionTrue,
		}

		cm = NewVirtualMachineInstanceConditionManager()
	})

	When("Adding a condition", func() {

		It("should report condition available", func() {
			cm.AddPodCondition(vmi, pc1)
			Expect(cm.HasCondition(vmi, v1.VirtualMachineInstanceConditionType(pc1.Type))).To(BeTrue())
		})

		It("should report different condition not available", func() {
			cm.AddPodCondition(vmi, pc1)
			Expect(cm.HasCondition(vmi, v1.VirtualMachineInstanceConditionType(v12.PodInitialized))).To(BeFalse())
		})

		When("adding a 2nd condition of same type", func() {
			It("should only have 1 condition", func() {
				cm.AddPodCondition(vmi, pc1)
				cm.AddPodCondition(vmi, pc2)
				Expect(len(vmi.Status.Conditions)).To(Equal(1))
			})
		})
	})

	When("VMI is nil", func() {
		It("should gracefully report condition not available", func() {
			var vmi2 *v1.VirtualMachineInstance
			Expect(cm.HasCondition(vmi2, v1.VirtualMachineInstanceConditionType(pc1.Type))).To(BeFalse())
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"encoding/json"
	"net/http"

	"github.com/grafeas/kritis/pkg/kritis/admission/constants"
	"k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// This is very basic admission controller which allows all pods.
func AdmissionReviewHandler(w http.ResponseWriter, r *http.Request) {
	// TODO: Add image security policy check
	// 1. Get all Image security Polices
	// 2. Get all pod specs
	// 3. For all pods Get Vulnerabilities for images
	// 4. Validate policy
	// 5. Attest.
	status := &v1beta1.AdmissionResponse{
		Allowed: true,
		Result: &metav1.Status{
			Status:  string(constants.SuccessStatus),
			Message: constants.SuccessMessage,
		},
	}
	WriteHttpResponse(status, w)
}

func WriteHttpResponse(status *v1beta1.AdmissionResponse, w http.ResponseWriter) {
	ar := v1beta1.AdmissionReview{
		Response: status,
	}
	data, err := json.Marshal(ar)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	w.Write(data)
}
<commit_msg>Add error logging<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"encoding/json"
	"log"
	"net/http"

	"github.com/grafeas/kritis/pkg/kritis/admission/constants"
	"k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// This is very basic admission controller which allows all pods.
func AdmissionReviewHandler(w http.ResponseWriter, r *http.Request) {
	// TODO: Add image security policy check
	// 1. Get all Image security Polices
	// 2. Get all pod specs
	// 3. For all pods Get Vulnerabilities for images
	// 4. Validate policy
	// 5. Attest.
	status := &v1beta1.AdmissionResponse{
		Allowed: true,
		Result: &metav1.Status{
			Status:  string(constants.SuccessStatus),
			Message: constants.SuccessMessage,
		},
	}
	error := WriteHttpResponse(status, w)
	if error != nil {
		log.Printf("\nError writing response %s", error)
	}
}

func WriteHttpResponse(status *v1beta1.AdmissionResponse, w http.ResponseWriter) error {
	ar := v1beta1.AdmissionReview{
		Response: status,
	}
	data, err := json.Marshal(ar)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return nil
	}
	w.WriteHeader(http.StatusOK)
	_, error := w.Write(data)
	return error
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"time"

	opentracing "github.com/opentracing/opentracing-go"
	tlog "github.com/opentracing/opentracing-go/log"

	"gopkg.in/macaron.v1"
)

func RequestTracing() macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)

		var span opentracing.Span
		opName := req.URL.Path
		carrier := opentracing.HTTPHeadersCarrier(req.Header)

		wireContext, err := opentracing.GlobalTracer().Extract(
			opentracing.HTTPHeaders, carrier)
		if err != nil {
			span = opentracing.StartSpan(opName)
		} else {
			span = opentracing.StartSpan(opName, opentracing.ChildOf(wireContext))
		}
		defer span.Finish()

		ctx := opentracing.ContextWithSpan(req.Context(), span)
		req = req.WithContext(ctx)
		start := time.Now()

		c.Next()

		status := rw.Status()

		span.LogFields(
			tlog.Int("http.status_code", status),
			tlog.Float64("waited.millis", float64(time.Since(start)/time.Millisecond)))
	}
}
<commit_msg>use opentracing ext package when possible<commit_after>package middleware

import (
	"fmt"
	"net/http"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"

	"gopkg.in/macaron.v1"
)

func RequestTracing() macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)

		var span opentracing.Span
		tracer := opentracing.GlobalTracer()
		wireContext, _ := tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))
		spanName := fmt.Sprintf("HTTP %s %s", req.Method, req.URL.Path)
		span = tracer.StartSpan(spanName, ext.RPCServerOption(wireContext))
		defer span.Finish()

		ctx := opentracing.ContextWithSpan(req.Context(), span)
		req = req.WithContext(ctx)

		c.Next()

		status := rw.Status()

		ext.HTTPStatusCode.Set(span, uint16(status))
		ext.HTTPUrl.Set(span, req.RequestURI)
		ext.HTTPMethod.Set(span, req.Method)
	}
}
<|endoftext|>