<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"syscall"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/util/exec"
)

const (
	CrushRootConfigKey = "crushRoot"
)

// CrushMap is the go representation of a CRUSH map
type CrushMap struct {
	Devices []struct {
		ID    int    `json:"id"`
		Name  string `json:"name"`
		Class string `json:"class"`
	} `json:"devices"`
	Types []struct {
		ID   int    `json:"type_id"`
		Name string `json:"name"`
	} `json:"types"`
	Buckets []struct {
		ID       int    `json:"id"`
		Name     string `json:"name"`
		TypeID   int    `json:"type_id"`
		TypeName string `json:"type_name"`
		Weight   int    `json:"weight"`
		Alg      string `json:"alg"`
		Hash     string `json:"hash"`
		Items    []struct {
			ID     int `json:"id"`
			Weight int `json:"weight"`
			Pos    int `json:"pos"`
		} `json:"items"`
	} `json:"buckets"`
	Rules    []ruleSpec `json:"rules"`
	Tunables struct {
		// Add if necessary
	} `json:"tunables"`
}

type ruleSpec struct {
	ID      int        `json:"rule_id"`
	Name    string     `json:"rule_name"`
	Ruleset int        `json:"ruleset"`
	Type    int        `json:"type"`
	MinSize int        `json:"min_size"`
	MaxSize int        `json:"max_size"`
	Steps   []stepSpec `json:"steps"`
}

type stepSpec struct {
	Operation string `json:"op"`
	Number    uint   `json:"num"`
	Item      int    `json:"item"`
	ItemName  string `json:"item_name"`
	Type      string `json:"type"`
}

// CrushFindResult is go representation of the Ceph osd find command output
type CrushFindResult struct {
	ID       int               `json:"osd"`
	IP       string            `json:"ip"`
	Host     string            `json:"host,omitempty"`
	Location map[string]string `json:"crush_location"`
}

// GetCrushMap fetches the Ceph CRUSH map
func GetCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo) (CrushMap, error) {
	var c CrushMap
	args := []string{"osd", "crush", "dump"}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		return c, errors.Wrapf(err, "failed to get crush map. %s", string(buf))
	}

	err = json.Unmarshal(buf, &c)
	if err != nil {
		return c, errors.Wrap(err, "failed to unmarshal crush map")
	}

	return c, nil
}

// GetCompiledCrushMap fetches the Ceph compiled version of the CRUSH map
func GetCompiledCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
	compiledCrushMapFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrap(err, "failed to generate temporarily file")
	}

	args := []string{"osd", "getcrushmap", "--out-file", compiledCrushMapFile.Name()}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return "", errors.Wrapf(err, "failed to get compiled crush map. %s", string(buf))
	}

	return compiledCrushMapFile.Name(), nil
}

// FindOSDInCrushMap finds an OSD in the CRUSH map
func FindOSDInCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int) (*CrushFindResult, error) {
	args := []string{"osd", "find", strconv.Itoa(osdID)}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		return nil, errors.Wrapf(err, "failed to find osd.%d in crush map: %s", osdID, string(buf))
	}

	var result CrushFindResult
	if err := json.Unmarshal(buf, &result); err != nil {
		return nil, errors.Wrapf(err, "failed to unmarshal crush find result: %s", string(buf))
	}

	return &result, nil
}

// GetCrushHostName gets the hostname where an OSD is running on
func GetCrushHostName(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int) (string, error) {
	result, err := FindOSDInCrushMap(context, clusterInfo, osdID)
	if err != nil {
		return "", err
	}

	return result.Location["host"], nil
}

// NormalizeCrushName replaces . with -
func NormalizeCrushName(name string) string {
	return strings.Replace(name, ".", "-", -1)
}

// Obtain the cluster-wide default crush root from the cluster spec
func GetCrushRootFromSpec(c *cephv1.ClusterSpec) string {
	if c.Storage.Config == nil {
		return cephv1.DefaultCRUSHRoot
	}
	if v, ok := c.Storage.Config[CrushRootConfigKey]; ok {
		return v
	}
	return cephv1.DefaultCRUSHRoot
}

// IsNormalizedCrushNameEqual returns true if normalized is either equal to or the normalized version of notNormalized
// a crush name is normalized if it comes from the crushmap or has passed through the NormalizeCrushName function.
func IsNormalizedCrushNameEqual(notNormalized, normalized string) bool {
	if notNormalized == normalized || NormalizeCrushName(notNormalized) == normalized {
		return true
	}
	return false
}

// UpdateCrushMapValue is for updating the location in the crush map
// this is not safe for incorrectly formatted strings
func UpdateCrushMapValue(pairs *[]string, key, value string) {
	found := false
	property := formatProperty(key, value)
	for i, pair := range *pairs {
		entry := strings.Split(pair, "=")
		if key == entry[0] {
			(*pairs)[i] = property
			found = true
		}
	}
	if !found {
		*pairs = append(*pairs, property)
	}
}

func formatProperty(name, value string) string {
	return fmt.Sprintf("%s=%s", name, value)
}

// GetOSDOnHost returns the list of osds running on a given host
func GetOSDOnHost(context *clusterd.Context, clusterInfo *ClusterInfo, node string) (string, error) {
	args := []string{"osd", "crush", "ls", node}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		// ENOENT as error means that node is not (yet) part of the ceph cluster
		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
			return "", nil
		}
		return "", errors.Wrapf(err, "failed to get osd list on host. %s", string(buf))
	}

	return string(buf), nil
}

func compileCRUSHMap(context *clusterd.Context, crushMapPath string) error {
	mapFile := buildCompileCRUSHFileName(crushMapPath)
	args := []string{"--compile", crushMapPath, "--outfn", mapFile}
	output, err := context.Executor.ExecuteCommandWithOutput("crushtool", args...)
	if err != nil {
		return errors.Wrapf(err, "failed to compile crush map %q. %s", mapFile, output)
	}

	return nil
}

func decompileCRUSHMap(context *clusterd.Context, crushMapPath string) error {
	mapFile := buildDecompileCRUSHFileName(crushMapPath)
	args := []string{"--decompile", crushMapPath, "--outfn", mapFile}
	output, err := context.Executor.ExecuteCommandWithOutput("crushtool", args...)
	if err != nil {
		return errors.Wrapf(err, "failed to decompile crush map %q. %s", mapFile, output)
	}

	return nil
}

func injectCRUSHMap(context *clusterd.Context, clusterInfo *ClusterInfo, crushMapPath string) error {
	args := []string{"osd", "setcrushmap", "--in-file", crushMapPath}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return errors.Wrapf(err, "failed to inject crush map %q. %s", crushMapPath, string(buf))
	}

	return nil
}

func setCRUSHMap(context *clusterd.Context, clusterInfo *ClusterInfo, crushMapPath string) error {
	args := []string{"osd", "crush", "set", crushMapPath}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return errors.Wrapf(err, "failed to set crush map %q. %s", crushMapPath, string(buf))
	}

	return nil
}

func buildDecompileCRUSHFileName(crushMapPath string) string {
	return fmt.Sprintf("%s.decompiled", crushMapPath)
}

func buildCompileCRUSHFileName(crushMapPath string) string {
	return fmt.Sprintf("%s.compiled", crushMapPath)
}
<commit_msg>ceph: revert new node is not provisioned when useallnodes is true<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package client

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
)

const (
	CrushRootConfigKey = "crushRoot"
)

// CrushMap is the go representation of a CRUSH map
type CrushMap struct {
	Devices []struct {
		ID    int    `json:"id"`
		Name  string `json:"name"`
		Class string `json:"class"`
	} `json:"devices"`
	Types []struct {
		ID   int    `json:"type_id"`
		Name string `json:"name"`
	} `json:"types"`
	Buckets []struct {
		ID       int    `json:"id"`
		Name     string `json:"name"`
		TypeID   int    `json:"type_id"`
		TypeName string `json:"type_name"`
		Weight   int    `json:"weight"`
		Alg      string `json:"alg"`
		Hash     string `json:"hash"`
		Items    []struct {
			ID     int `json:"id"`
			Weight int `json:"weight"`
			Pos    int `json:"pos"`
		} `json:"items"`
	} `json:"buckets"`
	Rules    []ruleSpec `json:"rules"`
	Tunables struct {
		// Add if necessary
	} `json:"tunables"`
}

type ruleSpec struct {
	ID      int        `json:"rule_id"`
	Name    string     `json:"rule_name"`
	Ruleset int        `json:"ruleset"`
	Type    int        `json:"type"`
	MinSize int        `json:"min_size"`
	MaxSize int        `json:"max_size"`
	Steps   []stepSpec `json:"steps"`
}

type stepSpec struct {
	Operation string `json:"op"`
	Number    uint   `json:"num"`
	Item      int    `json:"item"`
	ItemName  string `json:"item_name"`
	Type      string `json:"type"`
}

// CrushFindResult is go representation of the Ceph osd find command output
type CrushFindResult struct {
	ID       int               `json:"osd"`
	IP       string            `json:"ip"`
	Host     string            `json:"host,omitempty"`
	Location map[string]string `json:"crush_location"`
}

// GetCrushMap fetches the Ceph CRUSH map
func GetCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo) (CrushMap, error) {
	var c CrushMap
	args := []string{"osd", "crush", "dump"}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		return c, errors.Wrapf(err, "failed to get crush map. %s", string(buf))
	}

	err = json.Unmarshal(buf, &c)
	if err != nil {
		return c, errors.Wrap(err, "failed to unmarshal crush map")
	}

	return c, nil
}

// GetCompiledCrushMap fetches the Ceph compiled version of the CRUSH map
func GetCompiledCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
	compiledCrushMapFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrap(err, "failed to generate temporarily file")
	}

	args := []string{"osd", "getcrushmap", "--out-file", compiledCrushMapFile.Name()}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return "", errors.Wrapf(err, "failed to get compiled crush map. %s", string(buf))
	}

	return compiledCrushMapFile.Name(), nil
}

// FindOSDInCrushMap finds an OSD in the CRUSH map
func FindOSDInCrushMap(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int) (*CrushFindResult, error) {
	args := []string{"osd", "find", strconv.Itoa(osdID)}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		return nil, errors.Wrapf(err, "failed to find osd.%d in crush map: %s", osdID, string(buf))
	}

	var result CrushFindResult
	if err := json.Unmarshal(buf, &result); err != nil {
		return nil, errors.Wrapf(err, "failed to unmarshal crush find result: %s", string(buf))
	}

	return &result, nil
}

// GetCrushHostName gets the hostname where an OSD is running on
func GetCrushHostName(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int) (string, error) {
	result, err := FindOSDInCrushMap(context, clusterInfo, osdID)
	if err != nil {
		return "", err
	}

	return result.Location["host"], nil
}

// NormalizeCrushName replaces . with -
func NormalizeCrushName(name string) string {
	return strings.Replace(name, ".", "-", -1)
}

// Obtain the cluster-wide default crush root from the cluster spec
func GetCrushRootFromSpec(c *cephv1.ClusterSpec) string {
	if c.Storage.Config == nil {
		return cephv1.DefaultCRUSHRoot
	}
	if v, ok := c.Storage.Config[CrushRootConfigKey]; ok {
		return v
	}
	return cephv1.DefaultCRUSHRoot
}

// IsNormalizedCrushNameEqual returns true if normalized is either equal to or the normalized version of notNormalized
// a crush name is normalized if it comes from the crushmap or has passed through the NormalizeCrushName function.
func IsNormalizedCrushNameEqual(notNormalized, normalized string) bool {
	if notNormalized == normalized || NormalizeCrushName(notNormalized) == normalized {
		return true
	}
	return false
}

// UpdateCrushMapValue is for updating the location in the crush map
// this is not safe for incorrectly formatted strings
func UpdateCrushMapValue(pairs *[]string, key, value string) {
	found := false
	property := formatProperty(key, value)
	for i, pair := range *pairs {
		entry := strings.Split(pair, "=")
		if key == entry[0] {
			(*pairs)[i] = property
			found = true
		}
	}
	if !found {
		*pairs = append(*pairs, property)
	}
}

func formatProperty(name, value string) string {
	return fmt.Sprintf("%s=%s", name, value)
}

// GetOSDOnHost returns the list of osds running on a given host
func GetOSDOnHost(context *clusterd.Context, clusterInfo *ClusterInfo, node string) (string, error) {
	args := []string{"osd", "crush", "ls", node}
	buf, err := NewCephCommand(context, clusterInfo, args).Run()
	if err != nil {
		return "", errors.Wrapf(err, "failed to get osd list on host. %s", string(buf))
	}

	return string(buf), nil
}

func compileCRUSHMap(context *clusterd.Context, crushMapPath string) error {
	mapFile := buildCompileCRUSHFileName(crushMapPath)
	args := []string{"--compile", crushMapPath, "--outfn", mapFile}
	output, err := context.Executor.ExecuteCommandWithOutput("crushtool", args...)
	if err != nil {
		return errors.Wrapf(err, "failed to compile crush map %q. %s", mapFile, output)
	}

	return nil
}

func decompileCRUSHMap(context *clusterd.Context, crushMapPath string) error {
	mapFile := buildDecompileCRUSHFileName(crushMapPath)
	args := []string{"--decompile", crushMapPath, "--outfn", mapFile}
	output, err := context.Executor.ExecuteCommandWithOutput("crushtool", args...)
	if err != nil {
		return errors.Wrapf(err, "failed to decompile crush map %q. %s", mapFile, output)
	}

	return nil
}

func injectCRUSHMap(context *clusterd.Context, clusterInfo *ClusterInfo, crushMapPath string) error {
	args := []string{"osd", "setcrushmap", "--in-file", crushMapPath}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return errors.Wrapf(err, "failed to inject crush map %q. %s", crushMapPath, string(buf))
	}

	return nil
}

func setCRUSHMap(context *clusterd.Context, clusterInfo *ClusterInfo, crushMapPath string) error {
	args := []string{"osd", "crush", "set", crushMapPath}
	exec := NewCephCommand(context, clusterInfo, args)
	exec.OutputFile = false
	exec.JsonOutput = false
	buf, err := exec.Run()
	if err != nil {
		return errors.Wrapf(err, "failed to set crush map %q. %s", crushMapPath, string(buf))
	}

	return nil
}

func buildDecompileCRUSHFileName(crushMapPath string) string {
	return fmt.Sprintf("%s.decompiled", crushMapPath)
}

func buildCompileCRUSHFileName(crushMapPath string) string {
	return fmt.Sprintf("%s.compiled", crushMapPath)
}
<|endoftext|>


<commit_before>// Usage
//    migrate.To(2)
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//    	.Exec(db)
//
//    migrate.ToLatest()
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//    	.SetDialect(migrate.MySQL)
//    	.Exec(db)
//
//    migrate.ToLatest()
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//		.Backup(path)
//		.Exec()

package migrate

import (
	"database/sql"
	"log"
)

const migrationTableStmt = `
CREATE TABLE IF NOT EXISTS migration (
	revision NUMBER PRIMARY KEY
)
`

const migrationSelectStmt = `
SELECT revision FROM migration
WHERE revision = ?
`

const migrationSelectMaxStmt = `
SELECT max(revision) FROM migration
`

const insertRevisionStmt = `
INSERT INTO migration (revision) VALUES (?)
`

const deleteRevisionStmt = `
DELETE FROM migration where revision = ?
`

type Revision interface {
	Up(tx *sql.Tx) error
	Down(tx *sql.Tx) error
	Revision() int64
}

type Migration struct {
	db   *sql.DB
	revs []Revision
}

func New(db *sql.DB) *Migration {
	return &Migration{db: db}
}

// Add the Revision to the list of migrations.
func (m *Migration) Add(rev ...Revision) *Migration {
	m.revs = append(m.revs, rev...)
	return m
}

// Execute the full list of migrations.
func (m *Migration) Migrate() error {
	var target int64
	if len(m.revs) > 0 {
		// get the last revision number in
		// the list. This is what we'll
		// migrate toward.
		target = m.revs[len(m.revs)-1].Revision()
	}
	return m.MigrateTo(target)
}

// Execute all database migration until
// you are at the specified revision number.
// If the revision number is less than the
// current revision, then we will downgrade.
func (m *Migration) MigrateTo(target int64) error {

	// make sure the migration table is created.
	if _, err := m.db.Exec(migrationTableStmt); err != nil {
		return err
	}

	// get the current revision
	var current int64
	m.db.QueryRow(migrationSelectMaxStmt).Scan(&current)

	// already up to date
	if current == target {
		log.Println("Database already up-to-date.")
		return nil
	}

	// should we downgrade?
	if target < current {
		return m.down(target, current)
	}

	// else upgrade
	return m.up(target, current)
}

func (m *Migration) up(target, current int64) error {
	// create the database transaction
	tx, err := m.db.Begin()
	if err != nil {
		return err
	}

	// loop through and execute revisions
	for _, rev := range m.revs {
		if rev.Revision() >= target {
			current = rev.Revision()
			// execute the revision Upgrade.
			if err := rev.Up(tx); err != nil {
				log.Printf("Failed to upgrade to Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}
			// update the revision number in the database
			if _, err := tx.Exec(insertRevisionStmt, current); err != nil {
				log.Printf("Failed to register Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}

			log.Printf("Successfully upgraded to Revision %v\n", current)
		}
	}

	return tx.Commit()
}

func (m *Migration) down(target, current int64) error {
	// create the database transaction
	tx, err := m.db.Begin()
	if err != nil {
		return err
	}

	// reverse the list of revisions
	revs := []Revision{}
	for _, rev := range m.revs {
		revs = append([]Revision{rev}, revs...)
	}

	// loop through the (reversed) list of
	// revisions and execute.
	for _, rev := range revs {
		if rev.Revision() > target {
			current = rev.Revision()
			// execute the revision Upgrade.
			if err := rev.Down(tx); err != nil {
				log.Printf("Failed to downgrade to Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}
			// update the revision number in the database
			if _, err := tx.Exec(deleteRevisionStmt, current); err != nil {
				log.Printf("Failed to unregistser Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}

			log.Printf("Successfully downgraded to Revision %v\n", current)
		}
	}

	return tx.Commit()
}
<commit_msg>Integrate MigrationDriver to migrate.go<commit_after>// Usage
//    migrate.To(2)
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//    	.Exec(db)
//
//    migrate.ToLatest()
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//    	.SetDialect(migrate.MySQL)
//    	.Exec(db)
//
//    migrate.ToLatest()
//    	.Add(Version_1)
//    	.Add(Version_2)
//    	.Add(Version_3)
//		.Backup(path)
//		.Exec()

package migrate

import (
	"database/sql"
	"log"
)

const migrationTableStmt = `
CREATE TABLE IF NOT EXISTS migration (
	revision NUMBER PRIMARY KEY
)
`

const migrationSelectStmt = `
SELECT revision FROM migration
WHERE revision = ?
`

const migrationSelectMaxStmt = `
SELECT max(revision) FROM migration
`

const insertRevisionStmt = `
INSERT INTO migration (revision) VALUES (?)
`

const deleteRevisionStmt = `
DELETE FROM migration where revision = ?
`

// Operation interface covers basic migration operations.
// Implementation details is specific for each database,
// see migrate/sqlite.go for implementation reference.
type Operation interface {

	CreateTable(tableName string, args []string) (sql.Result, error)

	RenameTable(tableName, newName string) (sql.Result, error)

	DropTable(tableName string) (sql.Result, error)

	AddColumn(tableName, columnSpec string) (sql.Result, error)

	DropColumns(tableName string, columnsToDrop []string) (sql.Result, error)

	RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error)
}

type Revision interface {
	Up(op Operation) error
	Down(op Operation) error
	Revision() int64
}

type MigrationDriver struct {
	Tx *sql.Tx
}

type Migration struct {
	db   *sql.DB
	revs []Revision
}

var Driver func(tx *sql.Tx) Operation

func New(db *sql.DB) *Migration {
	return &Migration{db: db}
}

// Add the Revision to the list of migrations.
func (m *Migration) Add(rev ...Revision) *Migration {
	m.revs = append(m.revs, rev...)
	return m
}

// Execute the full list of migrations.
func (m *Migration) Migrate() error {
	var target int64
	if len(m.revs) > 0 {
		// get the last revision number in
		// the list. This is what we'll
		// migrate toward.
		target = m.revs[len(m.revs)-1].Revision()
	}
	return m.MigrateTo(target)
}

// Execute all database migration until
// you are at the specified revision number.
// If the revision number is less than the
// current revision, then we will downgrade.
func (m *Migration) MigrateTo(target int64) error {

	// make sure the migration table is created.
	if _, err := m.db.Exec(migrationTableStmt); err != nil {
		return err
	}

	// get the current revision
	var current int64
	m.db.QueryRow(migrationSelectMaxStmt).Scan(&current)

	// already up to date
	if current == target {
		log.Println("Database already up-to-date.")
		return nil
	}

	// should we downgrade?
	if target < current {
		return m.down(target, current)
	}

	// else upgrade
	return m.up(target, current)
}

func (m *Migration) up(target, current int64) error {
	// create the database transaction
	tx, err := m.db.Begin()
	if err != nil {
		return err
	}

	op := Driver(tx)

	// loop through and execute revisions
	for _, rev := range m.revs {
		if rev.Revision() >= target {
			current = rev.Revision()
			// execute the revision Upgrade.
			if err := rev.Up(op); err != nil {
				log.Printf("Failed to upgrade to Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}
			// update the revision number in the database
			if _, err := tx.Exec(insertRevisionStmt, current); err != nil {
				log.Printf("Failed to register Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}

			log.Printf("Successfully upgraded to Revision %v\n", current)
		}
	}

	return tx.Commit()
}

func (m *Migration) down(target, current int64) error {
	// create the database transaction
	tx, err := m.db.Begin()
	if err != nil {
		return err
	}

	op := Driver(tx)

	// reverse the list of revisions
	revs := []Revision{}
	for _, rev := range m.revs {
		revs = append([]Revision{rev}, revs...)
	}

	// loop through the (reversed) list of
	// revisions and execute.
	for _, rev := range revs {
		if rev.Revision() > target {
			current = rev.Revision()
			// execute the revision Upgrade.
			if err := rev.Down(op); err != nil {
				log.Printf("Failed to downgrade to Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}
			// update the revision number in the database
			if _, err := tx.Exec(deleteRevisionStmt, current); err != nil {
				log.Printf("Failed to unregistser Revision Number %v\n", current)
				log.Println(err)
				return tx.Rollback()
			}

			log.Printf("Successfully downgraded to Revision %v\n", current)
		}
	}

	return tx.Commit()
}
<|endoftext|>


<commit_before>package store_test

import (
	"testing"

	"github.com/Aptomi/aptomi/pkg/engine"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/stretchr/testify/assert"
)

func TestIndexes(t *testing.T) {
	indexes := store.Indexes(engine.TypeRevision)
	assert.NotNil(t, indexes)
	assert.Len(t, indexes, 3)
	assert.Contains(t, indexes, "PolicyGen")
	revision := &engine.Revision{
		TypeKind: engine.TypeRevision.GetTypeKind(),
		Status:   "some_status",
		Metadata: runtime.GenerationMetadata{
			Generation: 1,
		},
		PolicyGen: 42,
	}
	assert.Equal(t, "system/revision@PolicyGen@42", indexes["PolicyGen"].KeyForStorable(revision, NewJsonCodec()))
	assert.Equal(t, "system/revision@Status@some_status", indexes["Status"].KeyForStorable(revision, NewJsonCodec()))
	assert.Equal(t, "system/revision", indexes[""].KeyForStorable(revision, NewJsonCodec()))

	assert.Equal(t, "system/revision@PolicyGen@42", indexes["PolicyGen"].KeyForValue(engine.RevisionKey, 42, NewJsonCodec()))
}
<commit_msg>index_test fixed with lastest changes<commit_after>package store_test

import (
	"testing"

	"github.com/Aptomi/aptomi/pkg/engine"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/runtime/store"
	"github.com/stretchr/testify/assert"
)

func TestIndexes(t *testing.T) {
	indexes := store.IndexesFor(engine.TypeRevision)
	assert.NotNil(t, indexes)
	assert.Len(t, indexes, 3)
	assert.Contains(t, indexes, "PolicyGen")
	revision := &engine.Revision{
		TypeKind: engine.TypeRevision.GetTypeKind(),
		Status:   "some_status",
		Metadata: runtime.GenerationMetadata{
			Generation: 1,
		},
		PolicyGen: 42,
	}
	assert.Equal(t, "system/revision@PolicyGen@42", indexes.KeyForStorable("PolicyGen", revision, store.NewJsonCodec()))
	assert.Equal(t, "system/revision@Status@some_status", indexes.KeyForStorable("Status", revision, store.NewJsonCodec()))
	assert.Equal(t, "system/revision", indexes.KeyForStorable(store.LastGenIndex, revision, store.NewJsonCodec()))

	assert.Equal(t, "system/revision@PolicyGen@42", indexes.KeyForValue("PolicyGen", engine.RevisionKey, 42, store.NewJsonCodec()))
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/sha256"
	"encoding/hex"
	"net/http"
	"net/url"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

func init() {
	registry.RegisterService(&UserAuthTokenService{})
}

var now = time.Now

// UserAuthTokenService are used for generating and validating user auth tokens
type UserAuthTokenService struct {
	SQLStore *sqlstore.SqlStore `inject:""`
	log      log.Logger
}

// Init this service
func (s *UserAuthTokenService) Init() error {
	s.log = log.New("auth")
	return nil
}

const sessionCookieKey = "grafana_session"

func (s *UserAuthTokenService) UserAuthenticatedHook(user *models.User, c *models.ReqContext) error {
	userToken, err := s.CreateToken(user.Id, c.RemoteAddr(), c.Req.UserAgent())
	if err != nil {
		return err
	}

	c.Resp.Header().Del("Set-Cookie")
	cookie := http.Cookie{
		Name:     sessionCookieKey,
		Value:    url.QueryEscape(userToken.UnhashedToken),
		HttpOnly: true,
		//MaxAge:   600,
		Domain: setting.Domain,
		Path:   setting.AppSubUrl + "/",
	}

	c.Resp.Header().Add("Set-Cookie", cookie.String())

	return nil
}

func (s *UserAuthTokenService) UserSignedOutHook(c *models.ReqContext) {
	//c.SetCookie(sessionCookieKey, "", -1, setting.AppSubUrl+"/", setting.Domain, false, true)
	c.Resp.Header().Del("Set-Cookie")
	cookie := http.Cookie{
		Name:     sessionCookieKey,
		Value:    "",
		HttpOnly: true,
		MaxAge:   -1,
		Domain:   setting.Domain,
		Path:     setting.AppSubUrl + "/",
	}

	c.Resp.Header().Add("Set-Cookie", cookie.String())
}

// func (s *UserAuthTokenService) RequestMiddleware() macaron.Handler {
// 	return func(ctx *models.ReqContext) {
// 		authToken := ctx.GetCookie(sessionCookieKey)
// 		userToken, err := s.LookupToken(authToken)
// 		if err != nil {

// 		}

// 		ctx.Next()

// 		refreshed, err := s.RefreshToken(userToken, ctx.RemoteAddr(), ctx.Req.UserAgent())
// 		if err != nil {

// 		}

// 		if refreshed {
// 			ctx.Resp.Header().Del("Set-Cookie")
// 			ctx.SetCookie(sessionCookieKey, userToken.unhashedToken, setting.AppSubUrl+"/", setting.Domain, false, true)
// 		}
// 	}
// }

func (s *UserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*models.UserAuthToken, error) {
	clientIP = util.ParseIPAddress(clientIP)
	token, err := util.RandomHex(16)
	if err != nil {
		return nil, err
	}

	hashedToken := hashToken(token)

	userToken := models.UserAuthToken{
		UserId:        userId,
		AuthToken:     hashedToken,
		PrevAuthToken: hashedToken,
		ClientIp:      clientIP,
		UserAgent:     userAgent,
		RotatedAt:     now().Unix(),
		CreatedAt:     now().Unix(),
		UpdatedAt:     now().Unix(),
		SeenAt:        0,
		AuthTokenSeen: false,
	}
	_, err = s.SQLStore.NewSession().Insert(&userToken)
	if err != nil {
		return nil, err
	}

	userToken.UnhashedToken = token

	return &userToken, nil
}

func (s *UserAuthTokenService) LookupToken(unhashedToken string) (*models.UserAuthToken, error) {
	hashedToken := hashToken(unhashedToken)

	var userToken models.UserAuthToken
	exists, err := s.SQLStore.NewSession().Where("auth_token = ? OR prev_auth_token = ?", hashedToken, hashedToken).Get(&userToken)
	if err != nil {
		return nil, err
	}

	if !exists {
		return nil, ErrAuthTokenNotFound
	}

	if userToken.AuthToken != hashedToken && userToken.PrevAuthToken == hashedToken && userToken.AuthTokenSeen {
		userToken.AuthTokenSeen = false
		expireBefore := now().Add(-1 * time.Minute).Unix()
		affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND prev_auth_token = ? AND rotated_at < ?", userToken.Id, userToken.PrevAuthToken, expireBefore).AllCols().Update(&userToken)
		if err != nil {
			return nil, err
		}

		if affectedRows == 0 {
			s.log.Debug("prev seen token unchanged", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		} else {
			s.log.Debug("prev seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		}
	}

	if !userToken.AuthTokenSeen && userToken.AuthToken == hashedToken {
		userTokenCopy := userToken
		userTokenCopy.AuthTokenSeen = true
		userTokenCopy.SeenAt = now().Unix()
		affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND auth_token = ?", userTokenCopy.Id, userTokenCopy.AuthToken).AllCols().Update(&userTokenCopy)
		if err != nil {
			return nil, err
		}

		if affectedRows == 1 {
			userToken = userTokenCopy
		}

		if affectedRows == 0 {
			s.log.Debug("seen wrong token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		} else {
			s.log.Debug("seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		}
	}

	userToken.UnhashedToken = unhashedToken

	return &userToken, nil
}

func (s *UserAuthTokenService) RefreshToken(token *models.UserAuthToken, clientIP, userAgent string) (bool, error) {
	if token == nil {
		return false, nil
	}

	var needsRotation = false
	rotatedAt := time.Unix(token.RotatedAt, 0)
	if token.AuthTokenSeen {
		needsRotation = rotatedAt.Before(now().Add(time.Duration(-1) * time.Minute))
	} else {
		needsRotation = rotatedAt.Before(now().Add(time.Duration(-30) * time.Second))
	}

	s.log.Debug("refresh token", "needs rotation?", needsRotation, "auth_token_seen", token.AuthTokenSeen, "rotated_at", rotatedAt, "token.Id", token.Id)
	if !needsRotation {
		return false, nil
	}

	newToken, _ := util.RandomHex(16)
	hashedToken := hashToken(newToken)

	sql := `
		UPDATE user_auth_token
		SET
			auth_token_seen = false,
			seen_at = null,
			user_agent = ?,
			client_ip = ?,
			prev_auth_token = case when auth_token_seen then auth_token else prev_auth_token end,
			auth_token = ?,
			rotated_at = ?
		WHERE id = ? AND (auth_token_seen or rotated_at < ?)`

	res, err := s.SQLStore.NewSession().Exec(sql, userAgent, clientIP, hashedToken, now().Unix(), token.Id, now().Add(time.Duration(-30)*time.Second))
	if err != nil {
		return false, err
	}

	affected, _ := res.RowsAffected()
	s.log.Debug("rotated", "affected", affected, "auth_token_id", token.Id, "userId", token.UserId, "user_agent", userAgent, "client_ip", clientIP)
	if affected > 0 {
		token.UnhashedToken = newToken
		return true, nil
	}

	return false, nil
}

func hashToken(token string) string {
	hashBytes := sha256.Sum256([]byte(token + setting.SecretKey))
	return hex.EncodeToString(hashBytes[:])
}
<commit_msg>removes commented code<commit_after>package auth

import (
	"crypto/sha256"
	"encoding/hex"
	"net/http"
	"net/url"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
)

func init() {
	registry.RegisterService(&UserAuthTokenService{})
}

var now = time.Now

// UserAuthTokenService are used for generating and validating user auth tokens
type UserAuthTokenService struct {
	SQLStore *sqlstore.SqlStore `inject:""`
	log      log.Logger
}

// Init this service
func (s *UserAuthTokenService) Init() error {
	s.log = log.New("auth")
	return nil
}

const sessionCookieKey = "grafana_session"

func (s *UserAuthTokenService) UserAuthenticatedHook(user *models.User, c *models.ReqContext) error {
	userToken, err := s.CreateToken(user.Id, c.RemoteAddr(), c.Req.UserAgent())
	if err != nil {
		return err
	}

	c.Resp.Header().Del("Set-Cookie")
	cookie := http.Cookie{
		Name:     sessionCookieKey,
		Value:    url.QueryEscape(userToken.UnhashedToken),
		HttpOnly: true,
		Domain:   setting.Domain,
		Path:     setting.AppSubUrl + "/",
	}

	c.Resp.Header().Add("Set-Cookie", cookie.String())

	return nil
}

func (s *UserAuthTokenService) UserSignedOutHook(c *models.ReqContext) {
	c.Resp.Header().Del("Set-Cookie")
	cookie := http.Cookie{
		Name:     sessionCookieKey,
		Value:    "",
		HttpOnly: true,
		MaxAge:   -1,
		Domain:   setting.Domain,
		Path:     setting.AppSubUrl + "/",
	}

	c.Resp.Header().Add("Set-Cookie", cookie.String())
}

func (s *UserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*models.UserAuthToken, error) {
	clientIP = util.ParseIPAddress(clientIP)
	token, err := util.RandomHex(16)
	if err != nil {
		return nil, err
	}

	hashedToken := hashToken(token)

	userToken := models.UserAuthToken{
		UserId:        userId,
		AuthToken:     hashedToken,
		PrevAuthToken: hashedToken,
		ClientIp:      clientIP,
		UserAgent:     userAgent,
		RotatedAt:     now().Unix(),
		CreatedAt:     now().Unix(),
		UpdatedAt:     now().Unix(),
		SeenAt:        0,
		AuthTokenSeen: false,
	}
	_, err = s.SQLStore.NewSession().Insert(&userToken)
	if err != nil {
		return nil, err
	}

	userToken.UnhashedToken = token

	return &userToken, nil
}

func (s *UserAuthTokenService) LookupToken(unhashedToken string) (*models.UserAuthToken, error) {
	hashedToken := hashToken(unhashedToken)

	var userToken models.UserAuthToken
	exists, err := s.SQLStore.NewSession().Where("auth_token = ? OR prev_auth_token = ?", hashedToken, hashedToken).Get(&userToken)
	if err != nil {
		return nil, err
	}

	if !exists {
		return nil, ErrAuthTokenNotFound
	}

	if userToken.AuthToken != hashedToken && userToken.PrevAuthToken == hashedToken && userToken.AuthTokenSeen {
		userToken.AuthTokenSeen = false
		expireBefore := now().Add(-1 * time.Minute).Unix()
		affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND prev_auth_token = ? AND rotated_at < ?", userToken.Id, userToken.PrevAuthToken, expireBefore).AllCols().Update(&userToken)
		if err != nil {
			return nil, err
		}

		if affectedRows == 0 {
			s.log.Debug("prev seen token unchanged", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		} else {
			s.log.Debug("prev seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		}
	}

	if !userToken.AuthTokenSeen && userToken.AuthToken == hashedToken {
		userTokenCopy := userToken
		userTokenCopy.AuthTokenSeen = true
		userTokenCopy.SeenAt = now().Unix()
		affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND auth_token = ?", userTokenCopy.Id, userTokenCopy.AuthToken).AllCols().Update(&userTokenCopy)
		if err != nil {
			return nil, err
		}

		if affectedRows == 1 {
			userToken = userTokenCopy
		}

		if affectedRows == 0 {
			s.log.Debug("seen wrong token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		} else {
			s.log.Debug("seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent)
		}
	}

	userToken.UnhashedToken = unhashedToken

	return &userToken, nil
}

func (s *UserAuthTokenService) RefreshToken(token *models.UserAuthToken, clientIP, userAgent string) (bool, error) {
	if token == nil {
		return false, nil
	}

	var needsRotation = false
	rotatedAt := time.Unix(token.RotatedAt, 0)
	if token.AuthTokenSeen {
		needsRotation = rotatedAt.Before(now().Add(time.Duration(-1) * time.Minute))
	} else {
		needsRotation = rotatedAt.Before(now().Add(time.Duration(-30) * time.Second))
	}

	s.log.Debug("refresh token", "needs rotation?", needsRotation, "auth_token_seen", token.AuthTokenSeen, "rotated_at", rotatedAt, "token.Id", token.Id)
	if !needsRotation {
		return false, nil
	}

	newToken, _ := util.RandomHex(16)
	hashedToken := hashToken(newToken)

	sql := `
		UPDATE user_auth_token
		SET
			auth_token_seen = false,
			seen_at = null,
			user_agent = ?,
			client_ip = ?,
			prev_auth_token = case when auth_token_seen then auth_token else prev_auth_token end,
			auth_token = ?,
			rotated_at = ?
		WHERE id = ? AND (auth_token_seen or rotated_at < ?)`

	res, err := s.SQLStore.NewSession().Exec(sql, userAgent, clientIP, hashedToken, now().Unix(), token.Id, now().Add(time.Duration(-30)*time.Second))
	if err != nil {
		return false, err
	}

	affected, _ := res.RowsAffected()
	s.log.Debug("rotated", "affected", affected, "auth_token_id", token.Id, "userId", token.UserId, "user_agent", userAgent, "client_ip", clientIP)
	if affected > 0 {
		token.UnhashedToken = newToken
		return true, nil
	}

	return false, nil
}

func hashToken(token string) string {
	hashBytes := sha256.Sum256([]byte(token + setting.SecretKey))
	return hex.EncodeToString(hashBytes[:])
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manila

import (
	"fmt"

	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/api/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/sharebackends"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/shareoptions"
)

// Provisioner struct, implements controller.Provisioner interface
type Provisioner struct {
	clientset clientset.Interface
}

// NewProvisioner creates a new instance of Manila provisioner
func NewProvisioner(c clientset.Interface) *Provisioner {
	return &Provisioner{
		clientset: c,
	}
}

// Provision a share in Manila service
func (p *Provisioner) Provision(volOptions controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if volOptions.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}

	// Initialization

	shareOptions, err := shareoptions.NewShareOptions(&volOptions, p.clientset)
	if err != nil {
		return nil, fmt.Errorf("failed to create share options: %v", err)
	}

	shareBackend, err := getShareBackend(shareOptions.Backend)
	if err != nil {
		return nil, fmt.Errorf("failed to get share backend: %v", err)
	}

	client, err := NewManilaV2Client(&shareOptions.OpenStackOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to create Manila v2 client: %v", err)
	}

	// Share provision

	var share *shares.Share

	if shareOptions.OSShareAccessID == "" {
		// Dynamic provision - we're creating a new share

		share, err = createShare(&volOptions, shareOptions, client)
		if err != nil {
			return nil, fmt.Errorf("failed to create a share: %v", err)
		}

		defer func() {
			// Delete the share if any of its setup operations fail
			if err != nil {
				if delErr := deleteShare(share.ID, manilaProvisionTypeDynamic, &shareOptions.ShareSecretRef, client, p.clientset); delErr != nil {
					glog.Errorf("failed to delete share %s in a rollback procedure: %v", share.ID, delErr)
				}
			}
		}()

		if err = waitForShareStatus(share.ID, client, "available"); err != nil {
			return nil, fmt.Errorf("waiting for share %s to become created failed: %v", share.ID, err)
		}
	} else {
		// Static provision - we're using an existing share

		share, err = getShare(shareOptions, client)
		if err != nil {
			return nil, fmt.Errorf("failed to get share %s: %v", shareOptions.OSShareID, err)
		}
	}

	// Get the export location

	availableExportLocations, err := shares.GetExportLocations(client, share.ID).Extract()
	if err != nil {
		return nil, fmt.Errorf("failed to get export locations for share %s: %v", share.ID, err)
	}

	chosenExportLocation, err := chooseExportLocation(availableExportLocations)
	if err != nil {
		return nil, fmt.Errorf("failed to choose an export location for share %s: %v", share.ID, err)
	}

	accessRight, err := shareBackend.GrantAccess(&sharebackends.GrantAccessArgs{
		Share:     share,
		Options:   shareOptions,
		Clientset: p.clientset,
		Client:    client,
	})
	if err != nil {
		return nil, fmt.Errorf("failed to grant access for share %s: %v", share.ID, err)
	}

	// For deleteShare()
	registerBackendForShare(shareOptions.Backend, share.ID)

	volSource, err := shareBackend.BuildSource(&sharebackends.BuildSourceArgs{
		Share:       share,
		Options:     shareOptions,
		Location:    &chosenExportLocation,
		Clientset:   p.clientset,
		AccessRight: accessRight,
	})
	if err != nil {
		return nil, fmt.Errorf("backend %s failed to create volume source for share %s: %v", shareBackend.Name(), share.ID, err)
	}

	glog.Infof("share %s (%s/%s) provisioned successfully", share.ID, shareOptions.Protocol, shareOptions.Backend)

	return buildPersistentVolume(share, accessRight, volSource, &volOptions, shareOptions), nil
}

// Delete a share from Manila service
func (p *Provisioner) Delete(pv *v1.PersistentVolume) error {
	// Initialization

	shareID, err := getShareIDfromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get share ID for volume %s: %v", pv.GetName(), err)
	}

	osSecretRef, err := getOSSecretRefFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get OpenStack secret reference from PV for share %s: %v", shareID, err)
	}

	shareSecretRef, err := getShareSecretRefFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get share secret reference from PV for share %s: %v", shareID, err)
	}

	provisionType, err := getProvisionTypeFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get provision type for volume %s: %v", pv.GetName(), err)
	}

	osOptions, err := shareoptions.NewOpenStackOptions(p.clientset, osSecretRef)
	if err != nil {
		return fmt.Errorf("failed to create OpenStack options for share %s: %v", shareID, err)
	}

	client, err := NewManilaV2Client(osOptions)
	if err != nil {
		return fmt.Errorf("failed to create Manila v2 client for share %s: %v", shareID, err)
	}

	// Share deletion

	if err = deleteShare(shareID, provisionType, shareSecretRef, client, p.clientset); err != nil {
		return fmt.Errorf("failed to delete share %s: %v", shareID, err)
	}

	glog.Infof("share %s deleted successfully", shareID)

	return nil
}
<commit_msg>manila: refactoring for shareoptions.NewOpenStackOptionsFromSecret<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manila

import (
	"fmt"

	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/api/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/sharebackends"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/shareoptions"
)

// Provisioner struct, implements controller.Provisioner interface
type Provisioner struct {
	clientset clientset.Interface
}

// NewProvisioner creates a new instance of Manila provisioner
func NewProvisioner(c clientset.Interface) *Provisioner {
	return &Provisioner{
		clientset: c,
	}
}

// Provision a share in Manila service
func (p *Provisioner) Provision(volOptions controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if volOptions.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}

	// Initialization

	shareOptions, err := shareoptions.NewShareOptions(&volOptions, p.clientset)
	if err != nil {
		return nil, fmt.Errorf("failed to create share options: %v", err)
	}

	shareBackend, err := getShareBackend(shareOptions.Backend)
	if err != nil {
		return nil, fmt.Errorf("failed to get share backend: %v", err)
	}

	client, err := NewManilaV2Client(&shareOptions.OpenStackOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to create Manila v2 client: %v", err)
	}

	// Share provision

	var share *shares.Share

	if shareOptions.OSShareAccessID == "" {
		// Dynamic provision - we're creating a new share

		share, err = createShare(&volOptions, shareOptions, client)
		if err != nil {
			return nil, fmt.Errorf("failed to create a share: %v", err)
		}

		defer func() {
			// Delete the share if any of its setup operations fail
			if err != nil {
				if delErr := deleteShare(share.ID, manilaProvisionTypeDynamic, &shareOptions.ShareSecretRef, client, p.clientset); delErr != nil {
					glog.Errorf("failed to delete share %s in a rollback procedure: %v", share.ID, delErr)
				}
			}
		}()

		if err = waitForShareStatus(share.ID, client, "available"); err != nil {
			return nil, fmt.Errorf("waiting for share %s to become created failed: %v", share.ID, err)
		}
	} else {
		// Static provision - we're using an existing share

		share, err = getShare(shareOptions, client)
		if err != nil {
			return nil, fmt.Errorf("failed to get share %s: %v", shareOptions.OSShareID, err)
		}
	}

	// Get the export location

	availableExportLocations, err := shares.GetExportLocations(client, share.ID).Extract()
	if err != nil {
		return nil, fmt.Errorf("failed to get export locations for share %s: %v", share.ID, err)
	}

	chosenExportLocation, err := chooseExportLocation(availableExportLocations)
	if err != nil {
		return nil, fmt.Errorf("failed to choose an export location for share %s: %v", share.ID, err)
	}

	accessRight, err := shareBackend.GrantAccess(&sharebackends.GrantAccessArgs{
		Share:     share,
		Options:   shareOptions,
		Clientset: p.clientset,
		Client:    client,
	})
	if err != nil {
		return nil, fmt.Errorf("failed to grant access for share %s: %v", share.ID, err)
	}

	// For deleteShare()
	registerBackendForShare(shareOptions.Backend, share.ID)

	volSource, err := shareBackend.BuildSource(&sharebackends.BuildSourceArgs{
		Share:       share,
		Options:     shareOptions,
		Location:    &chosenExportLocation,
		Clientset:   p.clientset,
		AccessRight: accessRight,
	})
	if err != nil {
		return nil, fmt.Errorf("backend %s failed to create volume source for share %s: %v", shareBackend.Name(), share.ID, err)
	}

	glog.Infof("share %s (%s/%s) provisioned successfully", share.ID, shareOptions.Protocol, shareOptions.Backend)

	return buildPersistentVolume(share, accessRight, volSource, &volOptions, shareOptions), nil
}

// Delete a share from Manila service
func (p *Provisioner) Delete(pv *v1.PersistentVolume) error {
	// Initialization

	shareID, err := getShareIDfromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get share ID for volume %s: %v", pv.GetName(), err)
	}

	osSecretRef, err := getOSSecretRefFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get OpenStack secret reference from PV for share %s: %v", shareID, err)
	}

	shareSecretRef, err := getShareSecretRefFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get share secret reference from PV for share %s: %v", shareID, err)
	}

	provisionType, err := getProvisionTypeFromPV(pv)
	if err != nil {
		return fmt.Errorf("failed to get provision type for volume %s: %v", pv.GetName(), err)
	}

	osOptions, err := shareoptions.NewOpenStackOptionsFromSecret(p.clientset, osSecretRef)
	if err != nil {
		return fmt.Errorf("failed to create OpenStack options for share %s: %v", shareID, err)
	}

	client, err := NewManilaV2Client(osOptions)
	if err != nil {
		return fmt.Errorf("failed to create Manila v2 client for share %s: %v", shareID, err)
	}

	// Share deletion

	if err = deleteShare(shareID, provisionType, shareSecretRef, client, p.clientset); err != nil {
		return fmt.Errorf("failed to delete share %s: %v", shareID, err)
	}

	glog.Infof("share %s deleted successfully", shareID)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"fmt"
	"os"
	"reflect"
	re "regexp"
	"strings"

	cfg "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/yamltags"
	yamlpatch "github.com/krishicks/yaml-patch"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	yaml "gopkg.in/yaml.v2"
)

var (
	enableKubeContext = kubectx.ChangeKubeContext
)

// ApplyProfiles returns configuration modified by the application
// of a list of profiles.
func ApplyProfiles(c *latest.SkaffoldConfig, opts cfg.SkaffoldOptions) error {
	byName := profilesByName(c.Profiles)

	profiles, isContextSpecific, err := activatedProfiles(c.Profiles, opts)
	if err != nil {
		return errors.Wrap(err, "finding auto-activated profiles")
	}

	for _, name := range profiles {
		profile, present := byName[name]
		if !present {
			return fmt.Errorf("couldn't find profile %s", name)
		}

		if err := applyProfile(c, profile); err != nil {
			return errors.Wrapf(err, "applying profile %s", name)
		}
	}

	return enableEffectiveKubecontext(isContextSpecific, opts.KubeContext, c.Deploy.KubeContext)
}

func enableEffectiveKubecontext(isContextImmutable bool, cliContext, effectiveContext string) error {
	// cli flag takes precedence
	if cliContext != "" {
		return nil
	}

	kubeConfig, err := kubectx.CurrentConfig()
	if err != nil {
		return errors.Wrap(err, "getting current cluster context")
	}

	// nothing to do
	if effectiveContext == "" || effectiveContext == kubeConfig.CurrentContext {
		return nil
	}

	if isContextImmutable {
		return fmt.Errorf("some activated profile contains kubecontext specific settings for a different than the effective kubecontext, please revise your profile activations")
	}

	enableKubeContext(effectiveContext)
	return nil
}

func activatedProfiles(profiles []latest.Profile, opts cfg.SkaffoldOptions) ([]string, bool, error) {
	activated := opts.Profiles
	hasContextSpecificProfile := false

	// Auto-activated profiles
	for _, profile := range profiles {
		for _, cond := range profile.Activation {
			command := isCommand(cond.Command, opts)

			env, err := isEnv(cond.Env)
			if err != nil {
				return nil, false, err
			}

			kubeContext, err := isKubeContext(cond.KubeContext)
			if err != nil {
				return nil, false, err
			}

			if command && env && kubeContext {
				if cond.KubeContext != "" {
					hasContextSpecificProfile = true
				}
				activated = append(activated, profile.Name)
			}
		}
	}

	return activated, hasContextSpecificProfile, nil
}

func isEnv(env string) (bool, error) {
	if env == "" {
		return true, nil
	}

	keyValue := strings.SplitN(env, "=", 2)
	if len(keyValue) != 2 {
		return false, fmt.Errorf("invalid env variable format: %s, should be KEY=VALUE", env)
	}

	key := keyValue[0]
	value := keyValue[1]

	envValue := os.Getenv(key)

	// Special case, since otherwise the regex substring check (`re.Compile("").MatchString(envValue)`)
	// would always match which is most probably not what the user wanted.
	if value == "" {
		return envValue == "", nil
	}

	return satisfies(value, envValue), nil
}

func isCommand(command string, opts cfg.SkaffoldOptions) bool {
	if command == "" {
		return true
	}

	return satisfies(command, opts.Command)
}

func isKubeContext(kubeContext string) (bool, error) {
	if kubeContext == "" {
		return true, nil
	}

	currentKubeConfig, err := kubectx.CurrentConfig()
	if err != nil {
		return false, errors.Wrap(err, "getting current cluster context")
	}

	return satisfies(kubeContext, currentKubeConfig.CurrentContext), nil
}

func satisfies(expected, actual string) bool {
	if strings.HasPrefix(expected, "!") {
		notExpected := expected[1:]

		return !matches(notExpected, actual)
	}

	return matches(expected, actual)
}

func matches(expected, actual string) bool {
	if actual == expected {
		return true
	}

	matcher, err := re.Compile(expected)
	if err != nil {
		logrus.Infof("profile activation criteria '%s' is not a valid regexp, falling back to string", expected)
		return false
	}

	return matcher.MatchString(actual)
}

func applyProfile(config *latest.SkaffoldConfig, profile latest.Profile) error {
	logrus.Infof("applying profile: %s", profile.Name)

	// Apply profile, field by field
	mergedV := reflect.Indirect(reflect.ValueOf(&config.Pipeline))
	configV := reflect.ValueOf(config.Pipeline)
	profileV := reflect.ValueOf(profile.Pipeline)

	profileT := profileV.Type()
	for i := 0; i < profileT.NumField(); i++ {
		name := profileT.Field(i).Name
		merged := overlayProfileField(name, configV.FieldByName(name).Interface(), profileV.FieldByName(name).Interface())
		mergedV.FieldByName(name).Set(reflect.ValueOf(merged))
	}

	// Remove the Profiles field from the returned config
	config.Profiles = nil

	if len(profile.Patches) == 0 {
		return nil
	}

	// Apply profile patches
	buf, err := yaml.Marshal(*config)
	if err != nil {
		return err
	}

	var patches []yamlpatch.Operation
	for _, patch := range profile.Patches {
		// Default patch operation to `replace`
		op := patch.Op
		if op == "" {
			op = "replace"
		}

		var value *yamlpatch.Node
		if v := patch.Value; v != nil {
			value = &v.Node
		}

		patch := yamlpatch.Operation{
			Op:    yamlpatch.Op(op),
			Path:  yamlpatch.OpPath(patch.Path),
			From:  yamlpatch.OpPath(patch.From),
			Value: value,
		}

		if !tryPatch(patch, buf) {
			return fmt.Errorf("invalid path: %s", patch.Path)
		}

		patches = append(patches, patch)
	}

	buf, err = yamlpatch.Patch(patches).Apply(buf)
	if err != nil {
		return err
	}

	return yaml.Unmarshal(buf, config)
}

// tryPatch is here to verify patches one by one before we
// apply them because yamlpatch.Patch is known to panic when a path
// is not valid.
func tryPatch(patch yamlpatch.Operation, buf []byte) (valid bool) {
	defer func() {
		if errPanic := recover(); errPanic != nil {
			valid = false
		}
	}()

	_, err := yamlpatch.Patch([]yamlpatch.Operation{patch}).Apply(buf)
	return err == nil
}

func profilesByName(profiles []latest.Profile) map[string]latest.Profile {
	byName := make(map[string]latest.Profile)
	for _, profile := range profiles {
		byName[profile.Name] = profile
	}
	return byName
}

// if we find a oneOf tag, the fields in this struct are themselves pointers to structs,
// but should be treated as values. the first non-nil one we find is what we should use.
func overlayOneOfField(config interface{}, profile interface{}) interface{} {
	v := reflect.ValueOf(profile) // the profile itself
	t := reflect.TypeOf(profile)  // the type of the profile, used for getting struct field types
	for i := 0; i < v.NumField(); i++ {
		fieldType := t.Field(i)              // the field type (e.g. 'LocalBuild' for BuildConfig)
		fieldValue := v.Field(i).Interface() // the value of the field itself

		if fieldValue != nil && !reflect.ValueOf(fieldValue).IsNil() {
			ret := reflect.New(t)                                                   // New(t) returns a Value representing pointer to new zero value for type t
			ret.Elem().FieldByName(fieldType.Name).Set(reflect.ValueOf(fieldValue)) // set the value
			return reflect.Indirect(ret).Interface()                                // since ret is a pointer, dereference it
		}
	}
	// if we're here, we didn't find any values set in the profile config. just return the original.
	logrus.Infof("no values found in profile for field %s, using original config values", t.Name())
	return config
}

func overlayStructField(config interface{}, profile interface{}) interface{} {
	// we already know the top level fields for whatever struct we have are themselves structs
	// (and not one-of values), so we need to recursively overlay them
	configValue := reflect.ValueOf(config)
	profileValue := reflect.ValueOf(profile)
	t := reflect.TypeOf(profile)
	finalConfig := reflect.New(t)

	for i := 0; i < profileValue.NumField(); i++ {
		fieldType := t.Field(i)
		overlay := overlayProfileField(yamltags.YamlName(fieldType), configValue.Field(i).Interface(), profileValue.Field(i).Interface())
		finalConfig.Elem().FieldByName(fieldType.Name).Set(reflect.ValueOf(overlay))
	}
	return reflect.Indirect(finalConfig).Interface() // since finalConfig is a pointer, dereference it
}

func overlayProfileField(fieldName string, config interface{}, profile interface{}) interface{} {
	v := reflect.ValueOf(profile) // the profile itself
	t := reflect.TypeOf(profile)  // the type of the profile, used for getting struct field types
	logrus.Debugf("overlaying profile on config for field %s", fieldName)
	switch v.Kind() {
	case reflect.Struct:
		// check the first field of the struct for a oneOf yamltag.
		if util.IsOneOfField(t.Field(0)) {
			return overlayOneOfField(config, profile)
		}
		return overlayStructField(config, profile)
	case reflect.Slice:
		// either return the values provided in the profile, or the original values if none were provided.
		if v.Len() == 0 {
			return config
		}
		return v.Interface()
	case reflect.Ptr:
		// either return the values provided in the profile, or the original values if none were provided.
		if v.IsNil() {
			return config
		}
		return v.Interface()
	case reflect.Int:
		if v.Interface() == reflect.Zero(v.Type()).Interface() {
			return config
		}
		return v.Interface()
	case reflect.String:
		if reflect.DeepEqual("", v.Interface()) {
			return config
		}
		return v.Interface()
	default:
		logrus.Fatalf("Type mismatch in profile overlay for field '%s' with type %s; falling back to original config values", fieldName, v.Kind())
		return config
	}
}
<commit_msg>review comments<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"fmt"
	"os"
	"reflect"
	re "regexp"
	"strings"

	cfg "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/yamltags"
	yamlpatch "github.com/krishicks/yaml-patch"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	yaml "gopkg.in/yaml.v2"
)

var (
	enableKubeContext = kubectx.ChangeKubeContext
)

// ApplyProfiles returns configuration modified by the application
// of a list of profiles.
func ApplyProfiles(c *latest.SkaffoldConfig, opts cfg.SkaffoldOptions) error {
	byName := profilesByName(c.Profiles)

	profiles, hasContextActivatedProfile, err := activatedProfiles(c.Profiles, opts)
	if err != nil {
		return errors.Wrap(err, "finding auto-activated profiles")
	}

	for _, name := range profiles {
		profile, present := byName[name]
		if !present {
			return fmt.Errorf("couldn't find profile %s", name)
		}

		if err := applyProfile(c, profile); err != nil {
			return errors.Wrapf(err, "applying profile %s", name)
		}
	}

	return enableEffectiveKubecontext(hasContextActivatedProfile, opts.KubeContext, c.Deploy.KubeContext)
}

func enableEffectiveKubecontext(isContextImmutable bool, cliContext, effectiveContext string) error {
	// cli flag takes precedence
	if cliContext != "" {
		return nil
	}

	kubeConfig, err := kubectx.CurrentConfig()
	if err != nil {
		return errors.Wrap(err, "getting current cluster context")
	}

	// nothing to do
	if effectiveContext == "" || effectiveContext == kubeConfig.CurrentContext {
		return nil
	}

	if isContextImmutable {
		return fmt.Errorf("some activated profile contains kubecontext specific settings for a different than the effective kubecontext, please revise your profile activations")
	}

	enableKubeContext(effectiveContext)
	return nil
}

func activatedProfiles(profiles []latest.Profile, opts cfg.SkaffoldOptions) ([]string, bool, error) {
	activated := opts.Profiles
	hasContextActivatedProfile := false

	// Auto-activated profiles
	for _, profile := range profiles {
		for _, cond := range profile.Activation {
			command := isCommand(cond.Command, opts)

			env, err := isEnv(cond.Env)
			if err != nil {
				return nil, false, err
			}

			kubeContext, err := isKubeContext(cond.KubeContext)
			if err != nil {
				return nil, false, err
			}

			if command && env && kubeContext {
				if cond.KubeContext != "" {
					hasContextActivatedProfile = true
				}
				activated = append(activated, profile.Name)
			}
		}
	}

	return activated, hasContextActivatedProfile, nil
}

func isEnv(env string) (bool, error) {
	if env == "" {
		return true, nil
	}

	keyValue := strings.SplitN(env, "=", 2)
	if len(keyValue) != 2 {
		return false, fmt.Errorf("invalid env variable format: %s, should be KEY=VALUE", env)
	}

	key := keyValue[0]
	value := keyValue[1]

	envValue := os.Getenv(key)

	// Special case, since otherwise the regex substring check (`re.Compile("").MatchString(envValue)`)
	// would always match which is most probably not what the user wanted.
	if value == "" {
		return envValue == "", nil
	}

	return satisfies(value, envValue), nil
}

func isCommand(command string, opts cfg.SkaffoldOptions) bool {
	if command == "" {
		return true
	}

	return satisfies(command, opts.Command)
}

func isKubeContext(kubeContext string) (bool, error) {
	if kubeContext == "" {
		return true, nil
	}

	currentKubeConfig, err := kubectx.CurrentConfig()
	if err != nil {
		return false, errors.Wrap(err, "getting current cluster context")
	}

	return satisfies(kubeContext, currentKubeConfig.CurrentContext), nil
}

func satisfies(expected, actual string) bool {
	if strings.HasPrefix(expected, "!") {
		notExpected := expected[1:]

		return !matches(notExpected, actual)
	}

	return matches(expected, actual)
}

func matches(expected, actual string) bool {
	if actual == expected {
		return true
	}

	matcher, err := re.Compile(expected)
	if err != nil {
		logrus.Infof("profile activation criteria '%s' is not a valid regexp, falling back to string", expected)
		return false
	}

	return matcher.MatchString(actual)
}

func applyProfile(config *latest.SkaffoldConfig, profile latest.Profile) error {
	logrus.Infof("applying profile: %s", profile.Name)

	// Apply profile, field by field
	mergedV := reflect.Indirect(reflect.ValueOf(&config.Pipeline))
	configV := reflect.ValueOf(config.Pipeline)
	profileV := reflect.ValueOf(profile.Pipeline)

	profileT := profileV.Type()
	for i := 0; i < profileT.NumField(); i++ {
		name := profileT.Field(i).Name
		merged := overlayProfileField(name, configV.FieldByName(name).Interface(), profileV.FieldByName(name).Interface())
		mergedV.FieldByName(name).Set(reflect.ValueOf(merged))
	}

	// Remove the Profiles field from the returned config
	config.Profiles = nil

	if len(profile.Patches) == 0 {
		return nil
	}

	// Apply profile patches
	buf, err := yaml.Marshal(*config)
	if err != nil {
		return err
	}

	var patches []yamlpatch.Operation
	for _, patch := range profile.Patches {
		// Default patch operation to `replace`
		op := patch.Op
		if op == "" {
			op = "replace"
		}

		var value *yamlpatch.Node
		if v := patch.Value; v != nil {
			value = &v.Node
		}

		patch := yamlpatch.Operation{
			Op:    yamlpatch.Op(op),
			Path:  yamlpatch.OpPath(patch.Path),
			From:  yamlpatch.OpPath(patch.From),
			Value: value,
		}

		if !tryPatch(patch, buf) {
			return fmt.Errorf("invalid path: %s", patch.Path)
		}

		patches = append(patches, patch)
	}

	buf, err = yamlpatch.Patch(patches).Apply(buf)
	if err != nil {
		return err
	}

	return yaml.Unmarshal(buf, config)
}

// tryPatch is here to verify patches one by one before we
// apply them because yamlpatch.Patch is known to panic when a path
// is not valid.
func tryPatch(patch yamlpatch.Operation, buf []byte) (valid bool) {
	defer func() {
		if errPanic := recover(); errPanic != nil {
			valid = false
		}
	}()

	_, err := yamlpatch.Patch([]yamlpatch.Operation{patch}).Apply(buf)
	return err == nil
}

func profilesByName(profiles []latest.Profile) map[string]latest.Profile {
	byName := make(map[string]latest.Profile)
	for _, profile := range profiles {
		byName[profile.Name] = profile
	}
	return byName
}

// if we find a oneOf tag, the fields in this struct are themselves pointers to structs,
// but should be treated as values. the first non-nil one we find is what we should use.
func overlayOneOfField(config interface{}, profile interface{}) interface{} {
	v := reflect.ValueOf(profile) // the profile itself
	t := reflect.TypeOf(profile)  // the type of the profile, used for getting struct field types
	for i := 0; i < v.NumField(); i++ {
		fieldType := t.Field(i)              // the field type (e.g. 'LocalBuild' for BuildConfig)
		fieldValue := v.Field(i).Interface() // the value of the field itself

		if fieldValue != nil && !reflect.ValueOf(fieldValue).IsNil() {
			ret := reflect.New(t)                                                   // New(t) returns a Value representing pointer to new zero value for type t
			ret.Elem().FieldByName(fieldType.Name).Set(reflect.ValueOf(fieldValue)) // set the value
			return reflect.Indirect(ret).Interface()                                // since ret is a pointer, dereference it
		}
	}
	// if we're here, we didn't find any values set in the profile config. just return the original.
	logrus.Infof("no values found in profile for field %s, using original config values", t.Name())
	return config
}

func overlayStructField(config interface{}, profile interface{}) interface{} {
	// we already know the top level fields for whatever struct we have are themselves structs
	// (and not one-of values), so we need to recursively overlay them
	configValue := reflect.ValueOf(config)
	profileValue := reflect.ValueOf(profile)
	t := reflect.TypeOf(profile)
	finalConfig := reflect.New(t)

	for i := 0; i < profileValue.NumField(); i++ {
		fieldType := t.Field(i)
		overlay := overlayProfileField(yamltags.YamlName(fieldType), configValue.Field(i).Interface(), profileValue.Field(i).Interface())
		finalConfig.Elem().FieldByName(fieldType.Name).Set(reflect.ValueOf(overlay))
	}
	return reflect.Indirect(finalConfig).Interface() // since finalConfig is a pointer, dereference it
}

func overlayProfileField(fieldName string, config interface{}, profile interface{}) interface{} {
	v := reflect.ValueOf(profile) // the profile itself
	t := reflect.TypeOf(profile)  // the type of the profile, used for getting struct field types
	logrus.Debugf("overlaying profile on config for field %s", fieldName)
	switch v.Kind() {
	case reflect.Struct:
		// check the first field of the struct for a oneOf yamltag.
		if util.IsOneOfField(t.Field(0)) {
			return overlayOneOfField(config, profile)
		}
		return overlayStructField(config, profile)
	case reflect.Slice:
		// either return the values provided in the profile, or the original values if none were provided.
		if v.Len() == 0 {
			return config
		}
		return v.Interface()
	case reflect.Ptr:
		// either return the values provided in the profile, or the original values if none were provided.
		if v.IsNil() {
			return config
		}
		return v.Interface()
	case reflect.Int:
		if v.Interface() == reflect.Zero(v.Type()).Interface() {
			return config
		}
		return v.Interface()
	case reflect.String:
		if reflect.DeepEqual("", v.Interface()) {
			return config
		}
		return v.Interface()
	default:
		logrus.Fatalf("Type mismatch in profile overlay for field '%s' with type %s; falling back to original config values", fieldName, v.Kind())
		return config
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/jessevdk/go-flags"

	"github.com/zimmski/tavor"
	"github.com/zimmski/tavor/fuzz/strategy"
	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/parser"
)

const (
	returnOk = iota
	returnHelp
	returnBashCompletion
)

var opts struct {
	Config         func(s flags.Filename) error `long:"config" description:"INI config file" no-ini:"true"`
	ConfigWrite    flags.Filename               `long:"config-write" description:"Write all arguments to an INI config file or to STDOUT with \"-\" as argument." no-ini:"true"`
	Debug          bool                         `long:"debug" description:"Temporary debugging argument"`
	Help           bool                         `long:"help" description:"Show this help message" no-ini:"true"`
	ListStrategies bool                         `long:"list-strategies" description:"List all available strategies." no-ini:"true"`
	PrintInternal  bool                         `long:"print-internal" description:"Prints the internal AST of the parsed file"`
	Seed           int64                        `long:"seed" description:"Seed for all the randomness"`
	Strategy       string                       `long:"strategy" description:"The fuzzing strategy" default:"random"`
	Validate       bool                         `long:"validate" description:"Just validates the input file"`
	Verbose        bool                         `long:"verbose" description:"Do verbose output."`
	Version        bool                         `long:"version" description:"Print the version of this program." no-ini:"true"`

	Positional struct {
		FormatFile flags.Filename `description:"Input tavor format file" required:"true" no-ini:"true"`
	} `positional-args:"true" required:"true"`

	configFile string
}

func checkArguments() {
	p := flags.NewNamedParser("tavor", flags.PassDoubleDash)

	p.ShortDescription = "A fuzzing and delta-debugging platform."

	opts.Config = func(s flags.Filename) error {
		ini := flags.NewIniParser(p)

		opts.configFile = string(s)

		return ini.ParseFile(string(s))
	}

	p.AddGroup("Tavor", "Tavor arguments", &opts)

	_, err := p.Parse()

	if opts.Help || len(os.Args) == 1 {
		p.WriteHelp(os.Stdout)

		os.Exit(returnHelp)
	} else if opts.Version {
		fmt.Printf("Tavor v%s\n", tavor.Version)

		os.Exit(returnOk)
	} else if opts.ListStrategies {
		for _, name := range strategy.List() {
			fmt.Println(name)
		}

		os.Exit(returnOk)
	}

	if err != nil {
		panic(err)
	}

	if len(os.Getenv("GO_FLAGS_COMPLETION")) != 0 {
		os.Exit(returnBashCompletion)
	}

	if opts.ConfigWrite != "" {
		ini := flags.NewIniParser(p)

		var iniOptions flags.IniOptions = flags.IniIncludeComments | flags.IniIncludeDefaults | flags.IniCommentDefaults

		if opts.ConfigWrite == "-" {
			(ini.Write(os.Stdout, iniOptions))
		} else {
			ini.WriteFile(string(opts.ConfigWrite), iniOptions)
		}

		os.Exit(returnOk)
	}

	if opts.Seed == 0 {
		opts.Seed = time.Now().UTC().UnixNano()
	}

	if opts.Debug {
		log.LevelDebug()
	} else if opts.Verbose {
		log.LevelInfo()
	} else {
		log.LevelWarn()
	}
}

func main() {
	checkArguments()

	log.Infof("Open file %s", opts.Positional.FormatFile)

	file, err := os.Open(string(opts.Positional.FormatFile))
	if err != nil {
		panic(fmt.Errorf("cannot open tavor file %s: %v", opts.Positional.FormatFile, err))
	}
	defer file.Close()

	doc, err := parser.ParseTavor(file)
	if err != nil {
		panic(fmt.Errorf("cannot parse tavor file: %v", err))
	}

	log.Info("File is valid")

	if opts.PrintInternal {
		tavor.PrettyPrintInternalTree(os.Stdout, doc)
	}

	if opts.Validate {
		os.Exit(returnOk)
	}

	log.Infof("Using seed %d", opts.Seed)
	log.Infof("Counted %d overall permutations", doc.PermutationsAll())

	r := rand.New(rand.NewSource(opts.Seed))

	strat, err := strategy.New(opts.Strategy, doc)
	if err != nil {
		panic(err)
	}

	log.Infof("Using %s strategy", opts.Strategy)

	ch, err := strat.Fuzz(r)
	if err != nil {
		panic(err)
	}

	another := false
	for i := range ch {
		if !opts.Debug {
			if another {
				fmt.Println()
			} else {
				another = true
			}
		}

		log.Debug("Result:")

		fmt.Print(doc.String())
		if opts.Debug {
			fmt.Println()
		}

		ch <- i
	}
}
<commit_msg>add strategy completion<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/jessevdk/go-flags"

	"github.com/zimmski/tavor"
	"github.com/zimmski/tavor/fuzz/strategy"
	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/parser"
)

const (
	returnOk = iota
	returnHelp
	returnBashCompletion
)

var opts struct {
	Config         func(s flags.Filename) error `long:"config" description:"INI config file" no-ini:"true"`
	ConfigWrite    flags.Filename               `long:"config-write" description:"Write all arguments to an INI config file or to STDOUT with \"-\" as argument." no-ini:"true"`
	Debug          bool                         `long:"debug" description:"Temporary debugging argument"`
	FormatFile     flags.Filename               `long:"format-file" description:"Input tavor format file" required:"true" no-ini:"true"`
	Help           bool                         `long:"help" description:"Show this help message" no-ini:"true"`
	ListStrategies bool                         `long:"list-strategies" description:"List all available strategies." no-ini:"true"`
	PrintInternal  bool                         `long:"print-internal" description:"Prints the internal AST of the parsed file"`
	Seed           int64                        `long:"seed" description:"Seed for all the randomness"`
	Strategy       Strategy                     `long:"strategy" description:"The fuzzing strategy" default:"random"`
	Validate       bool                         `long:"validate" description:"Just validates the input file"`
	Verbose        bool                         `long:"verbose" description:"Do verbose output."`
	Version        bool                         `long:"version" description:"Print the version of this program." no-ini:"true"`

	configFile string
}

type Strategy string

func (s *Strategy) Complete(match string) []flags.Completion {
	var items []flags.Completion

	for _, name := range strategy.List() {
		if strings.HasPrefix(name, match) {
			items = append(items, flags.Completion{
				Item: name,
			})
		}
	}

	return items
}

func checkArguments() {
	p := flags.NewNamedParser("tavor", flags.PassDoubleDash)

	p.ShortDescription = "A fuzzing and delta-debugging platform."

	opts.Config = func(s flags.Filename) error {
		ini := flags.NewIniParser(p)

		opts.configFile = string(s)

		return ini.ParseFile(string(s))
	}

	p.AddGroup("Tavor", "Tavor arguments", &opts)

	_, err := p.Parse()
	if opts.Help || len(os.Args) == 1 {
		p.WriteHelp(os.Stdout)

		os.Exit(returnHelp)
	} else if opts.Version {
		fmt.Printf("Tavor v%s\n", tavor.Version)

		os.Exit(returnOk)
	} else if opts.ListStrategies {
		for _, name := range strategy.List() {
			fmt.Println(name)
		}

		os.Exit(returnOk)
	}

	if err != nil {
		panic(err)
	}

	if len(os.Getenv("GO_FLAGS_COMPLETION")) != 0 {
		os.Exit(returnBashCompletion)
	}

	if opts.ConfigWrite != "" {
		ini := flags.NewIniParser(p)

		var iniOptions flags.IniOptions = flags.IniIncludeComments | flags.IniIncludeDefaults | flags.IniCommentDefaults

		if opts.ConfigWrite == "-" {
			(ini.Write(os.Stdout, iniOptions))
		} else {
			ini.WriteFile(string(opts.ConfigWrite), iniOptions)
		}

		os.Exit(returnOk)
	}

	if opts.Seed == 0 {
		opts.Seed = time.Now().UTC().UnixNano()
	}

	if opts.Debug {
		log.LevelDebug()
	} else if opts.Verbose {
		log.LevelInfo()
	} else {
		log.LevelWarn()
	}
}

func main() {
	checkArguments()

	log.Infof("Open file %s", opts.FormatFile)

	file, err := os.Open(string(opts.FormatFile))
	if err != nil {
		panic(fmt.Errorf("cannot open tavor file %s: %v", opts.FormatFile, err))
	}
	defer file.Close()

	doc, err := parser.ParseTavor(file)
	if err != nil {
		panic(fmt.Errorf("cannot parse tavor file: %v", err))
	}

	log.Info("File is valid")

	if opts.PrintInternal {
		tavor.PrettyPrintInternalTree(os.Stdout, doc)
	}

	if opts.Validate {
		os.Exit(returnOk)
	}

	log.Infof("Using seed %d", opts.Seed)
	log.Infof("Counted %d overall permutations", doc.PermutationsAll())

	r := rand.New(rand.NewSource(opts.Seed))

	strat, err := strategy.New(string(opts.Strategy), doc)
	if err != nil {
		panic(err)
	}

	log.Infof("Using %s strategy", opts.Strategy)

	ch, err := strat.Fuzz(r)
	if err != nil {
		panic(err)
	}

	another := false
	for i := range ch {
		if !opts.Debug {
			if another {
				fmt.Println()
			} else {
				another = true
			}
		}

		log.Debug("Result:")

		fmt.Print(doc.String())
		if opts.Debug {
			fmt.Println()
		}

		ch <- i
	}
}
<|endoftext|>


<commit_before>package validation

import "github.com/prometheus/client_golang/prometheus"

const (
	discardReasonLabel = "reason"

	// RateLimited is one of the values for the reason to discard samples.
	// Declared here to avoid duplication in ingester and distributor.
	RateLimited = "rate_limited"
)

// DiscardedBytes is a metric of the total discarded bytes, by reason.
var DiscardedBytes = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "loki_discarded_bytes_total",
		Help: "The total number of bytes that were discarded.",
	},
	[]string{discardReasonLabel, "user"},
)

// DiscardedSamples is a metric of the number of discarded samples, by reason.
var DiscardedSamples = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "loki_discarded_samples_total",
		Help: "The total number of samples that were discarded.",
	},
	[]string{discardReasonLabel, "user"},
)

func init() {
	prometheus.MustRegister(DiscardedSamples)
}
<commit_msg>Expose discarded bytes metric (#1317)<commit_after>package validation

import "github.com/prometheus/client_golang/prometheus"

const (
	discardReasonLabel = "reason"

	// RateLimited is one of the values for the reason to discard samples.
	// Declared here to avoid duplication in ingester and distributor.
	RateLimited = "rate_limited"
)

// DiscardedBytes is a metric of the total discarded bytes, by reason.
var DiscardedBytes = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "loki_discarded_bytes_total",
		Help: "The total number of bytes that were discarded.",
	},
	[]string{discardReasonLabel, "user"},
)

// DiscardedSamples is a metric of the number of discarded samples, by reason.
var DiscardedSamples = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "loki_discarded_samples_total",
		Help: "The total number of samples that were discarded.",
	},
	[]string{discardReasonLabel, "user"},
)

func init() {
	prometheus.MustRegister(DiscardedSamples, DiscardedBytes)
}
<|endoftext|>


<commit_before>package hmac

import (
	"crypto/sha1"
	"hash"
	"sync"
)

func resetBytes(b []byte) {
	for i := range b {
		b[i] = 0
	}
}

func (h *hmac) resetTo(key []byte) {
	h.outer.Reset()
	h.inner.Reset()
	resetBytes(h.ipad)
	resetBytes(h.opad)
	if len(key) > h.blocksize {
		// If key is too big, hash it.
		h.outer.Write(key)
		key = h.outer.Sum(nil)
	}
	copy(h.ipad, key)
	copy(h.opad, key)
	for i := range h.ipad {
		h.ipad[i] ^= 0x36
	}
	for i := range h.opad {
		h.opad[i] ^= 0x5c
	}
	h.inner.Write(h.ipad)
}

var hmacSHA1Pool = &sync.Pool{
	New: func() interface{} {
		h := New(sha1.New, make([]byte, sha1.BlockSize))
		return h
	},
}

// AcquireSHA1 returns new HMAC from pool.
func AcquireSHA1(key []byte) hash.Hash {
	h := hmacSHA1Pool.Get().(*hmac)
	h.resetTo(key)
	return h
}

// PutSHA1 puts h to pool.
func PutSHA1(h hash.Hash) {
	hm := h.(*hmac)
	hmacSHA1Pool.Put(hm)
}
<commit_msg>hmac: rename resetBytes to setZeroes<commit_after>package hmac

import (
	"crypto/sha1"
	"hash"
	"sync"
)

func setZeroes(b []byte) {
	for i := range b {
		b[i] = 0
	}
}

func (h *hmac) resetTo(key []byte) {
	h.outer.Reset()
	h.inner.Reset()
	setZeroes(h.ipad)
	setZeroes(h.opad)
	if len(key) > h.blocksize {
		// If key is too big, hash it.
		h.outer.Write(key)
		key = h.outer.Sum(nil)
	}
	copy(h.ipad, key)
	copy(h.opad, key)
	for i := range h.ipad {
		h.ipad[i] ^= 0x36
	}
	for i := range h.opad {
		h.opad[i] ^= 0x5c
	}
	h.inner.Write(h.ipad)
}

var hmacSHA1Pool = &sync.Pool{
	New: func() interface{} {
		h := New(sha1.New, make([]byte, sha1.BlockSize))
		return h
	},
}

// AcquireSHA1 returns new HMAC from pool.
func AcquireSHA1(key []byte) hash.Hash {
	h := hmacSHA1Pool.Get().(*hmac)
	h.resetTo(key)
	return h
}

// PutSHA1 puts h to pool.
func PutSHA1(h hash.Hash) {
	hm := h.(*hmac)
	hmacSHA1Pool.Put(hm)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"fmt"
	"io"
	"regexp"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/fs"
	"github.com/jacobsa/syncutil"
)

// Given a base directory and a set of exclusions, list all file system nodes
// involved, filling in only the RelPath and Parent fields. Output nodes are
// guaranteed to be in reverse topologically sorted order: children appear
// before their parents.
func listNodes(
	ctx context.Context,
	basePath string,
	exclusions []*regexp.Regexp,
	nodes chan<- *fsNode) (err error) {
	b := syncutil.NewBundle(ctx)

	// Find the nodes in the appropriate order, writing them to a channel of
	// graph.Node.
	graphNodes := make(chan graph.Node, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(graphNodes)

		// Set up a root node.
		rootNode := &fsNode{
			RelPath: "",
			Info: fs.DirectoryEntry{
				Type: fs.TypeDirectory,
			},
			Parent: nil,
		}

		// Explore the graph.
		sf := newSuccessorFinder(basePath, exclusions)
		err = graph.ReverseTopsortTree(
			ctx,
			sf,
			rootNode,
			graphNodes)

		if err != nil {
			err = fmt.Errorf("ReverseTopsortTree: %v", err)
			return
		}

		return
	})

	// Convert to *fsNode.
	b.Add(func(ctx context.Context) (err error) {
		for graphNode := range graphNodes {
			n, ok := graphNode.(*fsNode)
			if !ok {
				err = fmt.Errorf("Unexpected node type: %T", graphNode)
				return
			}

			select {
			case nodes <- n:
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		return
	})

	err = b.Join()
	return
}

// Given a base directory and a set of exclusions, list the files and
// directories that would be saved by a backup job with the same info in a
// human-readable format. Write the output to the supplied writer.
func List(
	ctx context.Context,
	w io.Writer,
	basePath string,
	exclusions []*regexp.Regexp) (err error) {
	b := syncutil.NewBundle(ctx)

	// List nodes.
	nodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(nodes)
		err = listNodes(ctx, basePath, exclusions, nodes)
		if err != nil {
			err = fmt.Errorf("listNodes: %v", err)
			return
		}

		return
	})

	// Print out info about each node.
	b.Add(func(ctx context.Context) (err error) {
		for n := range nodes {
			_, err = fmt.Fprintf(w, "%q %d\n", n.RelPath, n.Info.Size)
			if err != nil {
				err = fmt.Errorf("Fprintf: %v", err)
				return
			}
		}

		return
	})

	err = b.Join()
	return
}
<commit_msg>Fixed list.go.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"errors"
	"fmt"
	"io"
	"regexp"

	"golang.org/x/net/context"

	"github.com/jacobsa/syncutil"
)

// Given a base directory and a set of exclusions, list all file system nodes
// involved.
func listNodes(
	ctx context.Context,
	basePath string,
	exclusions []*regexp.Regexp,
	nodes chan<- *fsNode) (err error) {
	// Visit all nodes in the graph with a visitor that simply writes them to a
	// channel.
	err = errors.New("TODO")
	return
}

// Given a base directory and a set of exclusions, list the files and
// directories that would be saved by a backup job with the same info in a
// human-readable format. Write the output to the supplied writer.
func List(
	ctx context.Context,
	w io.Writer,
	basePath string,
	exclusions []*regexp.Regexp) (err error) {
	b := syncutil.NewBundle(ctx)

	// List nodes.
	nodes := make(chan *fsNode, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(nodes)
		err = listNodes(ctx, basePath, exclusions, nodes)
		if err != nil {
			err = fmt.Errorf("listNodes: %v", err)
			return
		}

		return
	})

	// Print out info about each node.
	b.Add(func(ctx context.Context) (err error) {
		for n := range nodes {
			_, err = fmt.Fprintf(w, "%q %d\n", n.RelPath, n.Info.Size)
			if err != nil {
				err = fmt.Errorf("Fprintf: %v", err)
				return
			}
		}

		return
	})

	err = b.Join()
	return
}
<|endoftext|>