<commit_before>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package glog

import (
	"errors"
	"fmt"
	"strings"
)

// Note that the LEVEL_PANI and LEVEL_FATA levels are not used for logging output,
// but for prefix configurations.
const (
	// Deprecated.
	LEVEL_ALL = LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT
	// Deprecated.
	LEVEL_DEV = LEVEL_ALL
	// Deprecated.
	LEVEL_PROD = LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT

	LEVEL_DEBU = 1 << iota // 8
	LEVEL_INFO             // 16
	LEVEL_NOTI             // 32
	LEVEL_WARN             // 64
	LEVEL_ERRO             // 128
	LEVEL_CRIT             // 256
	LEVEL_PANI             // 512
	LEVEL_FATA             // 1024
)

// defaultLevelPrefixes defines the default level and its mapping prefix string.
var defaultLevelPrefixes = map[int]string{
	LEVEL_DEBU: "DEBU",
	LEVEL_INFO: "INFO",
	LEVEL_NOTI: "NOTI",
	LEVEL_WARN: "WARN",
	LEVEL_ERRO: "ERRO",
	LEVEL_CRIT: "CRIT",
	LEVEL_PANI: "PANI",
	LEVEL_FATA: "FATA",
}

// levelStringMap defines level string name to its level mapping.
var levelStringMap = map[string]int{
	"ALL":      LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEV":      LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEVELOP":  LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"PROD":     LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"PRODUCT":  LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEBU":     LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEBUG":    LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"INFO":     LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"NOTI":     LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"NOTICE":   LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"WARN":     LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"WARNING":  LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"ERRO":     LEVEL_ERRO | LEVEL_CRIT,
	"ERROR":    LEVEL_ERRO | LEVEL_CRIT,
	"CRIT":     LEVEL_CRIT,
	"CRITICAL": LEVEL_CRIT,
}

// SetLevel sets the logging level.
func (l *Logger) SetLevel(level int) {
	l.config.Level = level
}

// GetLevel returns the logging level value.
func (l *Logger) GetLevel() int {
	return l.config.Level
}

// SetLevelStr sets the logging level by level string.
func (l *Logger) SetLevelStr(levelStr string) error {
	if level, ok := levelStringMap[strings.ToUpper(levelStr)]; ok {
		l.config.Level = level
	} else {
		return errors.New(fmt.Sprintf(`invalid level string: %s`, levelStr))
	}
	return nil
}

// SetLevelPrefix sets the prefix string for specified level.
func (l *Logger) SetLevelPrefix(level int, prefix string) {
	l.config.LevelPrefixes[level] = prefix
}

// SetLevelPrefixes sets the level to prefix string mapping for the logger.
func (l *Logger) SetLevelPrefixes(prefixes map[int]string) {
	for k, v := range prefixes {
		l.config.LevelPrefixes[k] = v
	}
}

// GetLevelPrefix returns the prefix string for specified level.
func (l *Logger) GetLevelPrefix(level int) string {
	return l.config.LevelPrefixes[level]
}

// getLevelPrefixWithBrackets returns the prefix string with brackets for specified level.
func (l *Logger) getLevelPrefixWithBrackets(level int) string {
	if s, ok := l.config.LevelPrefixes[level]; ok {
		return "[" + s + "]"
	}
	return ""
}
<commit_msg>add level string feature for package glog<commit_after>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package glog

import (
	"errors"
	"fmt"
	"strings"
)

// Note that the LEVEL_PANI and LEVEL_FATA levels are not used for logging output,
// but for prefix configurations.
const (
	LEVEL_ALL  = LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT
	LEVEL_DEV  = LEVEL_ALL
	LEVEL_PROD = LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT
	LEVEL_DEBU = 1 << iota // 8
	LEVEL_INFO             // 16
	LEVEL_NOTI             // 32
	LEVEL_WARN             // 64
	LEVEL_ERRO             // 128
	LEVEL_CRIT             // 256
	LEVEL_PANI             // 512
	LEVEL_FATA             // 1024
)

// defaultLevelPrefixes defines the default level and its mapping prefix string.
var defaultLevelPrefixes = map[int]string{
	LEVEL_DEBU: "DEBU",
	LEVEL_INFO: "INFO",
	LEVEL_NOTI: "NOTI",
	LEVEL_WARN: "WARN",
	LEVEL_ERRO: "ERRO",
	LEVEL_CRIT: "CRIT",
	LEVEL_PANI: "PANI",
	LEVEL_FATA: "FATA",
}

// levelStringMap defines level string name to its level mapping.
var levelStringMap = map[string]int{
	"ALL":      LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEV":      LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEVELOP":  LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"PROD":     LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"PRODUCT":  LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEBU":     LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"DEBUG":    LEVEL_DEBU | LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"INFO":     LEVEL_INFO | LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"NOTI":     LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"NOTICE":   LEVEL_NOTI | LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"WARN":     LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"WARNING":  LEVEL_WARN | LEVEL_ERRO | LEVEL_CRIT,
	"ERRO":     LEVEL_ERRO | LEVEL_CRIT,
	"ERROR":    LEVEL_ERRO | LEVEL_CRIT,
	"CRIT":     LEVEL_CRIT,
	"CRITICAL": LEVEL_CRIT,
}

// SetLevel sets the logging level.
func (l *Logger) SetLevel(level int) {
	l.config.Level = level
}

// GetLevel returns the logging level value.
func (l *Logger) GetLevel() int {
	return l.config.Level
}

// SetLevelStr sets the logging level by level string.
func (l *Logger) SetLevelStr(levelStr string) error {
	if level, ok := levelStringMap[strings.ToUpper(levelStr)]; ok {
		l.config.Level = level
	} else {
		return errors.New(fmt.Sprintf(`invalid level string: %s`, levelStr))
	}
	return nil
}

// SetLevelPrefix sets the prefix string for specified level.
func (l *Logger) SetLevelPrefix(level int, prefix string) {
	l.config.LevelPrefixes[level] = prefix
}

// SetLevelPrefixes sets the level to prefix string mapping for the logger.
func (l *Logger) SetLevelPrefixes(prefixes map[int]string) {
	for k, v := range prefixes {
		l.config.LevelPrefixes[k] = v
	}
}

// GetLevelPrefix returns the prefix string for specified level.
func (l *Logger) GetLevelPrefix(level int) string {
	return l.config.LevelPrefixes[level]
}

// getLevelPrefixWithBrackets returns the prefix string with brackets for specified level.
func (l *Logger) getLevelPrefixWithBrackets(level int) string {
	if s, ok := l.config.LevelPrefixes[level]; ok {
		return "[" + s + "]"
	}
	return ""
}
<|endoftext|>


<commit_before>package gogl

import (
	"fmt"
	"math"
	"testing"
)

var fml = fmt.Println

var edgeSet = []Edge{
	&BaseEdge{"foo", "bar"},
	&BaseEdge{"bar", "baz"},
}

func TestEnsureIsGraph(t *testing.T) {
	// What is Go's best practice for ensuring the implementation of an interface?
	_ = Graph(NewDirectedAdjacencyList())
	_ = SimpleGraph(NewDirectedAdjacencyList())
	t.Log("Implements Graph interface as expected.")
}

func TestVertexMembership(t *testing.T) {
	g := NewDirectedAdjacencyList()

	if g.HasVertex("foo") != false {
		t.Error("Incorrectly reports nonexistent vertex as present.")
	}

	if g.AddVertex("foo") != true {
		t.Error("Fails to add string vertex correctly.")
	}

	if g.HasVertex("foo") != true {
		t.Error("Fails to locate existing string vertex.")
	}

	if g.AddVertex(1) != true {
		t.Error("Fails to add int vertex correctly.")
	}

	if g.RemoveVertex("foo") != true {
		t.Error("Reports incorrect failure on removing existing vertex.")
	}

	if g.HasVertex("foo") != false {
		t.Error("Reports vertex still present after removal.")
	}
}

func TestRemoveVertexWithEdges(t *testing.T) {
	g := NewDirectedAdjacencyListFromEdgeSet(edgeSet)

	g.RemoveVertex("bar")

	if count, _ := g.OutDegree("foo"); count != 0 {
		t.Error("Removal of vertex in edge pair does not result in decrement of outdegree of the other vertex.")
	}
}

func TestEachVertex(t *testing.T) {
	g := NewDirectedAdjacencyList()

	var hit uint
	f := func(v Vertex) {
		hit++
	}

	g.EachVertex(f)

	if hit != 0 {
		t.Error("EachVertex did not call provided closure expected number of times.")
	}

	g.AddVertex("foo")
	g.AddVertex("bar")
	g.EachVertex(f)

	if hit != 2 {
		t.Error("EachVertex did not call provided closure expected number of times.")
	}
}

func TestAddEdge(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"foo", "bar"})

	if g.HasVertex("foo") != true {
		t.Error("AddEdge did not create vertices as expected.")
	}

	f := func(adj Vertex) {
		if adj != "bar" {
			t.Error("Adjacency relationship from foo to bar not reported correctly, ", adj, " was passed.")
		}
	}

	g.EachAdjacent("foo", f)
}

func TestOutDegree(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"foo", "bar"})

	var count uint
	var exists bool
	count, exists = g.OutDegree("foo")

	if count != 1 {
		t.Error("Vertex should have outdegree of one, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.OutDegree("bar")

	if count != 0 {
		t.Error("Vertex should have outdegree of zero, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.OutDegree("baz")

	if count != 0 {
		t.Error("Zero outdegree count is reported when vertex does not exist.")
	}

	if exists != false {
		t.Error("Vertex should not exist.")
	}
}

func TestInDegree(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"bar", "foo"})

	var count uint
	var exists bool
	count, exists = g.InDegree("foo")

	if count != 1 {
		t.Error("Vertex should have indegree of one, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.InDegree("bar")

	if count != 0 {
		t.Error("Vertex should have indegree of zero, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.InDegree("baz")

	if count != 0 {
		t.Error("Zero indegree count is reported when vertex does not exist.")
	}

	if exists != false {
		t.Error("Vertex should not exist.")
	}
}

func TestEachEdge(t *testing.T) {
	g := NewDirectedAdjacencyListFromEdgeSet(edgeSet)

	var hit uint
	f := func(e Edge) {
		hit++
	}

	g.EachEdge(f)

	if hit != 2 {
		t.Error("Edge iterator should have been called twice, was called", hit, "times.")
	}
}

func TestSize(t *testing.T) {
	g := NewDirectedAdjacencyList()

	if g.Size() != 0 {
		t.Error("Graph initializes with non-zero size.")
	}

	g.AddEdge(&BaseEdge{"foo", "bar"})

	if g.Size() != 1 {
		t.Error("Graph does not increment size properly on edge addition.")
	}

	g.RemoveEdge(&BaseEdge{"foo", "bar"})

	if g.Size() != 0 {
		t.Error("Graph does not decrement size properly on edge removal.")
	}
}

func TestOrder(t *testing.T) {
	g := NewDirectedAdjacencyList()

	if g.Order() != 0 {
		t.Error("Graph initializes with non-zero order.")
	}

	g.AddVertex("foo")

	if g.Order() != 1 {
		t.Error("Adding a vertex does not increment order properly.")
	}

	g.RemoveVertex("foo")

	if g.Order() != 0 {
		t.Error("Removing a vertex does not decrement order properly.")
	}
}

func TestDensity(t *testing.T) {
	g := NewDirectedAdjacencyList()
	var density float64

	if !math.IsNaN(g.Density()) {
		t.Error("On graph initialize, Density should be NaN (divides zero by zero)).")
	}

	g.AddEdge(&BaseEdge{"foo", "bar"})

	density = g.Density()
	if density != 1 {
		t.Error("In undirected graph of V = 2 and E = 1, density should be 1; was", density)
	}

	g.AddEdge(&BaseEdge{"baz", "qux"})

	density = g.Density()
	if density != float64(1)/float64(3) {
		t.Error("In undirected graph of V = 4 and E = 2, density should be 0.3333; was", density)
	}
}
<commit_msg>Start experimenting with goconvey for tests.<commit_after>package gogl

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"math"
	"testing"
)

var fml = fmt.Println

var edgeSet = []Edge{
	&BaseEdge{"foo", "bar"},
	&BaseEdge{"bar", "baz"},
}

func TestEnsureGraphInterfaces(t *testing.T) {
	// What is Go's best practice for ensuring the implementation of an interface?
	_ = Graph(NewDirectedAdjacencyList())
	_ = SimpleGraph(NewDirectedAdjacencyList())
	t.Log("Implements Graph interface as expected.")
}

func TestVertexMembership(t *testing.T) {
	g := NewDirectedAdjacencyList()

	Convey("Test adding, removal, and membership of string literal vertex.", t, func() {
		So(g.HasVertex("foo"), ShouldEqual, false)
		So(g.AddVertex("foo"), ShouldEqual, true)
		So(g.AddVertex("foo"), ShouldEqual, false)
		So(g.HasVertex("foo"), ShouldEqual, true)
		So(g.RemoveVertex("foo"), ShouldEqual, true)
		So(g.HasVertex("foo"), ShouldEqual, false)
	})

	Convey("Test adding, removal, and membership of int literal vertex.", t, func() {
		So(g.HasVertex(1), ShouldEqual, false)
		So(g.AddVertex(1), ShouldEqual, true)
		So(g.AddVertex(1), ShouldEqual, false)
		So(g.HasVertex(1), ShouldEqual, true)
		So(g.RemoveVertex(1), ShouldEqual, true)
		So(g.HasVertex(1), ShouldEqual, false)
	})

	Convey("Test adding, removal, and membership of composite literal vertex.", t, func() {
		So(g.HasVertex(edgeSet[0]), ShouldEqual, false)
		So(g.AddVertex(edgeSet[0]), ShouldEqual, true)
		So(g.AddVertex(edgeSet[0]), ShouldEqual, false)
		So(g.HasVertex(edgeSet[0]), ShouldEqual, true)

		Convey("No membership match on new struct with same values or new pointer", func() {
			So(g.HasVertex(BaseEdge{"foo", "bar"}), ShouldEqual, false)
			So(g.HasVertex(&BaseEdge{"foo", "bar"}), ShouldEqual, false)
		})

		So(g.RemoveVertex(edgeSet[0]), ShouldEqual, true)
		So(g.HasVertex(edgeSet[0]), ShouldEqual, false)
	})
}

func TestRemoveVertexWithEdges(t *testing.T) {
	g := NewDirectedAdjacencyListFromEdgeSet(edgeSet)

	Convey("Ensure outdegree is decremented when vertex is removed.", t, func() {
		g.RemoveVertex("bar")
		count, exists := g.OutDegree("foo")
		So(count, ShouldEqual, 0)
		So(exists, ShouldEqual, true)
	})
}

func TestEachVertex(t *testing.T) {
	g := NewDirectedAdjacencyList()

	var hit uint
	f := func(v Vertex) {
		hit++
	}

	g.EachVertex(f)

	if hit != 0 {
		t.Error("EachVertex did not call provided closure expected number of times.")
	}

	g.AddVertex("foo")
	g.AddVertex("bar")
	g.EachVertex(f)

	if hit != 2 {
		t.Error("EachVertex did not call provided closure expected number of times.")
	}
}

func TestAddEdge(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"foo", "bar"})

	if g.HasVertex("foo") != true {
		t.Error("AddEdge did not create vertices as expected.")
	}

	f := func(adj Vertex) {
		if adj != "bar" {
			t.Error("Adjacency relationship from foo to bar not reported correctly, ", adj, " was passed.")
		}
	}

	g.EachAdjacent("foo", f)
}

func TestOutDegree(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"foo", "bar"})

	var count uint
	var exists bool
	count, exists = g.OutDegree("foo")

	if count != 1 {
		t.Error("Vertex should have outdegree of one, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.OutDegree("bar")

	if count != 0 {
		t.Error("Vertex should have outdegree of zero, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.OutDegree("baz")

	if count != 0 {
		t.Error("Zero outdegree count is reported when vertex does not exist.")
	}

	if exists != false {
		t.Error("Vertex should not exist.")
	}
}

func TestInDegree(t *testing.T) {
	g := NewDirectedAdjacencyList()

	g.AddEdge(&BaseEdge{"bar", "foo"})

	var count uint
	var exists bool
	count, exists = g.InDegree("foo")

	if count != 1 {
		t.Error("Vertex should have indegree of one, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.InDegree("bar")

	if count != 0 {
		t.Error("Vertex should have indegree of zero, but", count, "was reported.")
	}

	if exists != true {
		t.Error("Vertex should exist.")
	}

	count, exists = g.InDegree("baz")

	if count != 0 {
		t.Error("Zero indegree count is reported when vertex does not exist.")
	}

	if exists != false {
		t.Error("Vertex should not exist.")
	}
}

func TestEachEdge(t *testing.T) {
	g := NewDirectedAdjacencyListFromEdgeSet(edgeSet)

	var hit uint
	f := func(e Edge) {
		hit++
	}

	g.EachEdge(f)

	if hit != 2 {
		t.Error("Edge iterator should have been called twice, was called", hit, "times.")
	}
}

func TestSize(t *testing.T) {
	g := NewDirectedAdjacencyList()

	if g.Size() != 0 {
		t.Error("Graph initializes with non-zero size.")
	}

	g.AddEdge(&BaseEdge{"foo", "bar"})

	if g.Size() != 1 {
		t.Error("Graph does not increment size properly on edge addition.")
	}

	g.RemoveEdge(&BaseEdge{"foo", "bar"})

	if g.Size() != 0 {
		t.Error("Graph does not decrement size properly on edge removal.")
	}
}

func TestOrder(t *testing.T) {
	g := NewDirectedAdjacencyList()

	if g.Order() != 0 {
		t.Error("Graph initializes with non-zero order.")
	}

	g.AddVertex("foo")

	if g.Order() != 1 {
		t.Error("Adding a vertex does not increment order properly.")
	}

	g.RemoveVertex("foo")

	if g.Order() != 0 {
		t.Error("Removing a vertex does not decrement order properly.")
	}
}

func TestDensity(t *testing.T) {
	g := NewDirectedAdjacencyList()
	var density float64

	if !math.IsNaN(g.Density()) {
		t.Error("On graph initialize, Density should be NaN (divides zero by zero)).")
	}

	g.AddEdge(&BaseEdge{"foo", "bar"})

	density = g.Density()
	if density != 1 {
		t.Error("In undirected graph of V = 2 and E = 1, density should be 1; was", density)
	}

	g.AddEdge(&BaseEdge{"baz", "qux"})

	density = g.Density()
	if density != float64(1)/float64(3) {
		t.Error("In undirected graph of V = 4 and E = 2, density should be 0.3333; was", density)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package abac

// Policy authorizes Kubernetes API actions using an Attribute-based access
// control scheme.

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer"
)

// TODO: make this into a real API object.  Note that when that happens, it
// will get MetaData.  However, the Kind and Namespace in the struct below
// will be separate from the Kind and Namespace in the Metadata.  Obviously,
// meta.Kind will be something like policy, and policy.Kind has to be allowed
// to be different.  Less obviously, namespace needs to be different as well.
// This will allow wildcard matching strings to be used in the future for the
// body.Namespace, if we want to add that feature, without affecting the
// meta.Namespace.
type policy struct {
	User  string `json:"user,omitempty"`
	Group string `json:"group,omitempty"`
	// TODO: add support for robot accounts as well as human user accounts.
	// TODO: decide how to namespace user names when multiple authentication
	// providers are in use. Either add "Realm", or assume "user@example.com"
	// format.

	// TODO: Make the "cluster" Kinds be one API group (minions, bindings,
	// events, endpoints).  The "user" Kinds are another (pods, services,
	// replicationControllers, operations) Make a "plugin", e.g. build
	// controller, be another group.  That way when we add a new object to a
	// the API, we don't have to add lots of policy?

	// TODO: make this a proper REST object with its own registry.
	Readonly  bool   `json:"readonly,omitempty"`
	Resource  string `json:"resource,omitempty"`
	Namespace string `json:"namespace,omitempty"`

	// TODO: "expires" string in RFC3339 format.

	// TODO: want a way to allow some users to restart containers of a pod but
	// not delete or modify it.

	// TODO: want a way to allow a controller to create a pod based only on a
	// certain podTemplates.
}

type policyList []policy

// TODO: Have policies be created via an API call and stored in REST storage.
func NewFromFile(path string) (policyList, error) {
	// File format is one map per line.  This allows easy concatentation of files,
	// comments in files, and identification of errors by line number.
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	pl := make(policyList, 0)
	var p policy

	for scanner.Scan() {
		b := scanner.Bytes()
		// TODO: skip comment lines.
		err = json.Unmarshal(b, &p)
		if err != nil {
			// TODO: line number in errors.
			return nil, err
		}
		pl = append(pl, p)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return pl, nil
}

func (p policy) matches(a authorizer.Attributes) bool {
	if p.subjectMatches(a) {
		if p.Readonly == false || (p.Readonly == a.IsReadOnly()) {
			if p.Resource == "" || (p.Resource == a.GetResource()) {
				if p.Namespace == "" || (p.Namespace == a.GetNamespace()) {
					return true
				}
			}
		}
	}
	return false
}

func (p policy) subjectMatches(a authorizer.Attributes) bool {
	if p.User != "" {
		// Require user match
		if p.User != a.GetUserName() {
			return false
		}
	}

	if p.Group != "" {
		// Require group match
		for _, group := range a.GetGroups() {
			if p.Group == group {
				return true
			}
		}
		return false
	}

	return true
}

// Authorizer implements authorizer.Authorize
func (pl policyList) Authorize(a authorizer.Attributes) error {
	for _, p := range pl {
		if p.matches(a) {
			return nil
		}
	}
	return errors.New("No policy matched.")
	// TODO: Benchmark how much time policy matching takes with a medium size
	// policy file, compared to other steps such as encoding/decoding.
	// Then, add Caching only if needed.
}
<commit_msg>abac policy file parsing bug<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package abac

// Policy authorizes Kubernetes API actions using an Attribute-based access
// control scheme.

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer"
)

// TODO: make this into a real API object.  Note that when that happens, it
// will get MetaData.  However, the Kind and Namespace in the struct below
// will be separate from the Kind and Namespace in the Metadata.  Obviously,
// meta.Kind will be something like policy, and policy.Kind has to be allowed
// to be different.  Less obviously, namespace needs to be different as well.
// This will allow wildcard matching strings to be used in the future for the
// body.Namespace, if we want to add that feature, without affecting the
// meta.Namespace.
type policy struct {
	User  string `json:"user,omitempty"`
	Group string `json:"group,omitempty"`
	// TODO: add support for robot accounts as well as human user accounts.
	// TODO: decide how to namespace user names when multiple authentication
	// providers are in use. Either add "Realm", or assume "user@example.com"
	// format.

	// TODO: Make the "cluster" Kinds be one API group (minions, bindings,
	// events, endpoints).  The "user" Kinds are another (pods, services,
	// replicationControllers, operations) Make a "plugin", e.g. build
	// controller, be another group.  That way when we add a new object to a
	// the API, we don't have to add lots of policy?

	// TODO: make this a proper REST object with its own registry.
	Readonly  bool   `json:"readonly,omitempty"`
	Resource  string `json:"resource,omitempty"`
	Namespace string `json:"namespace,omitempty"`

	// TODO: "expires" string in RFC3339 format.

	// TODO: want a way to allow some users to restart containers of a pod but
	// not delete or modify it.

	// TODO: want a way to allow a controller to create a pod based only on a
	// certain podTemplates.
}

type policyList []policy

// TODO: Have policies be created via an API call and stored in REST storage.
func NewFromFile(path string) (policyList, error) {
	// File format is one map per line.  This allows easy concatentation of files,
	// comments in files, and identification of errors by line number.
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	pl := make(policyList, 0)

	for scanner.Scan() {
		var p policy
		b := scanner.Bytes()
		// TODO: skip comment lines.
		err = json.Unmarshal(b, &p)
		if err != nil {
			// TODO: line number in errors.
			return nil, err
		}
		pl = append(pl, p)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return pl, nil
}

func (p policy) matches(a authorizer.Attributes) bool {
	if p.subjectMatches(a) {
		if p.Readonly == false || (p.Readonly == a.IsReadOnly()) {
			if p.Resource == "" || (p.Resource == a.GetResource()) {
				if p.Namespace == "" || (p.Namespace == a.GetNamespace()) {
					return true
				}
			}
		}
	}
	return false
}

func (p policy) subjectMatches(a authorizer.Attributes) bool {
	if p.User != "" {
		// Require user match
		if p.User != a.GetUserName() {
			return false
		}
	}

	if p.Group != "" {
		// Require group match
		for _, group := range a.GetGroups() {
			if p.Group == group {
				return true
			}
		}
		return false
	}

	return true
}

// Authorizer implements authorizer.Authorize
func (pl policyList) Authorize(a authorizer.Attributes) error {
	for _, p := range pl {
		if p.matches(a) {
			return nil
		}
	}
	return errors.New("No policy matched.")
	// TODO: Benchmark how much time policy matching takes with a medium size
	// policy file, compared to other steps such as encoding/decoding.
	// Then, add Caching only if needed.
}
<|endoftext|>


<commit_before>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sink

import (
	"context"

	"github.com/cilium/cilium/pkg/hubble/recorder/pcap"
	"github.com/cilium/cilium/pkg/lock"
)

// sink wraps a pcap.RecordWriter by adding a queue and managing its statistics
// regarding written and dropped packets and bytes.
type sink struct {
	mutex     lock.Mutex
	queue     chan record
	shutdown  chan struct{}
	done      chan struct{}
	trigger   chan struct{}
	stats     Statistics
	lastError error
}

// startSink creates a queue and go routine for the sink. The spawned go
// routine will run until one of the following happens:
//  - sink.stop is called
//  - ctx is cancelled
//  - an error occurred
func startSink(ctx context.Context, p PcapSink, queueSize int) *sink {
	s := &sink{
		mutex:     lock.Mutex{},
		queue:     make(chan record, queueSize),
		shutdown:  make(chan struct{}),
		done:      make(chan struct{}),
		trigger:   make(chan struct{}, 1),
		stats:     Statistics{},
		lastError: nil,
	}

	go func() {
		// this defer executes w.Close(), but also makes sure set lastError and
		// close the channels when exiting.
		var err error
		defer func() {
			closeErr := p.Writer.Close()

			s.mutex.Lock()
			if err == nil {
				s.lastError = closeErr
			} else {
				s.lastError = err
			}
			close(s.done)
			s.mutex.Unlock()
		}()

		if err = p.Writer.WriteHeader(p.Header); err != nil {
			return
		}

		for {
			select {
			// s.queue will be closed when the sink is unregistered
			case rec := <-s.queue:
				pcapRecord := pcap.Record{
					Timestamp:      rec.timestamp,
					CaptureLength:  rec.inclLen,
					OriginalLength: rec.origLen,
				}

				if err = p.Writer.WriteRecord(pcapRecord, rec.data); err != nil {
					return
				}

				s.addToStatistics(Statistics{
					PacketsWritten: 1,
					BytesWritten:   uint64(rec.inclLen),
				})
			case <-s.shutdown:
				return
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}
	}()

	return s
}

// stop requests the sink to stop recording
func (s *sink) stop() {
	close(s.shutdown)
}

func (s *sink) addToStatistics(add Statistics) {
	s.mutex.Lock()
	s.stats.BytesWritten += add.BytesWritten
	s.stats.PacketsWritten += add.PacketsWritten
	s.stats.BytesLost += add.BytesLost
	s.stats.PacketsLost += add.PacketsLost

	// non-blocking send
	select {
	case s.trigger <- struct{}{}:
	default:
	}

	s.mutex.Unlock()
}

// enqueue submits a new record to this sink. If the sink is not keeping up,
// the record is dropped and the sink statistics are updated accordingly
func (s *sink) enqueue(rec record) {
	select {
	case <-s.shutdown:
		// early return if shutting down
		return
	default:
	}

	select {
	case s.queue <- rec:
		// successfully enqueued rec in sink
		return
	default:
	}

	// sink queue was full, update statistics
	s.addToStatistics(Statistics{
		PacketsLost: 1,
		BytesLost:   uint64(rec.inclLen),
	})
}

// copyStats creates a snapshot of the current statistics
func (s *sink) copyStats() Statistics {
	s.mutex.Lock()
	stats := s.stats
	s.mutex.Unlock()

	return stats
}

// err returns the last error which occurred in the sink.
// This will always return nil before sink.done has signalled.
func (s *sink) err() error {
	s.mutex.Lock()
	err := s.lastError
	s.mutex.Unlock()

	return err
}
<commit_msg>hubble/recorder: Be more explicit about mutex<commit_after>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sink

import (
	"context"

	"github.com/cilium/cilium/pkg/hubble/recorder/pcap"
	"github.com/cilium/cilium/pkg/lock"
)

// sink wraps a pcap.RecordWriter by adding a queue and managing its statistics
// regarding written and dropped packets and bytes.
type sink struct {
	// these channels are initialized in startSink and never reassigned, thus
	// they can be accessed without locking mutex
	queue    chan record
	shutdown chan struct{}
	done     chan struct{}
	trigger  chan struct{}
	// mutex protects writes to stats and lastError
	mutex     lock.Mutex
	stats     Statistics
	lastError error
}

// startSink creates a queue and go routine for the sink. The spawned go
// routine will run until one of the following happens:
//  - sink.stop is called
//  - ctx is cancelled
//  - an error occurred
func startSink(ctx context.Context, p PcapSink, queueSize int) *sink {
	s := &sink{
		queue:     make(chan record, queueSize),
		shutdown:  make(chan struct{}),
		done:      make(chan struct{}),
		trigger:   make(chan struct{}, 1),
		mutex:     lock.Mutex{},
		stats:     Statistics{},
		lastError: nil,
	}

	go func() {
		// this defer executes p.Writer.Close(), but also makes sure to set
		// lastError and close the s.done channel when exiting.
		var err error
		defer func() {
			closeErr := p.Writer.Close()

			s.mutex.Lock()
			if err == nil {
				s.lastError = closeErr
			} else {
				s.lastError = err
			}
			s.mutex.Unlock()
			close(s.done)
		}()

		if err = p.Writer.WriteHeader(p.Header); err != nil {
			return
		}

		for {
			select {
			// s.queue will be closed when the sink is unregistered
			case rec := <-s.queue:
				pcapRecord := pcap.Record{
					Timestamp:      rec.timestamp,
					CaptureLength:  rec.inclLen,
					OriginalLength: rec.origLen,
				}

				if err = p.Writer.WriteRecord(pcapRecord, rec.data); err != nil {
					return
				}

				s.addToStatistics(Statistics{
					PacketsWritten: 1,
					BytesWritten:   uint64(rec.inclLen),
				})
			case <-s.shutdown:
				return
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}
	}()

	return s
}

// stop requests the sink to stop recording
func (s *sink) stop() {
	close(s.shutdown)
}

func (s *sink) addToStatistics(add Statistics) {
	s.mutex.Lock()
	s.stats.BytesWritten += add.BytesWritten
	s.stats.PacketsWritten += add.PacketsWritten
	s.stats.BytesLost += add.BytesLost
	s.stats.PacketsLost += add.PacketsLost
	s.mutex.Unlock()

	// non-blocking send
	select {
	case s.trigger <- struct{}{}:
	default:
	}
}

// enqueue submits a new record to this sink. If the sink is not keeping up,
// the record is dropped and the sink statistics are updated accordingly
func (s *sink) enqueue(rec record) {
	select {
	case <-s.shutdown:
		// early return if shutting down
		return
	default:
	}

	select {
	case s.queue <- rec:
		// successfully enqueued rec in sink
		return
	default:
	}

	// sink queue was full, update statistics
	s.addToStatistics(Statistics{
		PacketsLost: 1,
		BytesLost:   uint64(rec.inclLen),
	})
}

// copyStats creates a snapshot of the current statistics
func (s *sink) copyStats() Statistics {
	s.mutex.Lock()
	stats := s.stats
	s.mutex.Unlock()

	return stats
}

// err returns the last error which occurred in the sink.
// This will always return nil before sink.done has signalled.
func (s *sink) err() error {
	s.mutex.Lock()
	err := s.lastError
	s.mutex.Unlock()

	return err
}
<|endoftext|>


<commit_before>// +build linux

package nsinit

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/label"
	"github.com/dotcloud/docker/pkg/libcontainer"
	"github.com/dotcloud/docker/pkg/system"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
)

// default mount point flags
const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV

// setupNewMountNamespace is used to initialize a new mount namespace for an new
// container in the rootfs that is specified.
//
// There is no need to unmount the new mounts because as soon as the mount namespace
// is no longer in use, the mounts will be removed automatically
func setupNewMountNamespace(rootfs string, bindMounts []libcontainer.Mount, console string, readonly, noPivotRoot bool, mountLabel string) error {
	flag := syscall.MS_PRIVATE
	if noPivotRoot {
		flag = syscall.MS_SLAVE
	}
	if err := system.Mount("", "/", "", uintptr(flag|syscall.MS_REC), ""); err != nil {
		return fmt.Errorf("mounting / as slave %s", err)
	}
	if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mouting %s as bind %s", rootfs, err)
	}
	if err := mountSystem(rootfs, mountLabel); err != nil {
		return fmt.Errorf("mount system %s", err)
	}

	for _, m := range bindMounts {
		flags := syscall.MS_BIND | syscall.MS_REC
		if !m.Writable {
			flags = flags | syscall.MS_RDONLY
		}
		dest := filepath.Join(rootfs, m.Destination)
		if err := system.Mount(m.Source, dest, "bind", uintptr(flags), ""); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.Source, dest, err)
		}
		if m.Private {
			if err := system.Mount("", dest, "none", uintptr(syscall.MS_PRIVATE), ""); err != nil {
				return fmt.Errorf("mounting %s private %s", dest, err)
			}
		}
	}

	if err := copyDevNodes(rootfs); err != nil {
		return fmt.Errorf("copy dev nodes %s", err)
	}
	// In non-privileged mode, this fails. Discard the error.
	setupLoopbackDevices(rootfs)
	if err := setupPtmx(rootfs, console, mountLabel); err != nil {
		return err
	}
	if err := system.Chdir(rootfs); err != nil {
		return fmt.Errorf("chdir into %s %s", rootfs, err)
	}

	if noPivotRoot {
		if err := rootMsMove(rootfs); err != nil {
			return err
		}
	} else {
		if err := rootPivot(rootfs); err != nil {
			return err
		}
	}

	if readonly {
		if err := system.Mount("/", "/", "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil {
			return fmt.Errorf("mounting %s as readonly %s", rootfs, err)
		}
	}

	system.Umask(0022)

	return nil
}

// use a pivot root to setup the rootfs
func rootPivot(rootfs string) error {
	pivotDir, err := ioutil.TempDir(rootfs, ".pivot_root")
	if err != nil {
		return fmt.Errorf("can't create pivot_root dir %s", pivotDir, err)
	}
	if err := system.Pivotroot(rootfs, pivotDir); err != nil {
		return fmt.Errorf("pivot_root %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}
	// path to pivot dir now changed, update
	pivotDir = filepath.Join("/", filepath.Base(pivotDir))
	if err := system.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
		return fmt.Errorf("unmount pivot_root dir %s", err)
	}
	if err := os.Remove(pivotDir); err != nil {
		return fmt.Errorf("remove pivot_root dir %s", err)
	}
	return nil
}

// use MS_MOVE and chroot to setup the rootfs
func rootMsMove(rootfs string) error {
	if err := system.Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil {
		return fmt.Errorf("mount move %s into / %s", rootfs, err)
	}
	if err := system.Chroot("."); err != nil {
		return fmt.Errorf("chroot . %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}
	return nil
}

// copyDevNodes mknods the hosts devices so the new container has access to them
func copyDevNodes(rootfs string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	for _, node := range []string{
		"null",
		"zero",
		"full",
		"random",
		"urandom",
		"tty",
	} {
		if err := copyDevNode(rootfs, node); err != nil {
			return err
		}
	}
	return nil
}

func setupLoopbackDevices(rootfs string) error {
	for i := 0; ; i++ {
		if err := copyDevNode(rootfs, fmt.Sprintf("loop%d", i)); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			break
		}

	}
	return nil
}

func copyDevNode(rootfs, node string) error {
	stat, err := os.Stat(filepath.Join("/dev", node))
	if err != nil {
		return err
	}
	var (
		dest = filepath.Join(rootfs, "dev", node)
		st   = stat.Sys().(*syscall.Stat_t)
	)
	if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) {
		return fmt.Errorf("copy %s %s", node, err)
	}
	return nil
}

// setupConsole ensures that the container has a proper /dev/console setup
func setupConsole(rootfs, console string, mountLabel string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	stat, err := os.Stat(console)
	if err != nil {
		return fmt.Errorf("stat console %s %s", console, err)
	}
	var (
		st   = stat.Sys().(*syscall.Stat_t)
		dest = filepath.Join(rootfs, "dev/console")
	)
	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove %s %s", dest, err)
	}
	if err := os.Chmod(console, 0600); err != nil {
		return err
	}
	if err := os.Chown(console, 0, 0); err != nil {
		return err
	}
	if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil {
		return fmt.Errorf("mknod %s %s", dest, err)
	}
	if err := label.SetFileLabel(console, mountLabel); err != nil {
		return fmt.Errorf("SetFileLabel Failed %s %s", dest, err)
	}
	if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", console, dest, err)
	}
	return nil
}

// mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts
// inside the mount namespace
func mountSystem(rootfs string, mountLabel string) error {
	for _, m := range []struct {
		source string
		path   string
		device string
		flags  int
		data   string
	}{
		{source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaultMountFlags},
		{source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaultMountFlags},
		{source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaultMountFlags, data: label.FormatMountLabel("mode=1755,size=65536k", mountLabel)},
		{source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: label.FormatMountLabel("newinstance,ptmxmode=0666,mode=620,gid=5", mountLabel)},
	} {
		if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) {
			return fmt.Errorf("mkdirall %s %s", m.path, err)
		}
		if err := system.Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err)
		}
	}
	return nil
}

// setupPtmx adds a symlink to pts/ptmx for /dev/ptmx and
// finishes setting up /dev/console
func setupPtmx(rootfs, console string, mountLabel string) error {
	ptmx := filepath.Join(rootfs, "dev/ptmx")
	if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) {
		return err
	}
	if err := os.Symlink("pts/ptmx", ptmx); err != nil {
		return fmt.Errorf("symlink dev ptmx %s", err)
	}
	if console != "" {
		if err := setupConsole(rootfs, console, mountLabel); err != nil {
			return err
		}
	}
	return nil
}

// remountProc is used to detach and remount the proc filesystem
// commonly needed with running a new process inside an existing container
func remountProc() error {
	if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil {
		return err
	}
	if err := system.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), ""); err != nil {
		return err
	}
	return nil
}

func remountSys() error {
	if err := system.Unmount("/sys", syscall.MNT_DETACH); err != nil {
		if err != syscall.EINVAL {
			return err
		}
	} else {
		if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaultMountFlags), ""); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Remove loopback setup for native driver Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)<commit_after>// +build linux

package nsinit

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/label"
	"github.com/dotcloud/docker/pkg/libcontainer"
	"github.com/dotcloud/docker/pkg/system"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
)

// default mount point flags
const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV

// setupNewMountNamespace is used to initialize a new mount namespace for an new
// container in the rootfs that is specified.
//
// There is no need to unmount the new mounts because as soon as the mount namespace
// is no longer in use, the mounts will be removed automatically
func setupNewMountNamespace(rootfs string, bindMounts []libcontainer.Mount, console string, readonly, noPivotRoot bool, mountLabel string) error {
	flag := syscall.MS_PRIVATE
	if noPivotRoot {
		flag = syscall.MS_SLAVE
	}
	if err := system.Mount("", "/", "", uintptr(flag|syscall.MS_REC), ""); err != nil {
		return fmt.Errorf("mounting / as slave %s", err)
	}
	if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mouting %s as bind %s", rootfs, err)
	}
	if err := mountSystem(rootfs, mountLabel); err != nil {
		return fmt.Errorf("mount system %s", err)
	}

	for _, m := range bindMounts {
		flags := syscall.MS_BIND | syscall.MS_REC
		if !m.Writable {
			flags = flags | syscall.MS_RDONLY
		}
		dest := filepath.Join(rootfs, m.Destination)
		if err := system.Mount(m.Source, dest, "bind", uintptr(flags), ""); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.Source, dest, err)
		}
		if m.Private {
			if err := system.Mount("", dest, "none", uintptr(syscall.MS_PRIVATE), ""); err != nil {
				return fmt.Errorf("mounting %s private %s", dest, err)
			}
		}
	}

	if err := copyDevNodes(rootfs); err != nil {
		return fmt.Errorf("copy dev nodes %s", err)
	}
	if err := setupPtmx(rootfs, console, mountLabel); err != nil {
		return err
	}
	if err := system.Chdir(rootfs); err != nil {
		return fmt.Errorf("chdir into %s %s", rootfs, err)
	}

	if noPivotRoot {
		if err := rootMsMove(rootfs); err != nil {
			return err
		}
	} else {
		if err := rootPivot(rootfs); err != nil {
			return err
		}
	}

	if readonly {
		if err := system.Mount("/", "/", "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil {
			return fmt.Errorf("mounting %s as readonly %s", rootfs, err)
		}
	}

	system.Umask(0022)

	return nil
}

// use a pivot root to setup the rootfs
func rootPivot(rootfs string) error {
	pivotDir, err := ioutil.TempDir(rootfs, ".pivot_root")
	if err != nil {
		return fmt.Errorf("can't create pivot_root dir %s", pivotDir, err)
	}
	if err := system.Pivotroot(rootfs, pivotDir); err != nil {
		return fmt.Errorf("pivot_root %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}
	// path to pivot dir now changed, update
	pivotDir = filepath.Join("/", filepath.Base(pivotDir))
	if err := system.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
		return fmt.Errorf("unmount pivot_root dir %s", err)
	}
	if err := os.Remove(pivotDir); err != nil {
		return fmt.Errorf("remove pivot_root dir %s", err)
	}
	return nil
}

// use MS_MOVE and chroot to setup the rootfs
func rootMsMove(rootfs string) error {
	if err := system.Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil {
		return fmt.Errorf("mount move %s into / %s", rootfs, err)
	}
	if err := system.Chroot("."); err != nil {
		return fmt.Errorf("chroot . %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}
	return nil
}

// copyDevNodes mknods the hosts devices so the new container has access to them
func copyDevNodes(rootfs string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	for _, node := range []string{
		"null",
		"zero",
		"full",
		"random",
		"urandom",
		"tty",
	} {
		if err := copyDevNode(rootfs, node); err != nil {
			return err
		}
	}
	return nil
}

func copyDevNode(rootfs, node string) error {
	stat, err := os.Stat(filepath.Join("/dev", node))
	if err != nil {
		return err
	}
	var (
		dest = filepath.Join(rootfs, "dev", node)
		st   = stat.Sys().(*syscall.Stat_t)
	)
	if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) {
		return fmt.Errorf("copy %s %s", node, err)
	}
	return nil
}

// setupConsole ensures that the container has a proper /dev/console setup
func setupConsole(rootfs, console string, mountLabel string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	stat, err := os.Stat(console)
	if err != nil {
		return fmt.Errorf("stat console %s %s", console, err)
	}
	var (
		st   = stat.Sys().(*syscall.Stat_t)
		dest = filepath.Join(rootfs, "dev/console")
	)
	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove %s %s", dest, err)
	}
	if err := os.Chmod(console, 0600); err != nil {
		return err
	}
	if err := os.Chown(console, 0, 0); err != nil {
		return err
	}
	if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil {
		return fmt.Errorf("mknod %s %s", dest, err)
	}
	if err := label.SetFileLabel(console, mountLabel); err != nil {
		return fmt.Errorf("SetFileLabel Failed %s %s", dest, err)
	}
	if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", console, dest, err)
	}
	return nil
}

// mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts
// inside the mount namespace
func mountSystem(rootfs string, mountLabel string) error {
	for _, m := range []struct {
		source string
		path   string
		device string
		flags  int
		data   string
	}{
		{source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaultMountFlags},
		{source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaultMountFlags},
		{source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaultMountFlags, data: label.FormatMountLabel("mode=1755,size=65536k", mountLabel)},
		{source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: label.FormatMountLabel("newinstance,ptmxmode=0666,mode=620,gid=5", mountLabel)},
	} {
		if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) {
			return fmt.Errorf("mkdirall %s %s", m.path, err)
		}
		if err := system.Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err)
		}
	}
	return nil
}

// setupPtmx adds a symlink to pts/ptmx for /dev/ptmx and
// finishes setting up /dev/console
func setupPtmx(rootfs, console string, mountLabel string) error {
	ptmx := filepath.Join(rootfs, "dev/ptmx")
	if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) {
		return err
	}
	if err := os.Symlink("pts/ptmx", ptmx); err != nil {
		return fmt.Errorf("symlink dev ptmx %s", err)
	}
	if console != "" {
		if err := setupConsole(rootfs, console, mountLabel); err != nil {
			return err
		}
	}
	return nil
}

// remountProc is used to detach and remount the proc filesystem
// commonly needed with running a new process inside an existing container
func remountProc() error {
	if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil {
		return err
	}
	if err := system.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), ""); err != nil {
		return err
	}
	return nil
}

func remountSys() error {
	if err := system.Unmount("/sys", syscall.MNT_DETACH); err != nil {
		if err != syscall.EINVAL {
			return err
		}
	} else {
		if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaultMountFlags), ""); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package searchV2

import (
	"context"
	"errors"

	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/accesscontrol"
	"github.com/grafana/grafana/pkg/services/featuremgmt"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/services/store"
	"github.com/grafana/grafana/pkg/setting"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
)

type StandardSearchService struct {
	registry.BackgroundService

	cfg  *setting.Cfg
	sql  *sqlstore.SQLStore
	auth FutureAuthService // eventually injected from elsewhere
	ac   accesscontrol.AccessControl

	logger         log.Logger
	dashboardIndex *dashboardIndex
	extender       DashboardIndexExtender
}

func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore store.EntityEventsService, ac accesscontrol.AccessControl) SearchService {
	extender := &NoopExtender{}
	s := &StandardSearchService{
		cfg: cfg,
		sql: sql,
		ac:  ac,
		auth: &simpleSQLAuthService{
			sql: sql,
			ac:  ac,
		},
		dashboardIndex: newDashboardIndex(
			newSQLDashboardLoader(sql),
			entityEventStore,
			extender.GetDocumentExtender(),
			newFolderIDLookup(sql),
		),
		logger:   log.New("searchV2"),
		extender: extender,
	}
	return s
}

func (s *StandardSearchService) IsDisabled() bool {
	if s.cfg == nil {
		return true
	}
	return !s.cfg.IsFeatureToggleEnabled(featuremgmt.FlagPanelTitleSearch)
}

func (s *StandardSearchService) Run(ctx context.Context) error {
	return s.dashboardIndex.run(ctx)
}

func (s *StandardSearchService) RegisterDashboardIndexExtender(ext DashboardIndexExtender) {
	s.extender = ext
	s.dashboardIndex.extender = ext.GetDocumentExtender()
}

func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backend.User, orgId int64) (*models.SignedInUser, error) {
	// TODO: get user & user's permissions from the request context

	getSignedInUserQuery := &models.GetSignedInUserQuery{
		Login: backendUser.Login,
		Email: backendUser.Email,
		OrgId: orgId,
	}

	err := s.sql.GetSignedInUser(ctx, getSignedInUserQuery)
	if err != nil {
		s.logger.Error("Error while retrieving user", "error", err, "email", backendUser.Email)
		return nil, errors.New("auth error")
	}

	if getSignedInUserQuery.Result == nil {
		s.logger.Error("No user found", "email", backendUser.Email)
		return nil, errors.New("auth error")
	}

	user := getSignedInUserQuery.Result

	if s.ac.IsDisabled() {
		return user, nil
	}

	if user.Permissions == nil {
		user.Permissions = make(map[int64]map[string][]string)
	}

	if _, ok := user.Permissions[orgId]; ok {
		// permissions as part of the `s.sql.GetSignedInUser` query - return early
		return user, nil
	}

	// TODO: ensure this is cached
	permissions, err := s.ac.GetUserPermissions(ctx, user,
		accesscontrol.Options{ReloadCache: false})
	if err != nil {
		s.logger.Error("failed to retrieve user permissions", "error", err, "email", backendUser.Email)
		return nil, errors.New("auth error")
	}

	user.Permissions[orgId] = accesscontrol.GroupScopesByAction(permissions)
	return user, nil
}

func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
	rsp := &backend.DataResponse{}
	signedInUser, err := s.getUser(ctx, user, orgID)
	if err != nil {
		rsp.Error = err
		return rsp
	}

	filter, err := s.auth.GetDashboardReadFilter(signedInUser)
	if err != nil {
		rsp.Error = err
		return rsp
	}

	reader, ok := s.dashboardIndex.getOrgReader(orgID)
	if !ok {
		go func() {
			s.dashboardIndex.buildSignals <- orgID
		}()
		rsp.Error = errors.New("search index is not ready, try again later")
		return rsp
	}

	return doSearchQuery(ctx, s.logger, reader, filter, q, s.extender.GetQueryExtender(q), s.cfg.AppSubURL)
}
<commit_msg>Search: fix anonymous user access (#51045)<commit_after>package searchV2

import (
	"context"
	"errors"

	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/registry"
	"github.com/grafana/grafana/pkg/services/accesscontrol"
	"github.com/grafana/grafana/pkg/services/featuremgmt"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/services/store"
	"github.com/grafana/grafana/pkg/setting"

	"github.com/grafana/grafana-plugin-sdk-go/backend"
)

type StandardSearchService struct {
	registry.BackgroundService

	cfg  *setting.Cfg
	sql  *sqlstore.SQLStore
	auth FutureAuthService // eventually injected from elsewhere
	ac   accesscontrol.AccessControl

	logger         log.Logger
	dashboardIndex *dashboardIndex
	extender       DashboardIndexExtender
}

func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore store.EntityEventsService, ac accesscontrol.AccessControl) SearchService {
	extender := &NoopExtender{}
	s := &StandardSearchService{
		cfg: cfg,
		sql: sql,
		ac:  ac,
		auth: &simpleSQLAuthService{
			sql: sql,
			ac:  ac,
		},
		dashboardIndex: newDashboardIndex(
			newSQLDashboardLoader(sql),
			entityEventStore,
			extender.GetDocumentExtender(),
			newFolderIDLookup(sql),
		),
		logger:   log.New("searchV2"),
		extender: extender,
	}
	return s
}

func (s *StandardSearchService) IsDisabled() bool {
	if s.cfg == nil {
		return true
	}
	return !s.cfg.IsFeatureToggleEnabled(featuremgmt.FlagPanelTitleSearch)
}

func (s *StandardSearchService) Run(ctx context.Context) error {
	return s.dashboardIndex.run(ctx)
}

func (s *StandardSearchService) RegisterDashboardIndexExtender(ext DashboardIndexExtender) {
	s.extender = ext
	s.dashboardIndex.extender = ext.GetDocumentExtender()
}

func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backend.User, orgId int64) (*models.SignedInUser, error) {
	// TODO: get user & user's permissions from the request context

	var user *models.SignedInUser
	if s.cfg.AnonymousEnabled && backendUser.Email == "" && backendUser.Login == "" {
		org, err := s.sql.GetOrgByName(s.cfg.AnonymousOrgName)
		if err != nil {
			s.logger.Error("Anonymous access organization error.", "org_name", s.cfg.AnonymousOrgName, "error", err)
			return nil, err
		}

		user = &models.SignedInUser{
			OrgId:       org.Id,
			OrgName:     org.Name,
			OrgRole:     models.RoleType(s.cfg.AnonymousOrgRole),
			IsAnonymous: true,
		}
	} else {
		getSignedInUserQuery := &models.GetSignedInUserQuery{
			Login: backendUser.Login,
			Email: backendUser.Email,
			OrgId: orgId,
		}
		err := s.sql.GetSignedInUser(ctx, getSignedInUserQuery)
		if err != nil {
			s.logger.Error("Error while retrieving user", "error", err, "email", backendUser.Email)
			return nil, errors.New("auth error")
		}

		if getSignedInUserQuery.Result == nil {
			s.logger.Error("No user found", "email", backendUser.Email)
			return nil, errors.New("auth error")
		}

		user = getSignedInUserQuery.Result
	}

	if s.ac.IsDisabled() {
		return user, nil
	}

	if user.Permissions == nil {
		user.Permissions = make(map[int64]map[string][]string)
	}

	if _, ok := user.Permissions[orgId]; ok {
		// permissions as part of the `s.sql.GetSignedInUser` query - return early
		return user, nil
	}

	// TODO: ensure this is cached
	permissions, err := s.ac.GetUserPermissions(ctx, user,
		accesscontrol.Options{ReloadCache: false})
	if err != nil {
		s.logger.Error("failed to retrieve user permissions", "error", err, "email", backendUser.Email)
		return nil, errors.New("auth error")
	}

	user.Permissions[orgId] = accesscontrol.GroupScopesByAction(permissions)
	return user, nil
}

func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
	rsp := &backend.DataResponse{}
	signedInUser, err := s.getUser(ctx, user, orgID)
	if err != nil {
		rsp.Error = err
		return rsp
	}

	filter, err := s.auth.GetDashboardReadFilter(signedInUser)
	if err != nil {
		rsp.Error = err
		return rsp
	}

	reader, ok := s.dashboardIndex.getOrgReader(orgID)
	if !ok {
		go func() {
			s.dashboardIndex.buildSignals <- orgID
		}()
		rsp.Error = errors.New("search index is not ready, try again later")
		return rsp
	}

	return doSearchQuery(ctx, s.logger, reader, filter, q, s.extender.GetQueryExtender(q), s.cfg.AppSubURL)
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && strings.Contains(event.Message, "connection refused") {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// this indicates a failure to create the sandbox that should not happen
			failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<commit_msg>test: Report FailedCreatePodSandBox for multus -> API as known issue (v2)<commit_after>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && (strings.Contains(event.Message, "connection refused") || strings.Contains(event.Message, "i/o timeout")) {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// this indicates a failure to create the sandbox that should not happen
			failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

type CinderDiskUtil struct{}

// Attaches a disk specified by a volume.CinderPersistenDisk to the current kubelet.
// Mounts the disk to its global path.
func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeMounter, globalPDPath string) error {
	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}
	cloud, err := b.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	diskid, err := cloud.AttachDisk(instanceid, b.pdName)
	if err != nil {
		return err
	}

	var devicePath string
	numTries := 0
	for {
		devicePath = cloud.GetDevicePath(diskid)
		probeAttachedVolume()

		_, err := os.Stat(devicePath)
		if err == nil {
			break
		}
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		numTries++
		if numTries == 10 {
			return errors.New("Could not attach disk: Timeout after 60s")
		}
		time.Sleep(time.Second * 6)
	}
	notmnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(globalPDPath, 0750); err != nil {
				return err
			}
			notmnt = true
		} else {
			return err
		}
	}
	if notmnt {
		err = b.blockDeviceMounter.FormatAndMount(devicePath, globalPDPath, b.fsType, options)
		if err != nil {
			os.Remove(globalPDPath)
			return err
		}
		glog.V(2).Infof("Safe mount successful: %q\n", devicePath)
	}
	return nil
}

// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error {
	globalPDPath := makeGlobalPDName(cd.plugin.host, cd.pdName)
	if err := cd.mounter.Unmount(globalPDPath); err != nil {
		return err
	}
	if err := os.Remove(globalPDPath); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath)

	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	if err = cloud.DetachDisk(instanceid, cd.pdName); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName)
	return nil
}

func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error {
	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(cd.pdName); err != nil {
		// OpenStack cloud provider returns volume.tryAgainError when necessary,
		// no handling needed here.
		glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName)
	return nil
}

func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) {
	// TODO: caching, currently it is overkill because it calls this function
	// only when it creates dynamic PV
	zones := make(sets.String)
	nodes, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		glog.V(2).Infof("Error listing nodes")
		return zones, err
	}
	for _, node := range nodes.Items {
		if zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]; ok {
			zones.Insert(zone)
		}
	}
	glog.V(4).Infof("zones found: %v", zones)
	return zones, nil
}

func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, volumeLabels map[string]string, fstype string, err error) {
	cloud, err := c.plugin.getCloudProvider()
	if err != nil {
		return "", 0, nil, "", err
	}

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	volSizeBytes := capacity.Value()
	// Cinder works with gigabytes, convert to GiB with rounding up
	volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024))
	name := volume.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // Cinder volume name can have up to 255 characters
	vtype := ""
	availability := ""
	// Apply ProvisionerParameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			vtype = v
		case "availability":
			availability = v
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}
	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Cinder")
	}

	if availability == "" {
		// No zone specified, choose one randomly in the same region
		zones, err := getZonesFromNodes(c.plugin.host.GetKubeClient())
		if err != nil {
			glog.V(2).Infof("error getting zone information: %v", err)
			return "", 0, nil, "", err
		}
		// if we did not get any zones, lets leave it blank and gophercloud will
		// use zone "nova" as default
		if len(zones) > 0 {
			availability = volume.ChooseZoneForVolume(zones, c.options.PVC.Name)
		}
	}

	volumeID, volumeAZ, IgnoreVolumeAZ, errr := cloud.CreateVolume(name, volSizeGB, vtype, availability, c.options.CloudTags)
	if errr != nil {
		glog.V(2).Infof("Error creating cinder volume: %v", errr)
		return "", 0, nil, "", errr
	}
	glog.V(2).Infof("Successfully created cinder volume %s", volumeID)

	// these are needed that pod is spawning to same AZ
	volumeLabels = make(map[string]string)
	if IgnoreVolumeAZ == false {
		volumeLabels[kubeletapis.LabelZoneFailureDomain] = volumeAZ
	}
	return volumeID, volSizeGB, volumeLabels, fstype, nil
}

func probeAttachedVolume() error {
	// rescan scsi bus
	scsiHostRescan()

	executor := exec.New()

        // udevadm settle waits for udevd to process the device creation
        // events for all hardware devices, thus ensuring that any device
        // nodes have been created successfully before proceeding.
	argsSettle := []string{"settle", "--timeout=1"}
	cmdSettle := executor.Command("udevadm", argsSettle...)
	_, errSettle := cmdSettle.CombinedOutput()
	if errSettle != nil {
		glog.Errorf("error running udevadm settle %v\n", errSettle)
		return errSettle
	}

	args := []string{"trigger"}
	cmd := executor.Command("udevadm", args...)
	_, err := cmd.CombinedOutput()
	if err != nil {
		glog.Errorf("error running udevadm trigger %v\n", err)
		return err
	}
	glog.V(4).Infof("Successfully probed all attachments")
	return nil
}

func scsiHostRescan() {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := ioutil.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			ioutil.WriteFile(name, data, 0666)
		}
	}
}
<commit_msg>modify some wording<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

type CinderDiskUtil struct{}

// Attaches a disk specified by a volume.CinderPersistenDisk to the current kubelet.
// Mounts the disk to its global path.
func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeMounter, globalPDPath string) error {
	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}
	cloud, err := b.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	diskid, err := cloud.AttachDisk(instanceid, b.pdName)
	if err != nil {
		return err
	}

	var devicePath string
	numTries := 0
	for {
		devicePath = cloud.GetDevicePath(diskid)
		probeAttachedVolume()

		_, err := os.Stat(devicePath)
		if err == nil {
			break
		}
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		numTries++
		if numTries == 10 {
			return errors.New("Could not attach disk: Timeout after 60s")
		}
		time.Sleep(time.Second * 6)
	}
	notmnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(globalPDPath, 0750); err != nil {
				return err
			}
			notmnt = true
		} else {
			return err
		}
	}
	if notmnt {
		err = b.blockDeviceMounter.FormatAndMount(devicePath, globalPDPath, b.fsType, options)
		if err != nil {
			os.Remove(globalPDPath)
			return err
		}
		glog.V(2).Infof("Safe mount successful: %q\n", devicePath)
	}
	return nil
}

// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error {
	globalPDPath := makeGlobalPDName(cd.plugin.host, cd.pdName)
	if err := cd.mounter.Unmount(globalPDPath); err != nil {
		return err
	}
	if err := os.Remove(globalPDPath); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath)

	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	if err = cloud.DetachDisk(instanceid, cd.pdName); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName)
	return nil
}

func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error {
	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(cd.pdName); err != nil {
		// OpenStack cloud provider returns volume.tryAgainError when necessary,
		// no handling needed here.
		glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName)
	return nil
}

func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) {
	// TODO: caching, currently it is overkill because it calls this function
	// only when it creates dynamic PV
	zones := make(sets.String)
	nodes, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		glog.V(2).Infof("Error listing nodes")
		return zones, err
	}
	for _, node := range nodes.Items {
		if zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]; ok {
			zones.Insert(zone)
		}
	}
	glog.V(4).Infof("zones found: %v", zones)
	return zones, nil
}

func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, volumeLabels map[string]string, fstype string, err error) {
	cloud, err := c.plugin.getCloudProvider()
	if err != nil {
		return "", 0, nil, "", err
	}

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	volSizeBytes := capacity.Value()
	// Cinder works with gigabytes, convert to GiB with rounding up
	volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024))
	name := volume.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // Cinder volume name can have up to 255 characters
	vtype := ""
	availability := ""
	// Apply ProvisionerParameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			vtype = v
		case "availability":
			availability = v
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}
	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Cinder")
	}

	if availability == "" {
		// No zone specified, choose one randomly in the same region
		zones, err := getZonesFromNodes(c.plugin.host.GetKubeClient())
		if err != nil {
			glog.V(2).Infof("error getting zone information: %v", err)
			return "", 0, nil, "", err
		}
		// if we did not get any zones, lets leave it blank and gophercloud will
		// use zone "nova" as default
		if len(zones) > 0 {
			availability = volume.ChooseZoneForVolume(zones, c.options.PVC.Name)
		}
	}

	volumeID, volumeAZ, IgnoreVolumeAZ, errr := cloud.CreateVolume(name, volSizeGB, vtype, availability, c.options.CloudTags)
	if errr != nil {
		glog.V(2).Infof("Error creating cinder volume: %v", errr)
		return "", 0, nil, "", errr
	}
	glog.V(2).Infof("Successfully created cinder volume %s", volumeID)

	// these are needed that pod is spawning to same AZ
	volumeLabels = make(map[string]string)
	if IgnoreVolumeAZ == false {
		volumeLabels[kubeletapis.LabelZoneFailureDomain] = volumeAZ
	}
	return volumeID, volSizeGB, volumeLabels, fstype, nil
}

func probeAttachedVolume() error {
	// rescan scsi bus
	scsiHostRescan()

	executor := exec.New()

	// udevadm settle waits for udevd to process the device creation
	// events for all hardware devices, thus ensuring that any device
	// nodes have been created successfully before proceeding.
	argsSettle := []string{"settle", "--timeout=1"}
	cmdSettle := executor.Command("udevadm", argsSettle...)
	_, errSettle := cmdSettle.CombinedOutput()
	if errSettle != nil {
		glog.Errorf("error running udevadm settle %v\n", errSettle)
		return errSettle
	}

	args := []string{"trigger"}
	cmd := executor.Command("udevadm", args...)
	_, err := cmd.CombinedOutput()
	if err != nil {
		glog.Errorf("error running udevadm trigger %v\n", err)
		return err
	}
	glog.V(4).Infof("Successfully probed all attachments")
	return nil
}

func scsiHostRescan() {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := ioutil.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			ioutil.WriteFile(name, data, 0666)
		}
	}
}
<|endoftext|>