<commit_before>package driver

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/docker/docker/pkg/jsonmessage"
	units "github.com/docker/go-units"
)

const (
	// dockerPullActivityDeadline is the default value set in the imageProgressManager
	// when newImageProgressManager is called
	dockerPullActivityDeadline = 2 * time.Minute

	// dockerImageProgressReportInterval is the default value set in the
	// imageProgressManager when newImageProgressManager is called
	dockerImageProgressReportInterval = 10 * time.Second

	// dockerImageSlowProgressReportInterval is the default value set in the
	// imageProgressManager when newImageProgressManager is called
	dockerImageSlowProgressReportInterval = 2 * time.Minute
)

// layerProgress tracks the state and downloaded bytes of a single layer within
// a docker image
type layerProgress struct {
	id           string
	status       layerProgressStatus
	currentBytes int64
	totalBytes   int64
}

type layerProgressStatus int

const (
	layerProgressStatusUnknown layerProgressStatus = iota
	layerProgressStatusStarting
	layerProgressStatusWaiting
	layerProgressStatusDownloading
	layerProgressStatusVerifying
	layerProgressStatusDownloaded
	layerProgressStatusExtracting
	layerProgressStatusComplete
	layerProgressStatusExists
)

func lpsFromString(status string) layerProgressStatus {
	switch status {
	case "Pulling fs layer":
		return layerProgressStatusStarting
	case "Waiting":
		return layerProgressStatusWaiting
	case "Downloading":
		return layerProgressStatusDownloading
	case "Verifying Checksum":
		return layerProgressStatusVerifying
	case "Download complete":
		return layerProgressStatusDownloaded
	case "Extracting":
		return layerProgressStatusExtracting
	case "Pull complete":
		return layerProgressStatusComplete
	case "Already exists":
		return layerProgressStatusExists
	default:
		return layerProgressStatusUnknown
	}
}

// imageProgress tracks the status of each child layer as its pulled from a
// docker image repo
type imageProgress struct {
	sync.RWMutex
	lastMessage *jsonmessage.JSONMessage
	timestamp   time.Time
	layers      map[string]*layerProgress
	pullStart   time.Time
}

// get returns a status message and the timestamp of the last status update
func (p *imageProgress) get() (string, time.Time) {
	p.RLock()
	defer p.RUnlock()

	if p.lastMessage == nil {
		return "No progress", p.timestamp
	}

	var pulled, pulling, waiting int
	for _, l := range p.layers {
		switch {
		case l.status == layerProgressStatusStarting ||
			l.status == layerProgressStatusWaiting:
			waiting++
		case l.status == layerProgressStatusDownloading ||
			l.status == layerProgressStatusVerifying:
			pulling++
		case l.status >= layerProgressStatusDownloaded:
			pulled++
		}
	}

	elapsed := time.Now().Sub(p.pullStart)
	cur := p.currentBytes()
	total := p.totalBytes()
	var est int64
	if cur != 0 {
		est = (elapsed.Nanoseconds() / cur * total) - elapsed.Nanoseconds()
	}

	return fmt.Sprintf("Pulled %d/%d (%s/%s) layers: %d waiting/%d pulling - est %.1fs remaining",
		pulled, len(p.layers), units.BytesSize(float64(cur)), units.BytesSize(float64(total)),
		waiting, pulling, time.Duration(est).Seconds()), p.timestamp
}

// set takes a status message received from the docker engine api during an image
// pull and updates the status of the corresponding layer
func (p *imageProgress) set(msg *jsonmessage.JSONMessage) {
	p.Lock()
	defer p.Unlock()

	p.lastMessage = msg
	p.timestamp = time.Now()

	lps := lpsFromString(msg.Status)
	if lps == layerProgressStatusUnknown {
		return
	}

	layer, ok := p.layers[msg.ID]
	if !ok {
		layer = &layerProgress{id: msg.ID}
		p.layers[msg.ID] = layer
	}
	layer.status = lps
	if msg.Progress != nil && lps == layerProgressStatusDownloading {
		layer.currentBytes = msg.Progress.Current
		layer.totalBytes = msg.Progress.Total
	} else if lps == layerProgressStatusDownloaded {
		layer.currentBytes = layer.totalBytes
	}
}

// currentBytes iterates through all image layers and sums the total of
// current bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) currentBytes() int64 {
	var b int64
	for _, l := range p.layers {
		b += l.currentBytes
	}
	return b
}

// totalBytes iterates through all image layers and sums the total of
// total bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) totalBytes() int64 {
	var b int64
	for _, l := range p.layers {
		b += l.totalBytes
	}
	return b
}

// progressReporterFunc defines the method for handling inactivity and report
// events from the imageProgressManager. The image name, current status message
// and timestamp of last received status update are passed in.
type progressReporterFunc func(image string, msg string, timestamp time.Time)

// imageProgressManager tracks the progress of pulling a docker image from an
// image repository.
// It also implemented the io.Writer interface so as to be passed to the docker
// client pull image method in order to receive status updates from the docker
// engine api.
type imageProgressManager struct {
	imageProgress      *imageProgress
	image              string
	activityDeadline   time.Duration
	inactivityFunc     progressReporterFunc
	reportInterval     time.Duration
	reporter           progressReporterFunc
	slowReportInterval time.Duration
	slowReporter       progressReporterFunc
	lastSlowReport     time.Time
	cancel             context.CancelFunc
	stopCh             chan struct{}
	buf                bytes.Buffer
}

func newImageProgressManager(
	image string, cancel context.CancelFunc,
	inactivityFunc, reporter, slowReporter progressReporterFunc) *imageProgressManager {

	pm := &imageProgressManager{
		image:              image,
		activityDeadline:   dockerPullActivityDeadline,
		inactivityFunc:     inactivityFunc,
		reportInterval:     dockerImageProgressReportInterval,
		reporter:           reporter,
		slowReportInterval: dockerImageSlowProgressReportInterval,
		slowReporter:       slowReporter,
		imageProgress: &imageProgress{
			timestamp: time.Now(),
			layers:    make(map[string]*layerProgress),
		},
		cancel: cancel,
		stopCh: make(chan struct{}),
	}

	pm.start()
	return pm
}

// start intiates the ticker to trigger the inactivity and reporter handlers
func (pm *imageProgressManager) start() {
	now := time.Now()
	pm.imageProgress.pullStart = now
	pm.lastSlowReport = now
	go func() {
		ticker := time.NewTicker(dockerImageProgressReportInterval)
		for {
			select {
			case <-ticker.C:
				msg, lastStatusTime := pm.imageProgress.get()
				t := time.Now()
				if t.Sub(lastStatusTime) > pm.activityDeadline {
					pm.inactivityFunc(pm.image, msg, lastStatusTime)
					pm.cancel()
					return
				}
				if t.Sub(pm.lastSlowReport) > pm.slowReportInterval {
					pm.slowReporter(pm.image, msg, lastStatusTime)
					pm.lastSlowReport = t
				}
				pm.reporter(pm.image, msg, lastStatusTime)
			case <-pm.stopCh:
				return
			}
		}
	}()
}

func (pm *imageProgressManager) stop() {
	close(pm.stopCh)
}

func (pm *imageProgressManager) Write(p []byte) (n int, err error) {
	n, err = pm.buf.Write(p)
	var msg jsonmessage.JSONMessage

	for {
		line, err := pm.buf.ReadBytes('\n')
		if err == io.EOF {
			// Partial write of line; push back onto buffer and break until full line
			pm.buf.Write(line)
			break
		}
		if err != nil {
			return n, err
		}
		err = json.Unmarshal(line, &msg)
		if err != nil {
			return n, err
		}

		if msg.Error != nil {
			// error received from the docker engine api
			return n, msg.Error
		}

		pm.imageProgress.set(&msg)
	}

	return
}
<commit_msg>client/driver: drop docker pull progress estimate if its < 0<commit_after>package driver

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/docker/docker/pkg/jsonmessage"
	units "github.com/docker/go-units"
)

const (
	// dockerPullActivityDeadline is the default value set in the imageProgressManager
	// when newImageProgressManager is called
	dockerPullActivityDeadline = 2 * time.Minute

	// dockerImageProgressReportInterval is the default value set in the
	// imageProgressManager when newImageProgressManager is called
	dockerImageProgressReportInterval = 10 * time.Second

	// dockerImageSlowProgressReportInterval is the default value set in the
	// imageProgressManager when newImageProgressManager is called
	dockerImageSlowProgressReportInterval = 2 * time.Minute
)

// layerProgress tracks the state and downloaded bytes of a single layer within
// a docker image
type layerProgress struct {
	id           string
	status       layerProgressStatus
	currentBytes int64
	totalBytes   int64
}

type layerProgressStatus int

const (
	layerProgressStatusUnknown layerProgressStatus = iota
	layerProgressStatusStarting
	layerProgressStatusWaiting
	layerProgressStatusDownloading
	layerProgressStatusVerifying
	layerProgressStatusDownloaded
	layerProgressStatusExtracting
	layerProgressStatusComplete
	layerProgressStatusExists
)

func lpsFromString(status string) layerProgressStatus {
	switch status {
	case "Pulling fs layer":
		return layerProgressStatusStarting
	case "Waiting":
		return layerProgressStatusWaiting
	case "Downloading":
		return layerProgressStatusDownloading
	case "Verifying Checksum":
		return layerProgressStatusVerifying
	case "Download complete":
		return layerProgressStatusDownloaded
	case "Extracting":
		return layerProgressStatusExtracting
	case "Pull complete":
		return layerProgressStatusComplete
	case "Already exists":
		return layerProgressStatusExists
	default:
		return layerProgressStatusUnknown
	}
}

// imageProgress tracks the status of each child layer as its pulled from a
// docker image repo
type imageProgress struct {
	sync.RWMutex
	lastMessage *jsonmessage.JSONMessage
	timestamp   time.Time
	layers      map[string]*layerProgress
	pullStart   time.Time
}

// get returns a status message and the timestamp of the last status update
func (p *imageProgress) get() (string, time.Time) {
	p.RLock()
	defer p.RUnlock()

	if p.lastMessage == nil {
		return "No progress", p.timestamp
	}

	var pulled, pulling, waiting int
	for _, l := range p.layers {
		switch {
		case l.status == layerProgressStatusStarting ||
			l.status == layerProgressStatusWaiting:
			waiting++
		case l.status == layerProgressStatusDownloading ||
			l.status == layerProgressStatusVerifying:
			pulling++
		case l.status >= layerProgressStatusDownloaded:
			pulled++
		}
	}

	elapsed := time.Now().Sub(p.pullStart)
	cur := p.currentBytes()
	total := p.totalBytes()
	var est int64
	if cur != 0 {
		est = (elapsed.Nanoseconds() / cur * total) - elapsed.Nanoseconds()
	}

	var msg strings.Builder
	fmt.Fprintf(&msg, "Pulled %d/%d (%s/%s) layers: %d waiting/%d pulling",
		pulled, len(p.layers), units.BytesSize(float64(cur)), units.BytesSize(float64(total)),
		waiting, pulling)

	if est > 0 {
		fmt.Fprintf(&msg, " - est %.1fs remaining", time.Duration(est).Seconds())
	}
	return msg.String(), p.timestamp
}

// set takes a status message received from the docker engine api during an image
// pull and updates the status of the corresponding layer
func (p *imageProgress) set(msg *jsonmessage.JSONMessage) {
	p.Lock()
	defer p.Unlock()

	p.lastMessage = msg
	p.timestamp = time.Now()

	lps := lpsFromString(msg.Status)
	if lps == layerProgressStatusUnknown {
		return
	}

	layer, ok := p.layers[msg.ID]
	if !ok {
		layer = &layerProgress{id: msg.ID}
		p.layers[msg.ID] = layer
	}
	layer.status = lps
	if msg.Progress != nil && lps == layerProgressStatusDownloading {
		layer.currentBytes = msg.Progress.Current
		layer.totalBytes = msg.Progress.Total
	} else if lps == layerProgressStatusDownloaded {
		layer.currentBytes = layer.totalBytes
	}
}

// currentBytes iterates through all image layers and sums the total of
// current bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) currentBytes() int64 {
	var b int64
	for _, l := range p.layers {
		b += l.currentBytes
	}
	return b
}

// totalBytes iterates through all image layers and sums the total of
// total bytes. The caller is responsible for acquiring a read lock on the
// imageProgress struct
func (p *imageProgress) totalBytes() int64 {
	var b int64
	for _, l := range p.layers {
		b += l.totalBytes
	}
	return b
}

// progressReporterFunc defines the method for handling inactivity and report
// events from the imageProgressManager. The image name, current status message
// and timestamp of last received status update are passed in.
type progressReporterFunc func(image string, msg string, timestamp time.Time)

// imageProgressManager tracks the progress of pulling a docker image from an
// image repository.
// It also implemented the io.Writer interface so as to be passed to the docker
// client pull image method in order to receive status updates from the docker
// engine api.
type imageProgressManager struct {
	imageProgress      *imageProgress
	image              string
	activityDeadline   time.Duration
	inactivityFunc     progressReporterFunc
	reportInterval     time.Duration
	reporter           progressReporterFunc
	slowReportInterval time.Duration
	slowReporter       progressReporterFunc
	lastSlowReport     time.Time
	cancel             context.CancelFunc
	stopCh             chan struct{}
	buf                bytes.Buffer
}

func newImageProgressManager(
	image string, cancel context.CancelFunc,
	inactivityFunc, reporter, slowReporter progressReporterFunc) *imageProgressManager {

	pm := &imageProgressManager{
		image:              image,
		activityDeadline:   dockerPullActivityDeadline,
		inactivityFunc:     inactivityFunc,
		reportInterval:     dockerImageProgressReportInterval,
		reporter:           reporter,
		slowReportInterval: dockerImageSlowProgressReportInterval,
		slowReporter:       slowReporter,
		imageProgress: &imageProgress{
			timestamp: time.Now(),
			layers:    make(map[string]*layerProgress),
		},
		cancel: cancel,
		stopCh: make(chan struct{}),
	}

	pm.start()
	return pm
}

// start intiates the ticker to trigger the inactivity and reporter handlers
func (pm *imageProgressManager) start() {
	now := time.Now()
	pm.imageProgress.pullStart = now
	pm.lastSlowReport = now
	go func() {
		ticker := time.NewTicker(dockerImageProgressReportInterval)
		for {
			select {
			case <-ticker.C:
				msg, lastStatusTime := pm.imageProgress.get()
				t := time.Now()
				if t.Sub(lastStatusTime) > pm.activityDeadline {
					pm.inactivityFunc(pm.image, msg, lastStatusTime)
					pm.cancel()
					return
				}
				if t.Sub(pm.lastSlowReport) > pm.slowReportInterval {
					pm.slowReporter(pm.image, msg, lastStatusTime)
					pm.lastSlowReport = t
				}
				pm.reporter(pm.image, msg, lastStatusTime)
			case <-pm.stopCh:
				return
			}
		}
	}()
}

func (pm *imageProgressManager) stop() {
	close(pm.stopCh)
}

func (pm *imageProgressManager) Write(p []byte) (n int, err error) {
	n, err = pm.buf.Write(p)
	var msg jsonmessage.JSONMessage

	for {
		line, err := pm.buf.ReadBytes('\n')
		if err == io.EOF {
			// Partial write of line; push back onto buffer and break until full line
			pm.buf.Write(line)
			break
		}
		if err != nil {
			return n, err
		}
		err = json.Unmarshal(line, &msg)
		if err != nil {
			return n, err
		}

		if msg.Error != nil {
			// error received from the docker engine api
			return n, msg.Error
		}

		pm.imageProgress.set(&msg)
	}

	return
}
<|endoftext|>


<commit_before>package measurement

import (
	"github.com/Sirupsen/logrus"
	"regexp"
	"strconv"
	"strings"
)

// two byte-oriented functions identical except for operator comparing c to 127.
func stripCtlFromBytes(str string) string {
	b := make([]byte, len(str))
	var bl int
	for i := 0; i < len(str); i++ {
		c := str[i]
		if c >= 32 && c != 127 {
			b[bl] = c
			bl++
		}
	}
	return string(b[:bl])
}

func stripCtlAndExtFromBytes(str string) string {
	b := make([]byte, len(str))
	var bl int
	for i := 0; i < len(str); i++ {
		c := str[i]
		if c >= 32 && c < 127 {
			b[bl] = c
			bl++
		}
	}
	return string(b[:bl])
}

func formatDec2ASCII(input string) string {
	sArray := strings.Split(input, ".")
	n := len(sArray)
	bArray := make([]byte, n)
	for i := 0; i < n; i++ {
		num, _ := strconv.Atoi(sArray[i])
		//fmt.Printf("num %d\n",num)
		bArray = append(bArray, byte(num))
	}
	return stripCtlAndExtFromBytes(string(bArray))
}

func formatReGexp(l *logrus.Logger, input string, pattern string, replace string) string {
	re, err := regexp.Compile(pattern)
	if err != nil {
		l.Errorf("FormatReGexp  Input[%s] Pattern [%s] Error [%s] ", input, pattern, err)
		return ""
	}
	match := re.FindStringSubmatch(input)
	l.Debugf("FormatReGexp Input[%s] Pattern [%s] Match %+v", input, pattern, match)

	final := replace
	for i := 1; i < len(match); i++ {
		final = strings.Replace(final, "\\"+strconv.Itoa(i), match[i], -1)
	}
	return final
}

func sectionDotSlice(input string, first int, last int) string {
	sArray := strings.Split(input, ".")
	n := len(sArray)

	if last == -1 {
		last = n - 1
	}
	if last < first {
		return input
	}

	if first > n {
		return input
	}

	if last < n {
		n = last
	}
	output := sArray[first]
	for i := first + 1; i <= n; i++ {
		output = output + "." + sArray[i]
	}
	return output
}

func formatTag(l *logrus.Logger, format string, data map[string]string, def string) string {
	if len(format) == 0 {
		return data[def]
	}

	final := format
	for k, v := range data {
		final = strings.Replace(final, "$"+k, v, -1)
	}
	//check if more varibles defined
	if !strings.Contains(final, "$") {
		return final
	}
	//continue evaluating for each variable
	for k, v := range data {
		//check as many times as repeaded variables it has
		for {
			// ${VARNAME|SECTION|TRANSFORMATION}
			pattern := "\\${" + k + "\\|([^|]*)\\|([^}]*)}"
			re, err := regexp.Compile(pattern)
			if err != nil {
				l.Debugf("FormatTag[%s]: Regex ${VAR|XXX} Error %s with pattern %s", format, err, pattern)
				continue
			}
			match := re.FindStringSubmatch(final)
			if len(match) < 3 {
				l.Debugf("FormatTag[%s]: match length: %d , match  %+v with pattern %s", format, len(match), match, pattern)
				break
				//continue
			}
			//here we hav found a tranformation to do
			sectionmode := match[1]
			transformation := match[2]
			//check defaultvalues
			if len(sectionmode) == 0 {
				sectionmode = "ALL"
			}
			if len(transformation) == 0 {
				transformation = "STRING"
			}
			// Getting Variable Section
			section := v
			switch {
			case sectionmode == "ALL":
				section = v
			case strings.HasPrefix(sectionmode, "DOT["):
				re2, err := regexp.Compile("DOT\\[(\\d*):(\\d*)\\]")
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT - Regex ERROR %s ", format, err)
					break
				}
				match2 := re2.FindStringSubmatch(sectionmode)
				if len(match2) < 3 {
					l.Warnf("FormatTag[%s]: DOT - ERROR on number or paramters %+v  for string %s", format, match2, sectionmode)
					break
				}

				dotInit := match2[1]
				dotEnd := match2[2]

				first, err := strconv.Atoi(dotInit) // if there is an error first will be 0
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT -  error decode first position [%s]   error  %s", format, dotInit, err)
				}
				last, err := strconv.Atoi(dotEnd) // if there is an error last
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT -  error decode last position [%s]   error  %s", format, dotEnd, err)
					last = -1
				}
				section = sectionDotSlice(v, first, last)
				l.Debugf("FormatTag[%s]: final section first/last[%d/%d] from [%s] took [%s] ", format, first, last, v, section)

			case strings.HasPrefix(sectionmode, "REGEX/"):
				re2, err := regexp.Compile("REGEX/(.*)/(.*)/")
				if err != nil {
					l.Warnf("FormatTag[%s]: REGEX - Regex ERROR %s ", format, err)
					break
				}
				match2 := re2.FindStringSubmatch(sectionmode)
				if len(match2) < 3 {
					l.Warnf("FormatTag[%s]: REGEX - ERROR on number or paramters %+v  for string %s", format, match2, sectionmode)
					break
				}
				userRegex := match2[1]
				userSubst := match2[2]
				section = formatReGexp(l, v, userRegex, userSubst)
				l.Debugf("FormatTag[%s]: final section REGEX/%s/%s/ from [%s] took [%s] ", format, userRegex, userSubst, v, section)

			default:
				l.Warnf("FormatTag[%s]: Unknown SECTION parameters %s ,  pattern %s", format, sectionmode, pattern)
			}

			//here we have the section we want to decode
			//Doing transfomations over de selected section
			decoded := section
			switch {
			case transformation == "STRING":
				decoded = section
			case strings.HasPrefix(transformation, "DEC2ASCII"):
				decoded = formatDec2ASCII(section)
				l.Debugf("FormatTag[%s]: DEC2ASCII : value %s : Decoded %s", format, v, decoded)
			default:
				l.Warnf("FormatTag[%s]: Unknown TRASNFORMATION parameters %s ,  pattern %s", format, transformation, pattern)
			}
			final = strings.Replace(final, match[0], decoded, -1)
			l.Debugf("Result After Iteration on var Instance [%s]-[%s]", k, final)
		}
		l.Debugf("Result After Iteration on var [%s]-[%s]", k, final)
	}
	l.Debugf("FormatTag[%s]: OK Final result: %s", format, final)
	return stripCtlAndExtFromBytes(final)
}
<commit_msg>fix #243<commit_after>package measurement

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"regexp"
	"strconv"
	"strings"
)

// two byte-oriented functions identical except for operator comparing c to 127.
func stripCtlFromBytes(str string) string {
	b := make([]byte, len(str))
	var bl int
	for i := 0; i < len(str); i++ {
		c := str[i]
		if c >= 32 && c != 127 {
			b[bl] = c
			bl++
		}
	}
	return string(b[:bl])
}

func stripCtlAndExtFromBytes(str string) string {
	b := make([]byte, len(str))
	var bl int
	for i := 0; i < len(str); i++ {
		c := str[i]
		if c >= 32 && c < 127 {
			b[bl] = c
			bl++
		}
	}
	return string(b[:bl])
}

func formatDec2ASCII(input string) string {
	sArray := strings.Split(input, ".")
	n := len(sArray)
	bArray := make([]byte, n)
	for i := 0; i < n; i++ {
		num, _ := strconv.Atoi(sArray[i])
		//fmt.Printf("num %d\n",num)
		bArray = append(bArray, byte(num))
	}
	return stripCtlAndExtFromBytes(string(bArray))
}

func formatReGexp(l *logrus.Logger, input string, pattern string, replace string) string {
	re, err := regexp.Compile(pattern)
	if err != nil {
		l.Errorf("FormatReGexp  Input[%s] Pattern [%s] Error [%s] ", input, pattern, err)
		return ""
	}
	match := re.FindStringSubmatch(input)
	l.Debugf("FormatReGexp Input[%s] Pattern [%s] Match %+v", input, pattern, match)

	final := replace
	for i := 1; i < len(match); i++ {
		final = strings.Replace(final, "\\"+strconv.Itoa(i), match[i], -1)
	}
	return final
}

func sectionDotSlice(input string, first int, last int) (string, error) {
	sArray := strings.Split(input, ".")
	n := len(sArray)

	if last == -1 {
		last = n - 1
	}
	if last < first {
		return input, fmt.Errorf("Last index (%d) should be greater than first(%d)", last, first)
	}

	if first >= n {
		return input, fmt.Errorf("First index (%d) should be lower than total num of doted strings (%d)", first, n)
	}

	var err error = nil

	if last >= n {
		err = fmt.Errorf("Last index (%d), shoud be lower than total number of doted separated strings (%d)", last, n)
		last = n - 1
	}

	output := sArray[first]
	for i := first + 1; i <= last; i++ {
		output = output + "." + sArray[i]
	}
	return output, err
}

func formatTag(l *logrus.Logger, format string, data map[string]string, def string) string {
	if len(format) == 0 {
		return data[def]
	}

	final := format
	for k, v := range data {
		final = strings.Replace(final, "$"+k, v, -1)
	}
	//check if more varibles defined
	if !strings.Contains(final, "$") {
		return final
	}
	//continue evaluating for each variable
	for k, v := range data {
		//check as many times as repeaded variables it has
		for {
			// ${VARNAME|SECTION|TRANSFORMATION}
			pattern := "\\${" + k + "\\|([^|]*)\\|([^}]*)}"
			re, err := regexp.Compile(pattern)
			if err != nil {
				l.Debugf("FormatTag[%s]: Regex ${VAR|XXX} Error %s with pattern %s", format, err, pattern)
				continue
			}
			match := re.FindStringSubmatch(final)
			if len(match) < 3 {
				l.Debugf("FormatTag[%s]: match length: %d , match  %+v with pattern %s", format, len(match), match, pattern)
				break
				//continue
			}
			//here we hav found a tranformation to do
			sectionmode := match[1]
			transformation := match[2]
			//check defaultvalues
			if len(sectionmode) == 0 {
				sectionmode = "ALL"
			}
			if len(transformation) == 0 {
				transformation = "STRING"
			}
			// Getting Variable Section
			section := v
			switch {
			case sectionmode == "ALL":
				section = v
			case strings.HasPrefix(sectionmode, "DOT["):
				re2, err := regexp.Compile("DOT\\[(\\d*):(\\d*)\\]")
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT - Regex ERROR %s ", format, err)
					break
				}
				match2 := re2.FindStringSubmatch(sectionmode)
				if len(match2) < 3 {
					l.Warnf("FormatTag[%s]: DOT - ERROR on number or paramters %+v  for string %s", format, match2, sectionmode)
					break
				}

				dotInit := match2[1]
				dotEnd := match2[2]

				first, err := strconv.Atoi(dotInit) // if there is an error first will be 0
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT -  error decode first position [%s]   error  %s", format, dotInit, err)
				}
				last, err := strconv.Atoi(dotEnd) // if there is an error last
				if err != nil {
					l.Warnf("FormatTag[%s]: DOT -  error decode last position [%s]   error  %s", format, dotEnd, err)
					last = -1
				}

				section, err = sectionDotSlice(v, first, last)
				if err != nil {
					l.Warnf("SectionDotSlice Error: %s", err)
				}
				l.Debugf("FormatTag[%s]: final section first/last[%d/%d] from [%s] took [%s] ", format, first, last, v, section)

			case strings.HasPrefix(sectionmode, "REGEX/"):
				re2, err := regexp.Compile("REGEX/(.*)/(.*)/")
				if err != nil {
					l.Warnf("FormatTag[%s]: REGEX - Regex ERROR %s ", format, err)
					break
				}
				match2 := re2.FindStringSubmatch(sectionmode)
				if len(match2) < 3 {
					l.Warnf("FormatTag[%s]: REGEX - ERROR on number or paramters %+v  for string %s", format, match2, sectionmode)
					break
				}
				userRegex := match2[1]
				userSubst := match2[2]
				section = formatReGexp(l, v, userRegex, userSubst)
				l.Debugf("FormatTag[%s]: final section REGEX/%s/%s/ from [%s] took [%s] ", format, userRegex, userSubst, v, section)

			default:
				l.Warnf("FormatTag[%s]: Unknown SECTION parameters %s ,  pattern %s", format, sectionmode, pattern)
			}

			//here we have the section we want to decode
			//Doing transfomations over de selected section
			decoded := section
			switch {
			case transformation == "STRING":
				decoded = section
			case strings.HasPrefix(transformation, "DEC2ASCII"):
				decoded = formatDec2ASCII(section)
				l.Debugf("FormatTag[%s]: DEC2ASCII : value %s : Decoded %s", format, v, decoded)
			default:
				l.Warnf("FormatTag[%s]: Unknown TRASNFORMATION parameters %s ,  pattern %s", format, transformation, pattern)
			}
			final = strings.Replace(final, match[0], decoded, -1)
			l.Debugf("Result After Iteration on var Instance [%s]-[%s]", k, final)
		}
		l.Debugf("Result After Iteration on var [%s]-[%s]", k, final)
	}
	l.Debugf("FormatTag[%s]: OK Final result: %s", format, final)
	return stripCtlAndExtFromBytes(final)
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/store"
	"sync"
)

const PolicyName = "policy"

type PolicyController interface {
	GetPolicy(object.Generation) (*language.Policy, error)
	GetPolicyData(object.Generation) (*PolicyData, error)
	GetPolicyFromData(policyData *PolicyData) (*language.Policy, error)
	UpdatePolicy([]object.Base) (bool, *PolicyData, error)
}

func NewPolicyController(store store.ObjectStore) PolicyController {
	return &PolicyControllerImpl{sync.Mutex{}, store}
}

var PolicyDataObject = &object.Info{
	Kind:        "policy",
	Versioned:   true,
	Constructor: func() object.Base { return &PolicyData{} },
}

type PolicyData struct {
	language.Metadata

	Objects map[string]map[string]object.Generation // kind -> name -> generation
}

func (p *PolicyData) Add(obj object.Base) {
	byKind, exist := p.Objects[obj.GetKind()]
	if !exist {
		byKind = make(map[string]object.Generation)
		p.Objects[obj.GetKind()] = byKind
	}
	byKind[obj.GetName()] = obj.GetGeneration()
}

type PolicyControllerImpl struct {
	update sync.Mutex
	store  store.ObjectStore
}

func (ctl *PolicyControllerImpl) GetPolicyData(gen object.Generation) (*PolicyData, error) {
	dataObj, err := ctl.store.GetByName(object.SystemNS, PolicyDataObject.Kind, PolicyName, gen)
	if err != nil {
		return nil, err
	}
	if dataObj == nil {
		return nil, nil
	}
	data, ok := dataObj.(*PolicyData)
	if !ok {
		return nil, fmt.Errorf("Unexpected type while getting PolicyData from DB")
	}
	return data, nil
}

func (ctl *PolicyControllerImpl) GetPolicyFromData(policyData *PolicyData) (*language.Policy, error) {
	policy := language.NewPolicy()

	// in case of first version of policy, we just need to have empty policy
	if policyData != nil && policyData.Objects != nil {
		for kind, nameGen := range policyData.Objects {
			for name, gen := range nameGen {
				obj, err := ctl.store.GetByName(object.SystemNS, kind, name, gen)
				if err != nil {
					return nil, err
				}
				policy.AddObject(obj)
			}
		}
	}
	return policy, nil
}

func (ctl *PolicyControllerImpl) GetPolicy(policyGen object.Generation) (*language.Policy, error) {
	// todo should we use RWMutex for get/update policy?
	policyData, err := ctl.GetPolicyData(policyGen)
	if err != nil {
		return nil, err
	}
	return ctl.GetPolicyFromData(policyData)
}

func (ctl *PolicyControllerImpl) UpdatePolicy(updatedObjects []object.Base) (bool, *PolicyData, error) {
	// we should process only a single policy update request at once
	ctl.update.Lock()
	defer ctl.update.Unlock()

	policyData, err := ctl.GetPolicyData(object.LastGen)
	if err != nil {
		return false, nil, err
	}

	changed := false

	// it could happen only for the fist time
	if policyData == nil {
		policyData = &PolicyData{
			Metadata: language.Metadata{
				Namespace: object.SystemNS,
				Kind:      PolicyDataObject.Kind,
				Name:      "policy",
			},
			Objects: make(map[string]map[string]object.Generation),
		}
		changed = true
	}

	for _, updatedObj := range updatedObjects {
		updated, err := ctl.store.Save(updatedObj)
		if err != nil {
			return false, nil, err
		}
		if updated {
			policyData.Add(updatedObj)
			changed = true
		}
	}

	if changed {
		_, err = ctl.store.Save(policyData)
		if err != nil {
			return false, nil, err
		}
	}

	// policy, err := ctl.getPolicyFromData(policyData)

	return changed, policyData, err
}
<commit_msg>Fix policy data handling after adding support for namespaces<commit_after>package controller

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/store"
	"sync"
)

const PolicyName = "policy"

type PolicyController interface {
	GetPolicy(object.Generation) (*language.Policy, error)
	GetPolicyData(object.Generation) (*PolicyData, error)
	GetPolicyFromData(policyData *PolicyData) (*language.Policy, error)
	UpdatePolicy([]object.Base) (bool, *PolicyData, error)
}

func NewPolicyController(store store.ObjectStore) PolicyController {
	return &PolicyControllerImpl{sync.Mutex{}, store}
}

var PolicyDataObject = &object.Info{
	Kind:        "policy",
	Versioned:   true,
	Constructor: func() object.Base { return &PolicyData{} },
}

type PolicyData struct {
	language.Metadata

	Objects map[string]map[string]map[string]object.Generation // ns -> kind -> name -> generation
}

func (p *PolicyData) Add(obj object.Base) {
	byNs, exist := p.Objects[obj.GetNamespace()]
	if !exist {
		byNs = make(map[string]map[string]object.Generation)
		p.Objects[obj.GetNamespace()] = byNs
	}
	byKind, exist := byNs[obj.GetKind()]
	if !exist {
		byKind = make(map[string]object.Generation)
		byNs[obj.GetKind()] = byKind
	}
	byKind[obj.GetName()] = obj.GetGeneration()
}

type PolicyControllerImpl struct {
	update sync.Mutex
	store  store.ObjectStore
}

func (ctl *PolicyControllerImpl) GetPolicyData(gen object.Generation) (*PolicyData, error) {
	dataObj, err := ctl.store.GetByName(object.SystemNS, PolicyDataObject.Kind, PolicyName, gen)
	if err != nil {
		return nil, err
	}
	if dataObj == nil {
		return nil, nil
	}
	data, ok := dataObj.(*PolicyData)
	if !ok {
		return nil, fmt.Errorf("Unexpected type while getting PolicyData from DB")
	}
	return data, nil
}

func (ctl *PolicyControllerImpl) GetPolicyFromData(policyData *PolicyData) (*language.Policy, error) {
	policy := language.NewPolicy()

	// in case of first version of policy, we just need to have empty policy
	if policyData != nil && policyData.Objects != nil {
		for ns, kindNameGen := range policyData.Objects {
			for kind, nameGen := range kindNameGen {
				for name, gen := range nameGen {
					obj, err := ctl.store.GetByName(ns, kind, name, gen)
					if err != nil {
						return nil, err
					}
					policy.AddObject(obj)
				}
			}
		}
	}
	return policy, nil
}

func (ctl *PolicyControllerImpl) GetPolicy(policyGen object.Generation) (*language.Policy, error) {
	// todo should we use RWMutex for get/update policy?
	policyData, err := ctl.GetPolicyData(policyGen)
	if err != nil {
		return nil, err
	}
	return ctl.GetPolicyFromData(policyData)
}

func (ctl *PolicyControllerImpl) UpdatePolicy(updatedObjects []object.Base) (bool, *PolicyData, error) {
	// we should process only a single policy update request at once
	ctl.update.Lock()
	defer ctl.update.Unlock()

	policyData, err := ctl.GetPolicyData(object.LastGen)
	if err != nil {
		return false, nil, err
	}

	changed := false

	// it could happen only for the fist time
	if policyData == nil {
		policyData = &PolicyData{
			Metadata: language.Metadata{
				Namespace: object.SystemNS,
				Kind:      PolicyDataObject.Kind,
				Name:      "policy",
			},
			Objects: make(map[string]map[string]map[string]object.Generation),
		}
		changed = true
	}

	for _, updatedObj := range updatedObjects {
		updated, err := ctl.store.Save(updatedObj)
		if err != nil {
			return false, nil, err
		}
		if updated {
			policyData.Add(updatedObj)
			changed = true
		}
	}

	if changed {
		_, err = ctl.store.Save(policyData)
		if err != nil {
			return false, nil, err
		}
	}

	// policy, err := ctl.getPolicyFromData(policyData)

	return changed, policyData, err
}
<|endoftext|>


<commit_before>package template

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kapi "k8s.io/kubernetes/pkg/apis/core"
)

const (
	GroupName       = "template.openshift.io"
	LegacyGroupName = ""
)

// SchemeGroupVersion is group version used to register these objects
var (
	SchemeGroupVersion       = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal}
	LegacySchemeGroupVersion = schema.GroupVersion{Group: LegacyGroupName, Version: runtime.APIVersionInternal}

	LegacySchemeBuilder    = runtime.NewSchemeBuilder(addLegacyKnownTypes)
	AddToSchemeInCoreGroup = LegacySchemeBuilder.AddToScheme

	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}

// Resource takes an unqualified resource and returns back a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Template{},
		&TemplateList{},
		&TemplateInstance{},
		&TemplateInstanceList{},
		&BrokerTemplateInstance{},
		&BrokerTemplateInstanceList{},
		&kapi.List{},
	)
	return nil
}

func addLegacyKnownTypes(scheme *runtime.Scheme) error {
	types := []runtime.Object{
		&Template{},
		&TemplateList{},
	}
	scheme.AddKnownTypes(LegacySchemeGroupVersion, types...)
	scheme.AddKnownTypeWithName(LegacySchemeGroupVersion.WithKind("TemplateConfig"), &Template{})
	scheme.AddKnownTypeWithName(LegacySchemeGroupVersion.WithKind("ProcessedTemplate"), &Template{})
	return nil
}
<commit_msg>fix template type registration<commit_after>package template

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

const (
	GroupName       = "template.openshift.io"
	LegacyGroupName = ""
)

// SchemeGroupVersion is group version used to register these objects
var (
	SchemeGroupVersion       = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal}
	LegacySchemeGroupVersion = schema.GroupVersion{Group: LegacyGroupName, Version: runtime.APIVersionInternal}

	LegacySchemeBuilder    = runtime.NewSchemeBuilder(addLegacyKnownTypes)
	AddToSchemeInCoreGroup = LegacySchemeBuilder.AddToScheme

	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}

// Resource takes an unqualified resource and returns back a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Template{},
		&TemplateList{},
		&TemplateInstance{},
		&TemplateInstanceList{},
		&BrokerTemplateInstance{},
		&BrokerTemplateInstanceList{},
	)
	return nil
}

func addLegacyKnownTypes(scheme *runtime.Scheme) error {
	types := []runtime.Object{
		&Template{},
		&TemplateList{},
	}
	scheme.AddKnownTypes(LegacySchemeGroupVersion, types...)
	scheme.AddKnownTypeWithName(LegacySchemeGroupVersion.WithKind("TemplateConfig"), &Template{})
	scheme.AddKnownTypeWithName(LegacySchemeGroupVersion.WithKind("ProcessedTemplate"), &Template{})
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build 386 amd64 amd64p32

package cpu

const CacheLineSize = 64

// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)

// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)

func init() {
	maxId, _, _, _ := cpuid(0, 0)

	if maxId < 1 {
		return
	}

	_, _, ecx1, edx1 := cpuid(1, 0)
	X86.HasSSE2 = isSet(26, edx1)

	X86.HasSSE3 = isSet(0, ecx1)
	X86.HasPCLMULQDQ = isSet(1, ecx1)
	X86.HasSSSE3 = isSet(9, ecx1)
	X86.HasSSE41 = isSet(19, ecx1)
	X86.HasSSE42 = isSet(20, ecx1)
	X86.HasPOPCNT = isSet(23, ecx1)
	X86.HasAES = isSet(25, ecx1)
	X86.HasOSXSAVE = isSet(27, ecx1)

	osSupportsAVX := false
	// For XGETBV, OSXSAVE bit is required and sufficient.
	if X86.HasOSXSAVE {
		eax, _ := xgetbv()
		// Check if XMM and YMM registers have OS support.
		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
	}

	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX

	if maxId < 7 {
		return
	}

	_, ebx7, _, _ := cpuid(7, 0)
	X86.HasBMI1 = isSet(3, ebx7)
	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
	X86.HasBMI2 = isSet(8, ebx7)
	X86.HasERMS = isSet(9, ebx7)
}

func isSet(bitpos uint, value uint32) bool {
	return value&(1<<bitpos) != 0
}
<commit_msg>internal/cpu: fix style nit in variable name<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build 386 amd64 amd64p32

package cpu

const CacheLineSize = 64

// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)

// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)

func init() {
	maxID, _, _, _ := cpuid(0, 0)

	if maxID < 1 {
		return
	}

	_, _, ecx1, edx1 := cpuid(1, 0)
	X86.HasSSE2 = isSet(26, edx1)

	X86.HasSSE3 = isSet(0, ecx1)
	X86.HasPCLMULQDQ = isSet(1, ecx1)
	X86.HasSSSE3 = isSet(9, ecx1)
	X86.HasSSE41 = isSet(19, ecx1)
	X86.HasSSE42 = isSet(20, ecx1)
	X86.HasPOPCNT = isSet(23, ecx1)
	X86.HasAES = isSet(25, ecx1)
	X86.HasOSXSAVE = isSet(27, ecx1)

	osSupportsAVX := false
	// For XGETBV, OSXSAVE bit is required and sufficient.
	if X86.HasOSXSAVE {
		eax, _ := xgetbv()
		// Check if XMM and YMM registers have OS support.
		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
	}

	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX

	if maxID < 7 {
		return
	}

	_, ebx7, _, _ := cpuid(7, 0)
	X86.HasBMI1 = isSet(3, ebx7)
	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
	X86.HasBMI2 = isSet(8, ebx7)
	X86.HasERMS = isSet(9, ebx7)
}

func isSet(bitpos uint, value uint32) bool {
	return value&(1<<bitpos) != 0
}
<|endoftext|>


<commit_before>package subv

import (
	"fmt"
	"log"
	"os/exec"
	"strings"

	"github.com/skeswa/gophr/common"
	"github.com/skeswa/gophr/common/github"
	"github.com/skeswa/gophr/common/models"
	"github.com/skeswa/gophr/common/verdeps"
)

var (
	folderName              string
	gitHubRemoteOrigin      = "git@github.com:gophr-packages/%s.git"
	navigateToPackageFolder = "cd /tmp/%s"
)

var (
	initalizeRepo    = "cd /tmp && mkdir %s && cd %s && git init"
	createBranch     = "%s && git checkout -b %s"
	setRemoteCommand = "%s && git remote add origin %s"
	fetchRepoArchive = "%s && wget https://github.com/%s/%s/archive/%s.zip"
	unzipRepoArchive = "%s && unzip %s.zip && cd %s && mv * ../ && cd .. && rm %s.zip && rm -rf %s"
	addFiles         = "%s && git add . "
	commitFiles      = "%s && git commit -m \" %s \""
	pushFiles        = "%s && git push --set-upstream origin %s"
)

// SubVersionPackageModel creates a github repo for the packageModel on gophr-packages
// versioned a the speicifed ref
func SubVersionPackageModel(packageModel *models.PackageModel, ref string) {
	// If no ref is present or it's master we need the current Master SHA
	if len(ref) == 0 || ref == "master" {
		curretRef, err := common.FetchRefs(*packageModel.Author, *packageModel.Repo)
		if err != nil || len(curretRef.MasterRefHash) == 0 {
			// TODO: return error here
			return
		}
		ref = curretRef.MasterRefHash
	}
	// First check if this ref has already been versioned for this packageModel
	log.Println("Checking if ref has been versioned before")
	exists, err := github.CheckIfRefExists(*packageModel.Author, *packageModel.Repo, ref)
	if exists == true && err == nil {
		log.Println("That ref has already been versioned")
		return
	}

	if err != nil {
		log.Printf("Error occured in checking if ref exists %s", err)
	}

	log.Printf("%s/%s@%s has not been versioned yet",
		github.GitHubGophrPackageOrgName,
		github.BuildNewGitHubRepoName(*packageModel.Author, *packageModel.Repo),
		github.BuildGitHubBranch(ref),
	)
	// Set working folderName for package
	folderName = github.BuildNewGitHubRepoName(*packageModel.Author, *packageModel.Repo)

	// Instantiate New Github Request Service
	log.Println("Initializing gitHub component")

	gitHubRequestService := github.NewGitHubRequestService()

	log.Printf("Creating new Github repo for %s/%s at %s",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	err = gitHubRequestService.CreateNewGitHubRepo(*packageModel)
	log.Printf("%s", err)

	log.Printf("Initializing folder and initializing git repo for %s \n", folderName)
	err = initializeRepoCMD(packageModel)
	checkError(err, folderName)

	log.Printf("Creating branch %s \n", github.BuildGitHubBranch(ref))
	err = createBranchCMD(packageModel, ref)
	checkError(err, folderName)

	log.Printf("Setting remote branch url %s \n", github.BuildGitHubBranch(ref))
	err = setRemoteCMD(packageModel, ref)
	checkError(err, folderName)

	log.Printf("Fetching github archive for %s/%s with tag %s \n",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	err = fetchArchiveCMD(packageModel, ref)
	checkError(err, folderName)

	log.Printf("Fetching github archive for %s/%s with tag %s \n",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	err = unzipArchiveCMD(packageModel, ref)
	checkError(err, folderName)

	// TODO subverisoning
	// Create array of all sub-dependencies
	commitDate, err := gitHubRequestService.FetchCommitTimestamp(packageModel, ref)
	if err != nil {
		log.Fatalln(err)
	}
	err = verdeps.VersionDeps( // VersionDepsArgs is the arguments struct for VersionDeps(...).
		verdeps.VersionDepsArgs{
			// SHA is the sha of the package being versioned.
			SHA: ref,
			// SHA is the path to the package source code to be versioned.
			Path: fmt.Sprintf("/tmp/%s", folderName),
			// Date is date that the version of the package with that matches SHA was created.
			Date: commitDate,
			// Model is the package model of the package.
			Model: packageModel,
			// GithubServcie is the service, with which, requests can be made of the Github API.
			GithubService: gitHubRequestService,
		})
	if err != nil {
		log.Fatalln(err)
	}

	log.Println("Adding unarchived repo data to branch")
	err = addFilesCMD()
	checkError(err, folderName)

	log.Println("Commiting repo data to branch")
	err = commitFilesCMD(packageModel, ref)
	checkError(err, folderName)

	log.Printf("Pushing files to branch %s \n", github.BuildRemoteURL(packageModel, ref))
	err = pushFilesCMD(packageModel, ref)
	checkError(err, folderName)

	cleanUpExitHook(folderName)
}

func initializeRepoCMD(packageModel *models.PackageModel) error {
	log.Println("Initializing folder and repo commmand")
	cmd := fmt.Sprintf(initalizeRepo, folderName, folderName)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func createBranchCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Initializing folder and repo commmand")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(createBranch, navigateFolderCMD, github.BuildGitHubBranch(ref))
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func setRemoteCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Initializing folder and repo commmand")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	remoteURL := github.BuildRemoteURL(packageModel, ref)
	cmd := fmt.Sprintf(setRemoteCommand, navigateFolderCMD, remoteURL)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}
func fetchArchiveCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Fetching and Unzipping Archive for tag")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(fetchRepoArchive, navigateFolderCMD, *packageModel.Author, *packageModel.Repo, ref)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

func unzipArchiveCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Fetching and Unzipping Archive for tag")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	zipFolder := *packageModel.Repo + "-" + ref
	cmd := fmt.Sprintf(unzipRepoArchive, navigateFolderCMD, ref, zipFolder, ref, zipFolder)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

func addFilesCMD() error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(addFiles, navigateFolderCMD)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

// TODO add version
func commitFilesCMD(packageModel *models.PackageModel, ref string) error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	commitMessage := fmt.Sprintf("Gophr versioned repo of %s / %s @ %s", *packageModel.Author, *packageModel.Repo, ref)
	cmd := fmt.Sprintf(commitFiles, navigateFolderCMD, commitMessage)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

func pushFilesCMD(packageModel *models.PackageModel, ref string) error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(pushFiles, navigateFolderCMD, github.BuildGitHubBranch(ref))
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s 		\n", out)
	return err
}

// Helper functions

// Exit Hook to clean up files
func cleanUpExitHook(folderName string) {
	log.Printf("Exit hook initiated deleting folder %s \n", folderName)
	// TODO:(Shikkic)
	if strings.HasPrefix(folderName, "/") == true {
		cmd := fmt.Sprintf("cd /tmp && rm -rf %s", folderName)
		out, err := exec.Command("sh", "-c", cmd).Output()
		if err != nil {
			log.Println("Could not properly engage exit hook")
			log.Fatalln(err)
		}
		log.Printf("Output: %s", out)
	} else {
		log.Println("Cowardly refusing to initiate exit hook. Will not rm -rf folder names that contains any leading '/'")
	}
}

// Check Error and Engage Exit Hook if fatal error occured
func checkError(err error, folderName string) {
	if err != nil {
		cleanUpExitHook(folderName)
		log.Fatalf("Error occured %s \n", err)
	}
}
<commit_msg>sub versioner now returns errors<commit_after>package subv

import (
	"fmt"
	"log"
	"os/exec"
	"strings"

	"github.com/skeswa/gophr/common"
	"github.com/skeswa/gophr/common/github"
	"github.com/skeswa/gophr/common/models"
	"github.com/skeswa/gophr/common/verdeps"
)

var (
	folderName              string
	gitHubRemoteOrigin      = "git@github.com:gophr-packages/%s.git"
	navigateToPackageFolder = "cd /tmp/%s"
)

var (
	initalizeRepo    = "cd /tmp && mkdir %s && cd %s && git init"
	createBranch     = "%s && git checkout -b %s"
	setRemoteCommand = "%s && git remote add origin %s"
	fetchRepoArchive = "%s && wget https://github.com/%s/%s/archive/%s.zip"
	unzipRepoArchive = "%s && unzip %s.zip && cd %s && mv * ../ && cd .. && rm %s.zip && rm -rf %s"
	addFiles         = "%s && git add . "
	commitFiles      = "%s && git commit -m \" %s \""
	pushFiles        = "%s && git push --set-upstream origin %s"
)

// SubVersionPackageModel creates a github repo for the packageModel on github.com/gophr/gophr-packages
// versioned a the speicifed ref
func SubVersionPackageModel(packageModel *models.PackageModel, ref string) error {
	log.Printf("Preparing to sub-version %s/%s@%s \n", *packageModel.Author, *packageModel.Repo, ref)
	// If the given ref is empty or refers to 'master' then we need to grab the current master SHA
	if len(ref) == 0 || ref == "master" {
		log.Println("Ref is empty or is 'master', fetching current master SHA")
		curretRef, err := common.FetchRefs(*packageModel.Author, *packageModel.Repo)
		if err != nil || len(curretRef.MasterRefHash) == 0 {
			return fmt.Errorf(
				"Error could not retrieve master ref of %s/%s, or packageModel does not exist",
				*packageModel.Author,
				*packageModel.Repo,
			)
		}
		ref = curretRef.MasterRefHash
	}

	// First check if this ref has already been versioned for this packageModel
	log.Printf("Checking if ref %s has been versioned before \n", ref)
	exists, err := github.CheckIfRefExists(*packageModel.Author, *packageModel.Repo, ref)
	if exists == true && err == nil {
		log.Println("That ref has already been versioned")
		return nil
	}
	if err != nil {
		return fmt.Errorf("Error occured in checking if ref exists. %s", err)
	}

	log.Printf("%s/%s@%s has not been versioned yet",
		github.GitHubGophrPackageOrgName,
		github.BuildNewGitHubRepoName(*packageModel.Author, *packageModel.Repo),
		github.BuildGitHubBranch(ref),
	)

	// Set working folderName for package
	folderName = github.BuildNewGitHubRepoName(*packageModel.Author, *packageModel.Repo)

	// Instantiate New Github Request Service
	log.Println("Initializing gitHub component")
	gitHubRequestService := github.NewGitHubRequestService()

	// Prepare to Create a new Github repo for packageModel if DNE
	log.Printf("Creating new Github repo for %s/%s at %s",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	err = gitHubRequestService.CreateNewGitHubRepo(*packageModel)
	// TODO:(Shikkic) figure out
	log.Printf("%s", err)

	log.Printf("Initializing folder and initializing git repo for %s \n", folderName)
	if err = initializeRepoCMD(packageModel); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in initializing the git repo. %s", err)
	}

	log.Printf("Creating branch %s \n", github.BuildGitHubBranch(ref))
	if err = createBranchCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in creating git branch. %s", err)
	}

	log.Printf("Setting remote branch url %s \n", github.BuildGitHubBranch(ref))
	if err = setRemoteCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in setting remote url. %s", err)
	}

	log.Printf("Fetching github archive for %s/%s with tag %s \n",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	if err = fetchArchiveCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in fetching ref archive. %s", err)
	}

	log.Printf("Fetching github archive for %s/%s with tag %s \n",
		*packageModel.Author,
		*packageModel.Repo,
		ref,
	)
	if err = unzipArchiveCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in unziping ref archive. %s", err)
	}

	// Fetch the timestamp of the ref commit
	commitDate, err := gitHubRequestService.FetchCommitTimestamp(packageModel, ref)
	if err != nil {
		return fmt.Errorf("Error occured in retrieving commit timestamp %s \n", err)
	}

	// Version lock all of the Github dependencies in the packageModel
	if err = verdeps.VersionDeps(
		verdeps.VersionDepsArgs{
			SHA:           ref,
			Path:          fmt.Sprintf("/tmp/%s", folderName),
			Date:          commitDate,
			Model:         packageModel,
			GithubService: gitHubRequestService,
		}); err != nil {
		return fmt.Errorf("Error occured in versioning deps %s \n", err)
	}

	log.Println("Adding unarchived repo data to branch")
	if err = addFilesCMD(); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in git add. %s", err)
	}

	log.Println("Commiting repo data to branch")
	if err = commitFilesCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in commit repo data. %s", err)
	}

	log.Printf("Pushing files to branch %s \n", github.BuildRemoteURL(packageModel, ref))
	if err = pushFilesCMD(packageModel, ref); err != nil {
		checkError(err, folderName)
		return fmt.Errorf("Error occured in pushing files to branch. %s", err)
	}

	cleanUpExitHook(folderName)

	return nil
}

func initializeRepoCMD(packageModel *models.PackageModel) error {
	log.Println("Initializing folder and repo commmand")
	cmd := fmt.Sprintf(initalizeRepo, folderName, folderName)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func createBranchCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Initializing folder and repo commmand")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(createBranch, navigateFolderCMD, github.BuildGitHubBranch(ref))
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func setRemoteCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Initializing folder and repo commmand")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	remoteURL := github.BuildRemoteURL(packageModel, ref)
	cmd := fmt.Sprintf(setRemoteCommand, navigateFolderCMD, remoteURL)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}
func fetchArchiveCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Fetching and Unzipping Archive for tag")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(fetchRepoArchive, navigateFolderCMD, *packageModel.Author, *packageModel.Repo, ref)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

func unzipArchiveCMD(packageModel *models.PackageModel, ref string) error {
	log.Println("Fetching and Unzipping Archive for tag")
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	zipFolder := *packageModel.Repo + "-" + ref
	cmd := fmt.Sprintf(unzipRepoArchive, navigateFolderCMD, ref, zipFolder, ref, zipFolder)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)

	return err
}

func addFilesCMD() error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(addFiles, navigateFolderCMD)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func commitFilesCMD(packageModel *models.PackageModel, ref string) error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	commitMessage := fmt.Sprintf("Gophr versioned repo of %s/%s@%s", *packageModel.Author, *packageModel.Repo, ref)
	cmd := fmt.Sprintf(commitFiles, navigateFolderCMD, commitMessage)
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s \n", out)
	return err
}

func pushFilesCMD(packageModel *models.PackageModel, ref string) error {
	navigateFolderCMD := fmt.Sprintf(navigateToPackageFolder, folderName)
	cmd := fmt.Sprintf(pushFiles, navigateFolderCMD, github.BuildGitHubBranch(ref))
	log.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	log.Printf("Output: %s 		\n", out)
	return err
}

// Helper functions

// Exit Hook to clean up files
func cleanUpExitHook(folderName string) {
	log.Printf("Exit hook initiated deleting folder %s \n", folderName)
	if strings.HasPrefix(folderName, "/") == true {
		cmd := fmt.Sprintf("cd /tmp && rm -rf %s", folderName)
		out, err := exec.Command("sh", "-c", cmd).Output()
		if err != nil {
			log.Println("Could not properly engage exit hook")
			log.Fatalln(err)
		}
		log.Printf("Output: %s", out)
	} else {
		log.Println("Cowardly refusing to initiate exit hook. Will not rm -rf folder names that contains any leading '/'")
	}
}

// Check Error and Engage Exit Hook if fatal error occured
func checkError(err error, folderName string) {
	if err != nil {
		cleanUpExitHook(folderName)
	}
}
<|endoftext|>


<commit_before>package guardian

import (
	"errors"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
)

var (
	ErrGuardianPermissionExists = errors.New("Permission already exists")
	ErrGuardianOverride         = errors.New("You can only override a permission to be higher")
)

// DashboardGuardian to be used for guard against operations without access on dashboard and acl
type DashboardGuardian interface {
	CanSave() (bool, error)
	CanEdit() (bool, error)
	CanView() (bool, error)
	CanAdmin() (bool, error)
	HasPermission(permission m.PermissionType) (bool, error)
	CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error)
	GetAcl() ([]*m.DashboardAclInfoDTO, error)
}

type dashboardGuardianImpl struct {
	user   *m.SignedInUser
	dashId int64
	orgId  int64
	acl    []*m.DashboardAclInfoDTO
	teams  []*m.TeamDTO
	log    log.Logger
}

// New factory for creating a new dashboard guardian instance
var New = func(dashId int64, orgId int64, user *m.SignedInUser) DashboardGuardian {
	return &dashboardGuardianImpl{
		user:   user,
		dashId: dashId,
		orgId:  orgId,
		log:    log.New("guardians.dashboard"),
	}
}

func (g *dashboardGuardianImpl) CanSave() (bool, error) {
	return g.HasPermission(m.PERMISSION_EDIT)
}

func (g *dashboardGuardianImpl) CanEdit() (bool, error) {
	if setting.ViewersCanEdit {
		return g.HasPermission(m.PERMISSION_VIEW)
	}

	return g.HasPermission(m.PERMISSION_EDIT)
}

func (g *dashboardGuardianImpl) CanView() (bool, error) {
	return g.HasPermission(m.PERMISSION_VIEW)
}

func (g *dashboardGuardianImpl) CanAdmin() (bool, error) {
	return g.HasPermission(m.PERMISSION_ADMIN)
}

func (g *dashboardGuardianImpl) HasPermission(permission m.PermissionType) (bool, error) {
	if g.user.OrgRole == m.ROLE_ADMIN {
		return true, nil
	}

	acl, err := g.GetAcl()
	if err != nil {
		return false, err
	}

	return g.checkAcl(permission, acl)
}

func (g *dashboardGuardianImpl) checkAcl(permission m.PermissionType, acl []*m.DashboardAclInfoDTO) (bool, error) {
	orgRole := g.user.OrgRole
	teamAclItems := []*m.DashboardAclInfoDTO{}

	for _, p := range acl {
		// user match
		if !g.user.IsAnonymous && p.UserId > 0 {
			if p.UserId == g.user.UserId && p.Permission >= permission {
				return true, nil
			}
		}

		// role match
		if p.Role != nil {
			if *p.Role == orgRole && p.Permission >= permission {
				return true, nil
			}
		}

		// remember this rule for later
		if p.TeamId > 0 {
			teamAclItems = append(teamAclItems, p)
		}
	}

	// do we have team rules?
	if len(teamAclItems) == 0 {
		return false, nil
	}

	// load teams
	teams, err := g.getTeams()
	if err != nil {
		return false, err
	}

	// evaluate team rules
	for _, p := range acl {
		for _, ug := range teams {
			if ug.Id == p.TeamId && p.Permission >= permission {
				return true, nil
			}
		}
	}

	return false, nil
}

func (g *dashboardGuardianImpl) CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error) {
	acl := []*m.DashboardAclInfoDTO{}
	adminRole := m.ROLE_ADMIN
	everyoneWithAdminRole := &m.DashboardAclInfoDTO{DashboardId: g.dashId, UserId: 0, TeamId: 0, Role: &adminRole, Permission: m.PERMISSION_ADMIN}

	// validate that duplicate permissions don't exists
	for _, p := range updatePermissions {
		aclItem := &m.DashboardAclInfoDTO{DashboardId: p.DashboardId, UserId: p.UserId, TeamId: p.TeamId, Role: p.Role, Permission: p.Permission}
		if aclItem.IsDuplicateOf(everyoneWithAdminRole) {
			return false, ErrGuardianPermissionExists
		}

		for _, a := range acl {
			if a.IsDuplicateOf(aclItem) {
				return false, ErrGuardianPermissionExists
			}
		}

		acl = append(acl, aclItem)
	}

	existingPermissions, err := g.GetAcl()
	if err != nil {
		return false, err
	}

	// validate overridden permissions to be higher
	for _, a := range acl {
		for _, existingPerm := range existingPermissions {
			if !existingPerm.Inherited {
				continue
			}

			if a.IsDuplicateOf(existingPerm) && a.Permission <= existingPerm.Permission {
				return false, ErrGuardianOverride
			}
		}
	}

	if g.user.OrgRole == m.ROLE_ADMIN {
		return true, nil
	}

	return g.checkAcl(permission, existingPermissions)
}

// GetAcl returns dashboard acl
func (g *dashboardGuardianImpl) GetAcl() ([]*m.DashboardAclInfoDTO, error) {
	if g.acl != nil {
		return g.acl, nil
	}

	query := m.GetDashboardAclInfoListQuery{DashboardId: g.dashId, OrgId: g.orgId}
	if err := bus.Dispatch(&query); err != nil {
		return nil, err
	}

	g.acl = query.Result
	return g.acl, nil
}

func (g *dashboardGuardianImpl) getTeams() ([]*m.TeamDTO, error) {
	if g.teams != nil {
		return g.teams, nil
	}

	query := m.GetTeamsByUserQuery{OrgId: g.orgId, UserId: g.user.UserId}
	err := bus.Dispatch(&query)

	g.teams = query.Result
	return query.Result, err
}

type FakeDashboardGuardian struct {
	DashId                           int64
	OrgId                            int64
	User                             *m.SignedInUser
	CanSaveValue                     bool
	CanEditValue                     bool
	CanViewValue                     bool
	CanAdminValue                    bool
	HasPermissionValue               bool
	CheckPermissionBeforeUpdateValue bool
	CheckPermissionBeforeUpdateError error
	GetAclValue                      []*m.DashboardAclInfoDTO
}

func (g *FakeDashboardGuardian) CanSave() (bool, error) {
	return g.CanSaveValue, nil
}

func (g *FakeDashboardGuardian) CanEdit() (bool, error) {
	return g.CanEditValue, nil
}

func (g *FakeDashboardGuardian) CanView() (bool, error) {
	return g.CanViewValue, nil
}

func (g *FakeDashboardGuardian) CanAdmin() (bool, error) {
	return g.CanAdminValue, nil
}

func (g *FakeDashboardGuardian) HasPermission(permission m.PermissionType) (bool, error) {
	return g.HasPermissionValue, nil
}

func (g *FakeDashboardGuardian) CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error) {
	return g.CheckPermissionBeforeUpdateValue, g.CheckPermissionBeforeUpdateError
}

func (g *FakeDashboardGuardian) GetAcl() ([]*m.DashboardAclInfoDTO, error) {
	return g.GetAclValue, nil
}

func MockDashboardGuardian(mock *FakeDashboardGuardian) {
	New = func(dashId int64, orgId int64, user *m.SignedInUser) DashboardGuardian {
		mock.OrgId = orgId
		mock.DashId = dashId
		mock.User = user
		return mock
	}
}
<commit_msg>add debug logging of folder/dashbord permission checks<commit_after>package guardian

import (
	"errors"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
)

var (
	ErrGuardianPermissionExists = errors.New("Permission already exists")
	ErrGuardianOverride         = errors.New("You can only override a permission to be higher")
)

// DashboardGuardian to be used for guard against operations without access on dashboard and acl
type DashboardGuardian interface {
	CanSave() (bool, error)
	CanEdit() (bool, error)
	CanView() (bool, error)
	CanAdmin() (bool, error)
	HasPermission(permission m.PermissionType) (bool, error)
	CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error)
	GetAcl() ([]*m.DashboardAclInfoDTO, error)
}

type dashboardGuardianImpl struct {
	user   *m.SignedInUser
	dashId int64
	orgId  int64
	acl    []*m.DashboardAclInfoDTO
	teams  []*m.TeamDTO
	log    log.Logger
}

// New factory for creating a new dashboard guardian instance
var New = func(dashId int64, orgId int64, user *m.SignedInUser) DashboardGuardian {
	return &dashboardGuardianImpl{
		user:   user,
		dashId: dashId,
		orgId:  orgId,
		log:    log.New("dashboard.permissions"),
	}
}

func (g *dashboardGuardianImpl) CanSave() (bool, error) {
	return g.HasPermission(m.PERMISSION_EDIT)
}

func (g *dashboardGuardianImpl) CanEdit() (bool, error) {
	if setting.ViewersCanEdit {
		return g.HasPermission(m.PERMISSION_VIEW)
	}

	return g.HasPermission(m.PERMISSION_EDIT)
}

func (g *dashboardGuardianImpl) CanView() (bool, error) {
	return g.HasPermission(m.PERMISSION_VIEW)
}

func (g *dashboardGuardianImpl) CanAdmin() (bool, error) {
	return g.HasPermission(m.PERMISSION_ADMIN)
}

func (g *dashboardGuardianImpl) HasPermission(permission m.PermissionType) (bool, error) {
	if g.user.OrgRole == m.ROLE_ADMIN {
		return g.logHasPermissionResult(permission, true, nil)
	}

	acl, err := g.GetAcl()
	if err != nil {
		return g.logHasPermissionResult(permission, false, err)
	}

	result, err := g.checkAcl(permission, acl)
	return g.logHasPermissionResult(permission, result, err)
}

func (g *dashboardGuardianImpl) logHasPermissionResult(permission m.PermissionType, hasPermission bool, err error) (bool, error) {
	if err != nil {
		return hasPermission, err
	}

	if hasPermission {
		g.log.Debug("User granted access to execute action", "userId", g.user.UserId, "orgId", g.orgId, "uname", g.user.Login, "dashId", g.dashId, "action", permission)
	} else {
		g.log.Debug("User denied access to execute action", "userId", g.user.UserId, "orgId", g.orgId, "uname", g.user.Login, "dashId", g.dashId, "action", permission)
	}

	return hasPermission, err
}

func (g *dashboardGuardianImpl) checkAcl(permission m.PermissionType, acl []*m.DashboardAclInfoDTO) (bool, error) {
	orgRole := g.user.OrgRole
	teamAclItems := []*m.DashboardAclInfoDTO{}

	for _, p := range acl {
		// user match
		if !g.user.IsAnonymous && p.UserId > 0 {
			if p.UserId == g.user.UserId && p.Permission >= permission {
				return true, nil
			}
		}

		// role match
		if p.Role != nil {
			if *p.Role == orgRole && p.Permission >= permission {
				return true, nil
			}
		}

		// remember this rule for later
		if p.TeamId > 0 {
			teamAclItems = append(teamAclItems, p)
		}
	}

	// do we have team rules?
	if len(teamAclItems) == 0 {
		return false, nil
	}

	// load teams
	teams, err := g.getTeams()
	if err != nil {
		return false, err
	}

	// evaluate team rules
	for _, p := range acl {
		for _, ug := range teams {
			if ug.Id == p.TeamId && p.Permission >= permission {
				return true, nil
			}
		}
	}

	return false, nil
}

func (g *dashboardGuardianImpl) CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error) {
	acl := []*m.DashboardAclInfoDTO{}
	adminRole := m.ROLE_ADMIN
	everyoneWithAdminRole := &m.DashboardAclInfoDTO{DashboardId: g.dashId, UserId: 0, TeamId: 0, Role: &adminRole, Permission: m.PERMISSION_ADMIN}

	// validate that duplicate permissions don't exists
	for _, p := range updatePermissions {
		aclItem := &m.DashboardAclInfoDTO{DashboardId: p.DashboardId, UserId: p.UserId, TeamId: p.TeamId, Role: p.Role, Permission: p.Permission}
		if aclItem.IsDuplicateOf(everyoneWithAdminRole) {
			return false, ErrGuardianPermissionExists
		}

		for _, a := range acl {
			if a.IsDuplicateOf(aclItem) {
				return false, ErrGuardianPermissionExists
			}
		}

		acl = append(acl, aclItem)
	}

	existingPermissions, err := g.GetAcl()
	if err != nil {
		return false, err
	}

	// validate overridden permissions to be higher
	for _, a := range acl {
		for _, existingPerm := range existingPermissions {
			if !existingPerm.Inherited {
				continue
			}

			if a.IsDuplicateOf(existingPerm) && a.Permission <= existingPerm.Permission {
				return false, ErrGuardianOverride
			}
		}
	}

	if g.user.OrgRole == m.ROLE_ADMIN {
		return true, nil
	}

	return g.checkAcl(permission, existingPermissions)
}

// GetAcl returns dashboard acl
func (g *dashboardGuardianImpl) GetAcl() ([]*m.DashboardAclInfoDTO, error) {
	if g.acl != nil {
		return g.acl, nil
	}

	query := m.GetDashboardAclInfoListQuery{DashboardId: g.dashId, OrgId: g.orgId}
	if err := bus.Dispatch(&query); err != nil {
		return nil, err
	}

	g.acl = query.Result
	return g.acl, nil
}

func (g *dashboardGuardianImpl) getTeams() ([]*m.TeamDTO, error) {
	if g.teams != nil {
		return g.teams, nil
	}

	query := m.GetTeamsByUserQuery{OrgId: g.orgId, UserId: g.user.UserId}
	err := bus.Dispatch(&query)

	g.teams = query.Result
	return query.Result, err
}

type FakeDashboardGuardian struct {
	DashId                           int64
	OrgId                            int64
	User                             *m.SignedInUser
	CanSaveValue                     bool
	CanEditValue                     bool
	CanViewValue                     bool
	CanAdminValue                    bool
	HasPermissionValue               bool
	CheckPermissionBeforeUpdateValue bool
	CheckPermissionBeforeUpdateError error
	GetAclValue                      []*m.DashboardAclInfoDTO
}

func (g *FakeDashboardGuardian) CanSave() (bool, error) {
	return g.CanSaveValue, nil
}

func (g *FakeDashboardGuardian) CanEdit() (bool, error) {
	return g.CanEditValue, nil
}

func (g *FakeDashboardGuardian) CanView() (bool, error) {
	return g.CanViewValue, nil
}

func (g *FakeDashboardGuardian) CanAdmin() (bool, error) {
	return g.CanAdminValue, nil
}

func (g *FakeDashboardGuardian) HasPermission(permission m.PermissionType) (bool, error) {
	return g.HasPermissionValue, nil
}

func (g *FakeDashboardGuardian) CheckPermissionBeforeUpdate(permission m.PermissionType, updatePermissions []*m.DashboardAcl) (bool, error) {
	return g.CheckPermissionBeforeUpdateValue, g.CheckPermissionBeforeUpdateError
}

func (g *FakeDashboardGuardian) GetAcl() ([]*m.DashboardAclInfoDTO, error) {
	return g.GetAclValue, nil
}

func MockDashboardGuardian(mock *FakeDashboardGuardian) {
	New = func(dashId int64, orgId int64, user *m.SignedInUser) DashboardGuardian {
		mock.OrgId = orgId
		mock.DashId = dashId
		mock.User = user
		return mock
	}
}
<|endoftext|>


<commit_before>//go:build !linux && !windows && !solaris && !freebsd
// +build !linux,!windows,!solaris,!freebsd

package system

// ReadMemInfo is not supported on platforms other than linux and windows.
func ReadMemInfo() (*MemInfo, error) {
	return nil, ErrNotSupportedPlatform
}
<commit_msg>Fix ReadMemInfo build for FreeBSD + nocgo<commit_after>//go:build !linux && !windows && !solaris && !(freebsd && cgo)
// +build !linux
// +build !windows
// +build !solaris
// +build !freebsd !cgo

package system

// ReadMemInfo is not supported on platforms other than linux and windows.
func ReadMemInfo() (*MemInfo, error) {
	return nil, ErrNotSupportedPlatform
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"context"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	cwapi "github.com/grafana/grafana/pkg/api/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "timeSeriesQuery":
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	default:
		err = fmt.Errorf("missing querytype")
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) {
	assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString()

	accessKey := ""
	secretKey := ""
	for key, value := range e.DataSource.SecureJsonData.Decrypt() {
		if key == "accessKey" {
			accessKey = value
		}
		if key == "secretKey" {
			secretKey = value
		}
	}

	datasourceInfo := &cwapi.DatasourceInfo{
		Region:        region,
		Profile:       e.DataSource.Database,
		AssumeRoleArn: assumeRoleArn,
		AccessKey:     accessKey,
		SecretKey:     secretKey,
	}

	credentials, err := cwapi.GetCredentials(datasourceInfo)
	if err != nil {
		return nil, err
	}

	cfg := &aws.Config{
		Region:      aws.String(region),
		Credentials: credentials,
	}

	sess, err := session.NewSession(cfg)
	if err != nil {
		return nil, err
	}

	client := cloudwatch.New(sess, cfg)
	return client, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]*string, []*string, error) {
	var statistics []*string
	var extendedStatistics []*string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, &ss)
			} else {
				extendedStatistics = append(extendedStatistics, &ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         statistics,
		ExtendedStatistics: extendedStatistics,
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<commit_msg>fix alert feature<commit_after>package cloudwatch

import (
	"context"
	"errors"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	cwapi "github.com/grafana/grafana/pkg/api/cloudwatch"
	"github.com/grafana/grafana/pkg/components/null"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/metrics"
)

type CloudWatchExecutor struct {
	*models.DataSource
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
		break
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
		break
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	result := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}

	errCh := make(chan error, 1)
	resCh := make(chan *tsdb.QueryResult, 1)

	currentlyExecuting := 0
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" {
			continue
		}
		currentlyExecuting++
		go func(refId string, index int) {
			queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
			currentlyExecuting--
			if err != nil {
				errCh <- err
			} else {
				queryRes.RefId = refId
				resCh <- queryRes
			}
		}(model.RefId, i)
	}

	for currentlyExecuting != 0 {
		select {
		case res := <-resCh:
			result.Results[res.RefId] = res
		case err := <-errCh:
			return result, err
		case <-ctx.Done():
			return result, ctx.Err()
		}
	}

	return result, nil
}

func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) {
	assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString()

	accessKey := ""
	secretKey := ""
	for key, value := range e.DataSource.SecureJsonData.Decrypt() {
		if key == "accessKey" {
			accessKey = value
		}
		if key == "secretKey" {
			secretKey = value
		}
	}

	datasourceInfo := &cwapi.DatasourceInfo{
		Region:        region,
		Profile:       e.DataSource.Database,
		AssumeRoleArn: assumeRoleArn,
		AccessKey:     accessKey,
		SecretKey:     secretKey,
	}

	credentials, err := cwapi.GetCredentials(datasourceInfo)
	if err != nil {
		return nil, err
	}

	cfg := &aws.Config{
		Region:      aws.String(region),
		Credentials: credentials,
	}

	sess, err := session.NewSession(cfg)
	if err != nil {
		return nil, err
	}

	client := cloudwatch.New(sess, cfg)
	return client, nil
}

func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
	query, err := parseQuery(parameters)
	if err != nil {
		return nil, err
	}

	client, err := e.getClient(query.Region)
	if err != nil {
		return nil, err
	}

	startTime, err := queryContext.TimeRange.ParseFrom()
	if err != nil {
		return nil, err
	}

	endTime, err := queryContext.TimeRange.ParseTo()
	if err != nil {
		return nil, err
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(query.Namespace),
		MetricName: aws.String(query.MetricName),
		Dimensions: query.Dimensions,
		Period:     aws.Int64(int64(query.Period)),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
	}
	if len(query.Statistics) > 0 {
		params.Statistics = query.Statistics
	}
	if len(query.ExtendedStatistics) > 0 {
		params.ExtendedStatistics = query.ExtendedStatistics
	}

	resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second))
	if err != nil {
		return nil, err
	}
	metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc()

	queryRes, err := parseResponse(resp, query)
	if err != nil {
		return nil, err
	}

	return queryRes, nil
}

func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
	var result []*cloudwatch.Dimension

	for k, v := range model.Get("dimensions").MustMap() {
		kk := k
		if vv, ok := v.(string); ok {
			result = append(result, &cloudwatch.Dimension{
				Name:  &kk,
				Value: &vv,
			})
		} else {
			return nil, errors.New("failed to parse")
		}
	}

	sort.Slice(result, func(i, j int) bool {
		return *result[i].Name < *result[j].Name
	})
	return result, nil
}

func parseStatistics(model *simplejson.Json) ([]*string, []*string, error) {
	var statistics []*string
	var extendedStatistics []*string

	for _, s := range model.Get("statistics").MustArray() {
		if ss, ok := s.(string); ok {
			if _, isStandard := standardStatistics[ss]; isStandard {
				statistics = append(statistics, &ss)
			} else {
				extendedStatistics = append(extendedStatistics, &ss)
			}
		} else {
			return nil, nil, errors.New("failed to parse")
		}
	}

	return statistics, extendedStatistics, nil
}

func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
	region, err := model.Get("region").String()
	if err != nil {
		return nil, err
	}

	namespace, err := model.Get("namespace").String()
	if err != nil {
		return nil, err
	}

	metricName, err := model.Get("metricName").String()
	if err != nil {
		return nil, err
	}

	dimensions, err := parseDimensions(model)
	if err != nil {
		return nil, err
	}

	statistics, extendedStatistics, err := parseStatistics(model)
	if err != nil {
		return nil, err
	}

	p := model.Get("period").MustString("")
	if p == "" {
		if namespace == "AWS/EC2" {
			p = "300"
		} else {
			p = "60"
		}
	}

	period := 300
	if regexp.MustCompile(`^\d+$`).Match([]byte(p)) {
		period, err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	} else {
		d, err := time.ParseDuration(p)
		if err != nil {
			return nil, err
		}
		period = int(d.Seconds())
	}

	alias := model.Get("alias").MustString("{{metric}}_{{stat}}")

	return &CloudWatchQuery{
		Region:             region,
		Namespace:          namespace,
		MetricName:         metricName,
		Dimensions:         dimensions,
		Statistics:         statistics,
		ExtendedStatistics: extendedStatistics,
		Period:             period,
		Alias:              alias,
	}, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
	data := map[string]string{}
	data["region"] = query.Region
	data["namespace"] = query.Namespace
	data["metric"] = query.MetricName
	data["stat"] = stat
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}

func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
	queryRes := tsdb.NewQueryResult()

	var value float64
	for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
		series := tsdb.TimeSeries{
			Tags: map[string]string{},
		}
		for _, d := range query.Dimensions {
			series.Tags[*d.Name] = *d.Value
		}
		series.Name = formatAlias(query, *s, series.Tags)

		lastTimestamp := make(map[string]time.Time)
		sort.Slice(resp.Datapoints, func(i, j int) bool {
			return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp)
		})
		for _, v := range resp.Datapoints {
			switch *s {
			case "Average":
				value = *v.Average
			case "Maximum":
				value = *v.Maximum
			case "Minimum":
				value = *v.Minimum
			case "Sum":
				value = *v.Sum
			case "SampleCount":
				value = *v.SampleCount
			default:
				if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil {
					value = *v.ExtendedStatistics[*s]
				}
			}

			// terminate gap of data points
			timestamp := *v.Timestamp
			if _, ok := lastTimestamp[*s]; ok {
				nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second)
				for timestamp.After(nextTimestampFromLast) {
					series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000)))
					nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second)
				}
			}
			lastTimestamp[*s] = timestamp

			series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000)))
		}

		queryRes.Series = append(queryRes.Series, &series)
	}

	return queryRes, nil
}
<|endoftext|>