<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"time"
	"io"
	"mime/multipart"
	"path"
	"path/filepath"
	"strings"
	"bytes"

	uploadClient "github.com/webx-top/client/upload"
	_ "github.com/webx-top/client/upload/driver"
	"github.com/webx-top/echo"

	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/common"
	modelFile "github.com/admpub/nging/application/model/file"
	"github.com/admpub/nging/application/registry/upload"
	"github.com/admpub/nging/application/registry/upload/driver/filesystem"
	"github.com/admpub/nging/application/registry/upload/helper"
	"github.com/admpub/nging/application/registry/upload/convert"
	"github.com/admpub/qrcode"
)

// 文件上传保存路径规则：
// 表名称/表行ID/文件名

// ResponseDataForUpload 根据不同的上传方式响应不同的数据格式
func ResponseDataForUpload(ctx echo.Context, field string, err error, imageURLs []string) (result echo.H, embed bool) {
	return upload.ResponserGet(field)(ctx, field, err, imageURLs)
}

var (
	StorerEngine = filesystem.Name
)

func File(ctx echo.Context) error {
	uploadType := ctx.Param(`type`)
	typ, _, _ := getTableInfo(uploadType)
	file := ctx.Param(`*`)
	file = filepath.Join(helper.UploadDir, typ, file)
	originalExtension := filepath.Ext(file)
	extension := strings.ToLower(originalExtension)
	convert, ok := convert.GetConverter(extension)
	if !ok {
		return ctx.File(file)
	}
	supported := strings.Contains(ctx.Header(echo.HeaderAccept), "image/" + strings.TrimPrefix(extension, `.`))
	originalFile := strings.TrimSuffix(file, originalExtension)
	if !supported {
		return ctx.File(originalFile)
	}
	if err := ctx.File(file); err != echo.ErrNotFound {
		return err
	}
	return ctx.ServeCallbackContent(func(_ echo.Context) (io.Reader, error) {
		newStore := upload.StorerGet(StorerEngine)
		if newStore == nil {
			return nil, ctx.E(`存储引擎“%s”未被登记`, StorerEngine)
		}
		storer := newStore(ctx, typ)
		f, err := storer.Get(`/` + originalFile)
		if err != nil {
			return nil, echo.ErrNotFound
		}
		defer f.Close()
		buf, err := convert(f, 70)
		if err != nil {
			return nil, err
		}
		b := buf.Bytes()
		saveFile := storer.URLToFile(`/` + file)
		_, _, err = storer.Put(saveFile, buf, int64(len(b)))
		return bytes.NewBuffer(b), err
	}, path.Base(file), time.Now())
}

// SaveFilename SaveFilename(`0/`,``,`img.jpg`)
func SaveFilename(subdir, name, postFilename string) (string, error) {
	ext := filepath.Ext(postFilename)
	fname := name
	if len(fname) == 0 {
		var err error
		fname, err = common.UniqueID()
		if err != nil {
			return ``, err
		}
	}
	fname += ext
	return subdir + fname, nil
}

// Upload 上传文件
func Upload(ctx echo.Context) error {
	ownerType := `user`
	user := handler.User(ctx)
	var ownerID uint64
	if user != nil {
		ownerID = uint64(user.Id)
	}
	if ownerID < 1 {
		ctx.Data().SetError(ctx.E(`请先登录`))
		return ctx.Redirect(handler.URLFor(`/login`))
	}
	return UploadByOwner(ctx, ownerType, ownerID)
}

// UploadByOwner 上传文件
func UploadByOwner(ctx echo.Context, ownerType string, ownerID uint64) error {
	uploadType := ctx.Param(`type`)
	field := ctx.Query(`field`) // 上传表单file输入框名称
	pipe := ctx.Form(`pipe`)
	var (
		err      error
		fileURLs []string
	)
	if len(uploadType) == 0 {
		err = ctx.E(`请提供参数“%s”`, ctx.Path())
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}
	fileType := ctx.Form(`filetype`)
	prepareData, err := upload.Prepare(ctx, uploadType, fileType, StorerEngine)
	if err != nil {
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
	}
	storer := prepareData.Storer(ctx)
	defer prepareData.Close()
	fileM := modelFile.NewFile(ctx)
	fileM.StorerName = StorerEngine
	fileM.TableId = ``
	fileM.SetFieldName(prepareData.FieldName)
	fileM.SetTableName(prepareData.TableName)
	fileM.OwnerId = ownerID
	fileM.OwnerType = ownerType
	fileM.Type = fileType

	subdir, name, err := prepareData.Checkin(ctx, fileM)
	if err != nil {
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}

	clientName := ctx.Form(`client`)
	if len(clientName) > 0 {
		result := &uploadClient.Result{}
		result.SetFileNameGenerator(func(filename string) (string, error) {
			return SaveFilename(subdir, name, filename)
		})

		client := uploadClient.Upload(ctx, clientName, result, storer, watermarkFile, prepareData.Checker)
		if client.GetError() != nil {
			if client.GetError() == upload.ErrExistsFile {
				client.SetError(nil)
			}
			return client.Response()
		}

		fileM.SetByUploadResult(result)

		var reader io.ReadCloser
		reader, err = storer.Get(result.SavePath)
		if reader != nil {
			defer reader.Close()
		}
		if err != nil {
			return client.SetError(err).Response()
		}
		err = prepareData.DBSaver(fileM, result, reader)
		return client.SetError(err).Response()
	}
	var results uploadClient.Results
	results, err = upload.BatchUpload(
		ctx,
		`files[]`,
		func(r *uploadClient.Result) (string, error) {
			if err := prepareData.Checker(r); err != nil {
				return ``, err
			}
			return SaveFilename(subdir, name, r.FileName)
		},
		storer,
		func(result *uploadClient.Result, file multipart.File) error {
			fileM.Id = 0
			fileM.SetByUploadResult(result)
			return prepareData.DBSaver(fileM, result, file)
		},
		watermarkFile,
	)
	datax, embed := ResponseDataForUpload(ctx, field, err, results.FileURLs())
	if err != nil {
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}

	if pipe == `deqr` { //解析二维码
		if len(results) > 0 {
			reader, err := storer.Get(results[0].SavePath)
			if reader != nil {
				defer reader.Close()
			}
			if err != nil {
				if !embed {
					datax[`raw`] = err.Error()
					return ctx.JSON(datax)
				}
				return err
			}
			raw, err := qrcode.Decode(reader, strings.TrimPrefix(path.Ext(results[0].SavePath), `.`))
			if err != nil {
				raw = err.Error()
			}
			datax[`raw`] = raw
		}
	}
	if !embed {
		return ctx.JSON(datax)
	}
	data := ctx.Data()
	data.SetData(datax)
	return ctx.JSON(data)
}

func getTableInfo(uploadType string) (tableName string, fieldName string, defaults []string) {
	return upload.GetTableInfo(uploadType)
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"time"
	"io"
	"mime/multipart"
	"path"
	"path/filepath"
	"strings"
	"bytes"

	uploadClient "github.com/webx-top/client/upload"
	_ "github.com/webx-top/client/upload/driver"
	"github.com/webx-top/echo"

	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/common"
	modelFile "github.com/admpub/nging/application/model/file"
	"github.com/admpub/nging/application/registry/upload"
	"github.com/admpub/nging/application/registry/upload/driver/filesystem"
	"github.com/admpub/nging/application/registry/upload/helper"
	"github.com/admpub/nging/application/registry/upload/convert"
	"github.com/admpub/qrcode"
)

// 文件上传保存路径规则：
// 表名称/表行ID/文件名

// ResponseDataForUpload 根据不同的上传方式响应不同的数据格式
func ResponseDataForUpload(ctx echo.Context, field string, err error, imageURLs []string) (result echo.H, embed bool) {
	return upload.ResponserGet(field)(ctx, field, err, imageURLs)
}

var (
	StorerEngine = filesystem.Name
)

func File(ctx echo.Context) error {
	uploadType := ctx.Param(`type`)
	typ, _, _ := getTableInfo(uploadType)
	file := ctx.Param(`*`)
	file = filepath.Join(helper.UploadDir, typ, file)
	originalExtension := filepath.Ext(file)
	extension := strings.ToLower(originalExtension)
	convert, ok := convert.GetConverter(extension)
	if !ok {
		return ctx.File(file)
	}
	supported := strings.Contains(ctx.Header(echo.HeaderAccept), "image/" + strings.TrimPrefix(extension, `.`))
	originalFile := strings.TrimSuffix(file, originalExtension)
	if !supported {
		return ctx.File(originalFile)
	}
	if err := ctx.File(file); err != echo.ErrNotFound {
		return err
	}
	return ctx.ServeCallbackContent(func(_ echo.Context) (io.Reader, error) {
		newStore := upload.StorerGet(StorerEngine)
		if newStore == nil {
			return nil, ctx.E(`存储引擎“%s”未被登记`, StorerEngine)
		}
		storer := newStore(ctx, typ)
		f, err := storer.Get(`/` + originalFile)
		if err != nil {
			return nil, echo.ErrNotFound
		}
		defer f.Close()
		buf, err := convert(f, 70)
		if err != nil {
			return nil, err
		}
		b := buf.Bytes()
		saveFile := storer.URLToFile(`/` + file)
		_, _, err = storer.Put(saveFile, buf, int64(len(b)))
		return bytes.NewBuffer(b), err
	}, path.Base(file), time.Unix(0, 0))
}

// SaveFilename SaveFilename(`0/`,``,`img.jpg`)
func SaveFilename(subdir, name, postFilename string) (string, error) {
	ext := filepath.Ext(postFilename)
	fname := name
	if len(fname) == 0 {
		var err error
		fname, err = common.UniqueID()
		if err != nil {
			return ``, err
		}
	}
	fname += ext
	return subdir + fname, nil
}

// Upload 上传文件
func Upload(ctx echo.Context) error {
	ownerType := `user`
	user := handler.User(ctx)
	var ownerID uint64
	if user != nil {
		ownerID = uint64(user.Id)
	}
	if ownerID < 1 {
		ctx.Data().SetError(ctx.E(`请先登录`))
		return ctx.Redirect(handler.URLFor(`/login`))
	}
	return UploadByOwner(ctx, ownerType, ownerID)
}

// UploadByOwner 上传文件
func UploadByOwner(ctx echo.Context, ownerType string, ownerID uint64) error {
	uploadType := ctx.Param(`type`)
	field := ctx.Query(`field`) // 上传表单file输入框名称
	pipe := ctx.Form(`pipe`)
	var (
		err      error
		fileURLs []string
	)
	if len(uploadType) == 0 {
		err = ctx.E(`请提供参数“%s”`, ctx.Path())
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}
	fileType := ctx.Form(`filetype`)
	prepareData, err := upload.Prepare(ctx, uploadType, fileType, StorerEngine)
	if err != nil {
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
	}
	storer := prepareData.Storer(ctx)
	defer prepareData.Close()
	fileM := modelFile.NewFile(ctx)
	fileM.StorerName = StorerEngine
	fileM.TableId = ``
	fileM.SetFieldName(prepareData.FieldName)
	fileM.SetTableName(prepareData.TableName)
	fileM.OwnerId = ownerID
	fileM.OwnerType = ownerType
	fileM.Type = fileType

	subdir, name, err := prepareData.Checkin(ctx, fileM)
	if err != nil {
		datax, embed := ResponseDataForUpload(ctx, field, err, fileURLs)
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}

	clientName := ctx.Form(`client`)
	if len(clientName) > 0 {
		result := &uploadClient.Result{}
		result.SetFileNameGenerator(func(filename string) (string, error) {
			return SaveFilename(subdir, name, filename)
		})

		client := uploadClient.Upload(ctx, clientName, result, storer, watermarkFile, prepareData.Checker)
		if client.GetError() != nil {
			if client.GetError() == upload.ErrExistsFile {
				client.SetError(nil)
			}
			return client.Response()
		}

		fileM.SetByUploadResult(result)

		var reader io.ReadCloser
		reader, err = storer.Get(result.SavePath)
		if reader != nil {
			defer reader.Close()
		}
		if err != nil {
			return client.SetError(err).Response()
		}
		err = prepareData.DBSaver(fileM, result, reader)
		return client.SetError(err).Response()
	}
	var results uploadClient.Results
	results, err = upload.BatchUpload(
		ctx,
		`files[]`,
		func(r *uploadClient.Result) (string, error) {
			if err := prepareData.Checker(r); err != nil {
				return ``, err
			}
			return SaveFilename(subdir, name, r.FileName)
		},
		storer,
		func(result *uploadClient.Result, file multipart.File) error {
			fileM.Id = 0
			fileM.SetByUploadResult(result)
			return prepareData.DBSaver(fileM, result, file)
		},
		watermarkFile,
	)
	datax, embed := ResponseDataForUpload(ctx, field, err, results.FileURLs())
	if err != nil {
		if !embed {
			return ctx.JSON(datax)
		}
		return err
	}

	if pipe == `deqr` { //解析二维码
		if len(results) > 0 {
			reader, err := storer.Get(results[0].SavePath)
			if reader != nil {
				defer reader.Close()
			}
			if err != nil {
				if !embed {
					datax[`raw`] = err.Error()
					return ctx.JSON(datax)
				}
				return err
			}
			raw, err := qrcode.Decode(reader, strings.TrimPrefix(path.Ext(results[0].SavePath), `.`))
			if err != nil {
				raw = err.Error()
			}
			datax[`raw`] = raw
		}
	}
	if !embed {
		return ctx.JSON(datax)
	}
	data := ctx.Data()
	data.SetData(datax)
	return ctx.JSON(data)
}

func getTableInfo(uploadType string) (tableName string, fieldName string, defaults []string) {
	return upload.GetTableInfo(uploadType)
}
<|endoftext|>


<commit_before>package emitter_test

import (
	"github.com/concourse/concourse/atc/metric/emitter"
	"github.com/concourse/concourse/atc/metric/emitter/emitterfakes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/prometheus/client_golang/prometheus"
)

var _ = Describe("PrometheusEmitter garbage collector", func() {
	var (
		fake emitterfakes.FakePrometheusGarbageCollectable

		labelsLong  prometheus.Labels
		labelsShort prometheus.Labels

		workerContainers *prometheus.GaugeVec
		workerVolumes    *prometheus.GaugeVec
		workerTasks      *prometheus.GaugeVec

		workerContainersLabels map[string]map[string]prometheus.Labels
		workerVolumesLabels    map[string]map[string]prometheus.Labels
		workerTasksLabels      map[string]map[string]prometheus.Labels
	)

	BeforeEach(func() {
		workerContainers = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "containers",
				Help:      "Number of containers per worker",
			},
			[]string{"worker", "platform", "team", "tags"},
		)
		prometheus.Register(workerContainers)

		workerVolumes = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "volumes",
				Help:      "Number of volumes per worker",
			},
			[]string{"worker", "platform", "team", "tags"},
		)
		prometheus.Register(workerVolumes)

		workerTasks = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "tasks",
				Help:      "Number of active tasks per worker",
			},
			[]string{"worker", "platform"},
		)
		prometheus.Register(workerTasks)

		workerContainersLabels = map[string]map[string]prometheus.Labels{}
		workerVolumesLabels = map[string]map[string]prometheus.Labels{}
		workerTasksLabels = map[string]map[string]prometheus.Labels{}

		labelsLong = prometheus.Labels{
			"worker":   "foo",
			"platform": "linux",
			"team":     "main",
			"tags":     "",
		}

		labelsShort = prometheus.Labels{
			"worker":   "foo",
			"platform": "linux",
		}
	})
	JustBeforeEach(func() {
		fake = emitterfakes.FakePrometheusGarbageCollectable{
			WorkerContainersStub: func() *prometheus.GaugeVec { return workerContainers },
			WorkerVolumesStub:    func() *prometheus.GaugeVec { return workerVolumes },
			WorkerTasksStub:      func() *prometheus.GaugeVec { return workerTasks },

			WorkerContainersLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerContainersLabels
			},
			WorkerVolumesLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerVolumesLabels
			},
			WorkerTasksLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerTasksLabels
			},
		}

		// Deep copy the labels so we can use them to verify the test results later
		labels := make(prometheus.Labels)
		for k, v := range labelsLong {
			labels[k] = v
		}
		fake.WorkerContainers().With(labels).Set(42.0)
		fake.WorkerContainersLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerContainersLabels()["foo"]["foo_linux_main__"] = labels

		fake.WorkerVolumes().With(labels).Set(42.0)
		fake.WorkerVolumesLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerVolumesLabels()["foo"]["foo_linux_main__"] = labels

		labels = make(prometheus.Labels)
		for k, v := range labelsShort {
			labels[k] = v
		}
		fake.WorkerTasks().With(labels).Set(42.0)
		fake.WorkerTasksLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerTasksLabels()["foo"]["foo_linux"] = labels
	})

	It("should remove all metrics from the emitter", func() {
		Expect(fake.WorkerContainersLabels()).To(HaveLen(1))
		Expect(fake.WorkerVolumesLabels()).To(HaveLen(1))
		Expect(fake.WorkerTasksLabels()).To(HaveLen(1))

		emitter.DoGarbageCollection(&fake, "foo")

		Expect(fake.WorkerContainersLabels()).To(HaveLen(0))
		Expect(fake.WorkerVolumesLabels()).To(HaveLen(0))
		Expect(fake.WorkerTasksLabels()).To(HaveLen(0))

		// Delete should return false if the metrics no longer exist
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(false))
	})

	// There is no easy way to detect whether metrics are REALLY garbage collected due to the
	// limitations of the Prometheus client library, so here we verify that the metrics that were
	// deleted in the previous spec were actually present from the beginning.
	It("should not do anything if there are no metrics", func() {
		// Delete should return true if the metrics are actually deleted
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(true))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(true))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(true))

		emitter.DoGarbageCollection(&fake, "foo")

		// Delete should return false if the metrics no longer exist
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(false))

	})

	AfterEach(func() {
		workerContainers.Reset()
		workerVolumes.Reset()
		workerTasks.Reset()

		workerContainersLabels = map[string]map[string]prometheus.Labels{}
		workerVolumesLabels = map[string]map[string]prometheus.Labels{}
		workerTasksLabels = map[string]map[string]prometheus.Labels{}
	})
})
<commit_msg>add test to check if task waiting metric is exposed by PrometheusEmitter<commit_after>package emitter_test

import (
	"fmt"
	"io/ioutil"
	"net/http"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"code.cloudfoundry.org/lager/lagertest"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/metric/emitter"
	"github.com/concourse/concourse/atc/metric/emitter/emitterfakes"
)

var _ = Describe("PrometheusEmitter garbage collector", func() {
	var (
		fake emitterfakes.FakePrometheusGarbageCollectable

		labelsLong  prometheus.Labels
		labelsShort prometheus.Labels

		workerContainers *prometheus.GaugeVec
		workerVolumes    *prometheus.GaugeVec
		workerTasks      *prometheus.GaugeVec

		workerContainersLabels map[string]map[string]prometheus.Labels
		workerVolumesLabels    map[string]map[string]prometheus.Labels
		workerTasksLabels      map[string]map[string]prometheus.Labels
	)

	BeforeEach(func() {
		workerContainers = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "containers",
				Help:      "Number of containers per worker",
			},
			[]string{"worker", "platform", "team", "tags"},
		)
		prometheus.Register(workerContainers)

		workerVolumes = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "volumes",
				Help:      "Number of volumes per worker",
			},
			[]string{"worker", "platform", "team", "tags"},
		)
		prometheus.Register(workerVolumes)

		workerTasks = prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: "concourse",
				Subsystem: "workers",
				Name:      "tasks",
				Help:      "Number of active tasks per worker",
			},
			[]string{"worker", "platform"},
		)
		prometheus.Register(workerTasks)

		workerContainersLabels = map[string]map[string]prometheus.Labels{}
		workerVolumesLabels = map[string]map[string]prometheus.Labels{}
		workerTasksLabels = map[string]map[string]prometheus.Labels{}

		labelsLong = prometheus.Labels{
			"worker":   "foo",
			"platform": "linux",
			"team":     "main",
			"tags":     "",
		}

		labelsShort = prometheus.Labels{
			"worker":   "foo",
			"platform": "linux",
		}
	})
	JustBeforeEach(func() {
		fake = emitterfakes.FakePrometheusGarbageCollectable{
			WorkerContainersStub: func() *prometheus.GaugeVec { return workerContainers },
			WorkerVolumesStub:    func() *prometheus.GaugeVec { return workerVolumes },
			WorkerTasksStub:      func() *prometheus.GaugeVec { return workerTasks },

			WorkerContainersLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerContainersLabels
			},
			WorkerVolumesLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerVolumesLabels
			},
			WorkerTasksLabelsStub: func() map[string]map[string]prometheus.Labels {
				return workerTasksLabels
			},
		}

		// Deep copy the labels so we can use them to verify the test results later
		labels := make(prometheus.Labels)
		for k, v := range labelsLong {
			labels[k] = v
		}
		fake.WorkerContainers().With(labels).Set(42.0)
		fake.WorkerContainersLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerContainersLabels()["foo"]["foo_linux_main__"] = labels

		fake.WorkerVolumes().With(labels).Set(42.0)
		fake.WorkerVolumesLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerVolumesLabels()["foo"]["foo_linux_main__"] = labels

		labels = make(prometheus.Labels)
		for k, v := range labelsShort {
			labels[k] = v
		}
		fake.WorkerTasks().With(labels).Set(42.0)
		fake.WorkerTasksLabels()["foo"] = make(map[string]prometheus.Labels)
		fake.WorkerTasksLabels()["foo"]["foo_linux"] = labels
	})

	It("should remove all metrics from the emitter", func() {
		Expect(fake.WorkerContainersLabels()).To(HaveLen(1))
		Expect(fake.WorkerVolumesLabels()).To(HaveLen(1))
		Expect(fake.WorkerTasksLabels()).To(HaveLen(1))

		emitter.DoGarbageCollection(&fake, "foo")

		Expect(fake.WorkerContainersLabels()).To(HaveLen(0))
		Expect(fake.WorkerVolumesLabels()).To(HaveLen(0))
		Expect(fake.WorkerTasksLabels()).To(HaveLen(0))

		// Delete should return false if the metrics no longer exist
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(false))
	})

	// There is no easy way to detect whether metrics are REALLY garbage collected due to the
	// limitations of the Prometheus client library, so here we verify that the metrics that were
	// deleted in the previous spec were actually present from the beginning.
	It("should not do anything if there are no metrics", func() {
		// Delete should return true if the metrics are actually deleted
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(true))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(true))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(true))

		emitter.DoGarbageCollection(&fake, "foo")

		// Delete should return false if the metrics no longer exist
		Expect(fake.WorkerContainers().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerVolumes().Delete(labelsLong)).To(Equal(false))
		Expect(fake.WorkerTasks().Delete(labelsShort)).To(Equal(false))

	})

	AfterEach(func() {
		workerContainers.Reset()
		workerVolumes.Reset()
		workerTasks.Reset()

		workerContainersLabels = map[string]map[string]prometheus.Labels{}
		workerVolumesLabels = map[string]map[string]prometheus.Labels{}
		workerTasksLabels = map[string]map[string]prometheus.Labels{}

		prometheus.Unregister(workerContainers)
		prometheus.Unregister(workerVolumes)
		prometheus.Unregister(workerTasks)
	})
})

var _ = Describe("PrometheusEmitter", func() {
	var (
		prometheusConfig  *emitter.PrometheusConfig
		prometheusEmitter metric.Emitter
		logger            *lagertest.TestLogger
		err               error
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")
		prometheusConfig = &emitter.PrometheusConfig{
			BindIP:   "localhost",
			BindPort: "9090",
		}
	})

	JustBeforeEach(func() {
		prometheusEmitter, err = prometheusConfig.NewEmitter()
	})

	It("emits task waiting metric", func() {
		prometheusEmitter.Emit(logger, metric.Event{
			Name:  "tasks waiting",
			Value: 4,
		})

		res, _ := http.Get(fmt.Sprintf("http://%s:%s/metrics", prometheusConfig.BindIP, prometheusConfig.BindPort))
		defer res.Body.Close()
		body, _ := ioutil.ReadAll(res.Body)

		Expect(res.StatusCode).To(Equal(http.StatusOK))
		Expect(string(body)).To(ContainSubstring("concourse_tasks_waiting 4"))
		Expect(err).To(BeNil())
	})
})
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/hashcode"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceAwsVolumeAttachment() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVolumeAttachmentCreate,
		Read:   resourceAwsVolumeAttachmentRead,
		Update: resourceAwsVolumeAttachmentUpdate,
		Delete: resourceAwsVolumeAttachmentDelete,

		Schema: map[string]*schema.Schema{
			"device_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"instance_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"volume_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"force_detach": {
				Type:     schema.TypeBool,
				Optional: true,
			},
			"skip_destroy": {
				Type:     schema.TypeBool,
				Optional: true,
			},
		},
	}
}

func resourceAwsVolumeAttachmentCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	name := d.Get("device_name").(string)
	iID := d.Get("instance_id").(string)
	vID := d.Get("volume_id").(string)

	// Find out if the volume is already attached to the instance, in which case
	// we have nothing to do
	request := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String(vID)},
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("attachment.instance-id"),
				Values: []*string{aws.String(iID)},
			},
			{
				Name:   aws.String("attachment.device"),
				Values: []*string{aws.String(name)},
			},
		},
	}

	vols, err := conn.DescribeVolumes(request)
	if (err != nil) || (len(vols.Volumes) == 0) {
		// This handles the situation where the instance is created by
		// a spot request and whilst the request has been fulfilled the
		// instance is not running yet
		stateConf := &resource.StateChangeConf{
			Pending:    []string{"pending", "stopping"},
			Target:     []string{"running", "stopped"},
			Refresh:    InstanceStateRefreshFunc(conn, iID, []string{"terminated"}),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		_, err = stateConf.WaitForState()
		if err != nil {
			return fmt.Errorf(
				"Error waiting for instance (%s) to become ready: %s",
				iID, err)
		}

		// not attached
		opts := &ec2.AttachVolumeInput{
			Device:     aws.String(name),
			InstanceId: aws.String(iID),
			VolumeId:   aws.String(vID),
		}

		log.Printf("[DEBUG] Attaching Volume (%s) to Instance (%s)", vID, iID)
		_, err := conn.AttachVolume(opts)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				return fmt.Errorf("Error attaching volume (%s) to instance (%s), message: \"%s\", code: \"%s\"",
					vID, iID, awsErr.Message(), awsErr.Code())
			}
			return err
		}
	}

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"attaching"},
		Target:     []string{"attached"},
		Refresh:    volumeAttachmentStateRefreshFunc(conn, name, vID, iID),
		Timeout:    5 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for Volume (%s) to attach to Instance: %s, error: %s",
			vID, iID, err)
	}

	d.SetId(volumeAttachmentID(name, vID, iID))
	return resourceAwsVolumeAttachmentRead(d, meta)
}

func volumeAttachmentStateRefreshFunc(conn *ec2.EC2, name, volumeID, instanceID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		request := &ec2.DescribeVolumesInput{
			VolumeIds: []*string{aws.String(volumeID)},
			Filters: []*ec2.Filter{
				{
					Name:   aws.String("attachment.device"),
					Values: []*string{aws.String(name)},
				},
				{
					Name:   aws.String("attachment.instance-id"),
					Values: []*string{aws.String(instanceID)},
				},
			},
		}

		resp, err := conn.DescribeVolumes(request)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				return nil, "failed", fmt.Errorf("code: %s, message: %s", awsErr.Code(), awsErr.Message())
			}
			return nil, "failed", err
		}

		if len(resp.Volumes) > 0 {
			v := resp.Volumes[0]
			for _, a := range v.Attachments {
				if a.InstanceId != nil && *a.InstanceId == instanceID {
					return a, *a.State, nil
				}
			}
		}
		// assume detached if volume count is 0
		return 42, "detached", nil
	}
}

func resourceAwsVolumeAttachmentRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	request := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String(d.Get("volume_id").(string))},
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("attachment.device"),
				Values: []*string{aws.String(d.Get("device_name").(string))},
			},
			{
				Name:   aws.String("attachment.instance-id"),
				Values: []*string{aws.String(d.Get("instance_id").(string))},
			},
		},
	}

	vols, err := conn.DescribeVolumes(request)
	if err != nil {
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidVolume.NotFound" {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading EC2 volume %s for instance: %s: %#v", d.Get("volume_id").(string), d.Get("instance_id").(string), err)
	}

	if len(vols.Volumes) == 0 || *vols.Volumes[0].State == "available" {
		log.Printf("[DEBUG] Volume Attachment (%s) not found, removing from state", d.Id())
		d.SetId("")
	}

	return nil
}

func resourceAwsVolumeAttachmentUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Attaching Volume (%s) is updating which does nothing but updates a few params in state", d.Id())
	return nil
}

func resourceAwsVolumeAttachmentDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	if _, ok := d.GetOk("skip_destroy"); ok {
		return nil
	}

	name := d.Get("device_name").(string)
	vID := d.Get("volume_id").(string)
	iID := d.Get("instance_id").(string)

	opts := &ec2.DetachVolumeInput{
		Device:     aws.String(name),
		InstanceId: aws.String(iID),
		VolumeId:   aws.String(vID),
		Force:      aws.Bool(d.Get("force_detach").(bool)),
	}

	_, err := conn.DetachVolume(opts)
	if err != nil {
		return fmt.Errorf("Failed to detach Volume (%s) from Instance (%s): %s",
			vID, iID, err)
	}
	stateConf := &resource.StateChangeConf{
		Pending:    []string{"detaching"},
		Target:     []string{"detached"},
		Refresh:    volumeAttachmentStateRefreshFunc(conn, name, vID, iID),
		Timeout:    5 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	log.Printf("[DEBUG] Detaching Volume (%s) from Instance (%s)", vID, iID)
	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for Volume (%s) to detach from Instance: %s",
			vID, iID)
	}

	return nil
}

func volumeAttachmentID(name, volumeID, instanceID string) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("%s-", name))
	buf.WriteString(fmt.Sprintf("%s-", instanceID))
	buf.WriteString(fmt.Sprintf("%s-", volumeID))

	return fmt.Sprintf("vai-%d", hashcode.String(buf.String()))
}
<commit_msg>resource/aws_volume_attachment: Do not swallow error when detaching volume (#12596)<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/hashcode"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceAwsVolumeAttachment() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVolumeAttachmentCreate,
		Read:   resourceAwsVolumeAttachmentRead,
		Update: resourceAwsVolumeAttachmentUpdate,
		Delete: resourceAwsVolumeAttachmentDelete,

		Schema: map[string]*schema.Schema{
			"device_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"instance_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"volume_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"force_detach": {
				Type:     schema.TypeBool,
				Optional: true,
			},
			"skip_destroy": {
				Type:     schema.TypeBool,
				Optional: true,
			},
		},
	}
}

func resourceAwsVolumeAttachmentCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	name := d.Get("device_name").(string)
	iID := d.Get("instance_id").(string)
	vID := d.Get("volume_id").(string)

	// Find out if the volume is already attached to the instance, in which case
	// we have nothing to do
	request := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String(vID)},
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("attachment.instance-id"),
				Values: []*string{aws.String(iID)},
			},
			{
				Name:   aws.String("attachment.device"),
				Values: []*string{aws.String(name)},
			},
		},
	}

	vols, err := conn.DescribeVolumes(request)
	if (err != nil) || (len(vols.Volumes) == 0) {
		// This handles the situation where the instance is created by
		// a spot request and whilst the request has been fulfilled the
		// instance is not running yet
		stateConf := &resource.StateChangeConf{
			Pending:    []string{"pending", "stopping"},
			Target:     []string{"running", "stopped"},
			Refresh:    InstanceStateRefreshFunc(conn, iID, []string{"terminated"}),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		_, err = stateConf.WaitForState()
		if err != nil {
			return fmt.Errorf(
				"Error waiting for instance (%s) to become ready: %s",
				iID, err)
		}

		// not attached
		opts := &ec2.AttachVolumeInput{
			Device:     aws.String(name),
			InstanceId: aws.String(iID),
			VolumeId:   aws.String(vID),
		}

		log.Printf("[DEBUG] Attaching Volume (%s) to Instance (%s)", vID, iID)
		_, err := conn.AttachVolume(opts)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				return fmt.Errorf("Error attaching volume (%s) to instance (%s), message: \"%s\", code: \"%s\"",
					vID, iID, awsErr.Message(), awsErr.Code())
			}
			return err
		}
	}

	stateConf := &resource.StateChangeConf{
		Pending:    []string{"attaching"},
		Target:     []string{"attached"},
		Refresh:    volumeAttachmentStateRefreshFunc(conn, name, vID, iID),
		Timeout:    5 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for Volume (%s) to attach to Instance: %s, error: %s",
			vID, iID, err)
	}

	d.SetId(volumeAttachmentID(name, vID, iID))
	return resourceAwsVolumeAttachmentRead(d, meta)
}

func volumeAttachmentStateRefreshFunc(conn *ec2.EC2, name, volumeID, instanceID string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		request := &ec2.DescribeVolumesInput{
			VolumeIds: []*string{aws.String(volumeID)},
			Filters: []*ec2.Filter{
				{
					Name:   aws.String("attachment.device"),
					Values: []*string{aws.String(name)},
				},
				{
					Name:   aws.String("attachment.instance-id"),
					Values: []*string{aws.String(instanceID)},
				},
			},
		}

		resp, err := conn.DescribeVolumes(request)
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				return nil, "failed", fmt.Errorf("code: %s, message: %s", awsErr.Code(), awsErr.Message())
			}
			return nil, "failed", err
		}

		if len(resp.Volumes) > 0 {
			v := resp.Volumes[0]
			for _, a := range v.Attachments {
				if a.InstanceId != nil && *a.InstanceId == instanceID {
					return a, *a.State, nil
				}
			}
		}
		// assume detached if volume count is 0
		return 42, "detached", nil
	}
}

func resourceAwsVolumeAttachmentRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	request := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String(d.Get("volume_id").(string))},
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("attachment.device"),
				Values: []*string{aws.String(d.Get("device_name").(string))},
			},
			{
				Name:   aws.String("attachment.instance-id"),
				Values: []*string{aws.String(d.Get("instance_id").(string))},
			},
		},
	}

	vols, err := conn.DescribeVolumes(request)
	if err != nil {
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidVolume.NotFound" {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading EC2 volume %s for instance: %s: %#v", d.Get("volume_id").(string), d.Get("instance_id").(string), err)
	}

	if len(vols.Volumes) == 0 || *vols.Volumes[0].State == "available" {
		log.Printf("[DEBUG] Volume Attachment (%s) not found, removing from state", d.Id())
		d.SetId("")
	}

	return nil
}

func resourceAwsVolumeAttachmentUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Attaching Volume (%s) is updating which does nothing but updates a few params in state", d.Id())
	return nil
}

func resourceAwsVolumeAttachmentDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	if _, ok := d.GetOk("skip_destroy"); ok {
		return nil
	}

	name := d.Get("device_name").(string)
	vID := d.Get("volume_id").(string)
	iID := d.Get("instance_id").(string)

	opts := &ec2.DetachVolumeInput{
		Device:     aws.String(name),
		InstanceId: aws.String(iID),
		VolumeId:   aws.String(vID),
		Force:      aws.Bool(d.Get("force_detach").(bool)),
	}

	_, err := conn.DetachVolume(opts)
	if err != nil {
		return fmt.Errorf("Failed to detach Volume (%s) from Instance (%s): %s",
			vID, iID, err)
	}
	stateConf := &resource.StateChangeConf{
		Pending:    []string{"detaching"},
		Target:     []string{"detached"},
		Refresh:    volumeAttachmentStateRefreshFunc(conn, name, vID, iID),
		Timeout:    5 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	log.Printf("[DEBUG] Detaching Volume (%s) from Instance (%s)", vID, iID)
	_, err = stateConf.WaitForState()
	if err != nil {
		return fmt.Errorf(
			"Error waiting for Volume (%s) to detach from Instance (%s): %s",
			vID, iID, err)
	}

	return nil
}

func volumeAttachmentID(name, volumeID, instanceID string) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("%s-", name))
	buf.WriteString(fmt.Sprintf("%s-", instanceID))
	buf.WriteString(fmt.Sprintf("%s-", volumeID))

	return fmt.Sprintf("vai-%d", hashcode.String(buf.String()))
}
<|endoftext|>


<commit_before>package local

import (
	"context"
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/configs/configload"
	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/zclconf/go-cty/cty"
)

func TestLocal_refresh(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()

	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	// p.RefreshFn = nil
	// p.RefreshReturn = &terraform.InstanceState{ID: "yes"}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshNoConfig(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", &terraform.ProviderSchema{})
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	// p.RefreshFn = nil
	// p.RefreshReturn = &terraform.InstanceState{ID: "yes"}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/empty")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

// GH-12174
func TestLocal_refreshNilModuleWithInput(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", &terraform.ProviderSchema{})
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	// p.RefreshFn = nil
	// p.RefreshReturn = &terraform.InstanceState{ID: "yes"}

	b.OpInput = true

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/empty")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshInput(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", nil)
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	p.GetSchemaReturn = &terraform.ProviderSchema{
		Provider: &configschema.Block{
			Attributes: map[string]*configschema.Attribute{
				"value": {Type: cty.String, Optional: true},
			},
		},
		ResourceTypes: map[string]*configschema.Block{
			"test_instance": {
				Attributes: map[string]*configschema.Attribute{
					"foo": {Type: cty.String, Optional: true},
				},
			},
		},
	}
	p.ConfigureFn = func(c *terraform.ResourceConfig) error {
		if v, ok := c.Get("value"); !ok || v != "bar" {
			return fmt.Errorf("no value set")
		}

		return nil
	}

	// p.RefreshFn = nil
	// p.RefreshReturn = &terraform.InstanceState{ID: "yes"}

	// Enable input asking since it is normally disabled by default
	b.OpInput = true
	b.ContextOpts.UIInput = &terraform.MockUIInput{InputReturnString: "bar"}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh-var-unset")
	defer configCleanup()
	op.UIIn = b.ContextOpts.UIInput

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshValidate(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	// p.RefreshFn = nil
	// p.RefreshReturn = &terraform.InstanceState{ID: "yes"}

	// Enable validation
	b.OpValidation = true

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	// what are we validating?
	if !p.ValidateProviderConfigCalled {
		t.Fatal("validate should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func testOperationRefresh(t *testing.T, configDir string) (*backend.Operation, func()) {
	t.Helper()

	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)

	return &backend.Operation{
		Type:         backend.OperationTypeRefresh,
		ConfigDir:    configDir,
		ConfigLoader: configLoader,
	}, configCleanup
}

// testRefreshState is just a common state that we use for testing refresh.
func testRefreshState() *terraform.State {
	return &terraform.State{
		Version: 2,
		Modules: []*terraform.ModuleState{
			&terraform.ModuleState{
				Path: []string{"root"},
				Resources: map[string]*terraform.ResourceState{
					"test_instance.foo": &terraform.ResourceState{
						Type: "test_instance",
						Primary: &terraform.InstanceState{
							ID: "bar",
						},
					},
				},
				Outputs: map[string]*terraform.OutputState{},
			},
		},
	}
}

// refreshFixtureSchema returns a schema suitable for processing the
// configuration in test-fixtures/refresh . This schema should be
// assigned to a mock provider named "test".
func refreshFixtureSchema() *terraform.ProviderSchema {
	return &terraform.ProviderSchema{
		ResourceTypes: map[string]*configschema.Block{
			"test_instance": {
				Attributes: map[string]*configschema.Attribute{
					"ami": {Type: cty.String, Optional: true},
				},
			},
		},
	}
}
<commit_msg>backend/local refresh tests refactor<commit_after>package local

import (
	"context"
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/providers"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/configs/configload"
	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/terraform"
	"github.com/zclconf/go-cty/cty"
)

func TestLocal_refresh(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()

	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	p.ReadResourceFn = nil
	p.ReadResourceResponse = providers.ReadResourceResponse{NewState: cty.ObjectVal(map[string]cty.Value{
		"id": cty.StringVal("yes"),
	})}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshNoConfig(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())
	p.ReadResourceFn = nil
	p.ReadResourceResponse = providers.ReadResourceResponse{NewState: cty.ObjectVal(map[string]cty.Value{
		"id": cty.StringVal("yes"),
	})}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/empty")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

// GH-12174
func TestLocal_refreshNilModuleWithInput(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())
	p.ReadResourceFn = nil
	p.ReadResourceResponse = providers.ReadResourceResponse{NewState: cty.ObjectVal(map[string]cty.Value{
		"id": cty.StringVal("yes"),
	})}

	b.OpInput = true

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/empty")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshInput(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())

	p.GetSchemaReturn = &terraform.ProviderSchema{
		Provider: &configschema.Block{
			Attributes: map[string]*configschema.Attribute{
				"value": {Type: cty.String, Optional: true},
			},
		},
		ResourceTypes: map[string]*configschema.Block{
			"test_instance": {
				Attributes: map[string]*configschema.Attribute{
					"foo": {Type: cty.String, Optional: true},
					"id":  {Type: cty.String, Optional: true},
				},
			},
		},
	}
	p.ReadResourceFn = nil
	p.ReadResourceResponse = providers.ReadResourceResponse{NewState: cty.ObjectVal(map[string]cty.Value{
		"id": cty.StringVal("yes"),
	})}
	p.ConfigureFn = func(c *terraform.ResourceConfig) error {
		if v, ok := c.Get("value"); !ok || v != "bar" {
			return fmt.Errorf("no value set")
		}

		return nil
	}

	// Enable input asking since it is normally disabled by default
	b.OpInput = true
	b.ContextOpts.UIInput = &terraform.MockUIInput{InputReturnString: "bar"}

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh-var-unset")
	defer configCleanup()
	op.UIIn = b.ContextOpts.UIInput

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	if !p.ReadResourceCalled {
		t.Fatal("ReadResource should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func TestLocal_refreshValidate(t *testing.T) {
	b, cleanup := TestLocal(t)
	defer cleanup()
	p := TestLocalProvider(t, b, "test", refreshFixtureSchema())
	terraform.TestStateFile(t, b.StatePath, testRefreshState())
	p.ReadResourceFn = nil
	p.ReadResourceResponse = providers.ReadResourceResponse{NewState: cty.ObjectVal(map[string]cty.Value{
		"id": cty.StringVal("yes"),
	})}

	// Enable validation
	b.OpValidation = true

	op, configCleanup := testOperationRefresh(t, "./test-fixtures/refresh")
	defer configCleanup()

	run, err := b.Operation(context.Background(), op)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	<-run.Done()

	// what are we validating?
	if !p.ValidateProviderConfigCalled {
		t.Fatal("validate should be called")
	}

	checkState(t, b.StateOutPath, `
test_instance.foo:
  ID = yes
  provider = provider.test
	`)
}

func testOperationRefresh(t *testing.T, configDir string) (*backend.Operation, func()) {
	t.Helper()

	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)

	return &backend.Operation{
		Type:         backend.OperationTypeRefresh,
		ConfigDir:    configDir,
		ConfigLoader: configLoader,
	}, configCleanup
}

// testRefreshState is just a common state that we use for testing refresh.
func testRefreshState() *terraform.State {
	return &terraform.State{
		Version: 2,
		Modules: []*terraform.ModuleState{
			&terraform.ModuleState{
				Path: []string{"root"},
				Resources: map[string]*terraform.ResourceState{
					"test_instance.foo": &terraform.ResourceState{
						Type: "test_instance",
						Primary: &terraform.InstanceState{
							ID: "bar",
						},
					},
				},
				Outputs: map[string]*terraform.OutputState{},
			},
		},
	}
}

// refreshFixtureSchema returns a schema suitable for processing the
// configuration in test-fixtures/refresh . This schema should be
// assigned to a mock provider named "test".
func refreshFixtureSchema() *terraform.ProviderSchema {
	return &terraform.ProviderSchema{
		ResourceTypes: map[string]*configschema.Block{
			"test_instance": {
				Attributes: map[string]*configschema.Attribute{
					"ami": {Type: cty.String, Optional: true},
					"id":  {Type: cty.String, Computed: true},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package std

import (
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/mh-cbon/emd/emd"
)

// Register standard helpers to the generator.
func Register(g *emd.Generator) error {

	g.AddFunc("file", func(f string) (string, error) {
		s, err := ioutil.ReadFile(f)
		res := `
__> ` + f + `__
` + "```go" + `
` + strings.TrimSpace(string(s)) + `
` + "```"
		return res, err
	})

	g.AddFunc("cli", func(bin string, args ...string) (string, error) {
		cmd := exec.Command(bin, args...)
		out, err := cmd.CombinedOutput()
		fbin := filepath.Base(bin)
		res := `
__$ ` + fbin + ` ` + strings.Join(args, " ") + `__
` + "```sh" + `
` + strings.TrimSpace(string(out)) + `
` + "```"
		return res, err
	})

	g.AddTemplate(`{{define "gh/releases" -}}
Check the [release page](https://github.com/{{.User}}/{{.Name}}/releases)!
{{- end}}`)

	g.AddTemplate(`{{define "badge/travis" -}}
[![travis Status](https://travis-ci.org/{{.User}}/{{.Name}}.svg?branch={{.Branch}})](https://travis-ci.org/{{.User}}/{{.Name}})
{{- end}}`)

	g.AddTemplate(`{{define "badge/appveyor" -}}
[![appveyor Status](https://ci.appveyor.com/api/projects/status/.ProviderName/{{.User}}/{{.Name}}?branch={{.Branch}}&svg=true)](https://ci.appveyor.com/api/projects/status/.ProviderName/{{.User}}/{{.Name}}?branch={{.Branch}}&svg=true)
{{- end}}`)

	g.AddTemplate(`{{define "choco/install" -}}
` + "```sh" + `
choco install {{.Name}}
` + "```" + `
{{- end}}`)

	g.AddTemplate(`{{define "linux/gh_src_repo" -}}
` + "```sh" + `
wget -O - https://raw.githubusercontent.com/mh-cbon/latest/master/source.sh \
| GH={{.User}}/{{.Name}} sh -xe
# or
curl -L https://raw.githubusercontent.com/mh-cbon/latest/master/source.sh \
| GH={{.User}}/{{.Name}} sh -xe
` + "```" + `
{{- end}}`)

	g.AddTemplate(`{{define "linux/gh_pkg" -}}
` + "```sh" + `
curl -L https://raw.githubusercontent.com/mh-cbon/latest/master/install.sh \
| GH={{.User}}/{{.Name}} sh -xe
# or
wget -q -O - --no-check-certificate \
https://raw.githubusercontent.com/mh-cbon/latest/master/install.sh \
| GH={{.User}}/{{.Name}} sh -xe
` + "```" + `
{{- end}}`)

	return nil
}
<commit_msg>badge(fix): fix appveyor badge<commit_after>package std

import (
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/mh-cbon/emd/emd"
)

// Register standard helpers to the generator.
func Register(g *emd.Generator) error {

	g.AddFunc("file", func(f string) (string, error) {
		s, err := ioutil.ReadFile(f)
		res := `
__> ` + f + `__
` + "```go" + `
` + strings.TrimSpace(string(s)) + `
` + "```"
		return res, err
	})

	g.AddFunc("cli", func(bin string, args ...string) (string, error) {
		cmd := exec.Command(bin, args...)
		out, err := cmd.CombinedOutput()
		fbin := filepath.Base(bin)
		res := `
__$ ` + fbin + ` ` + strings.Join(args, " ") + `__
` + "```sh" + `
` + strings.TrimSpace(string(out)) + `
` + "```"
		return res, err
	})

	g.AddTemplate(`{{define "gh/releases" -}}
Check the [release page](https://github.com/{{.User}}/{{.Name}}/releases)!
{{- end}}`)

	g.AddTemplate(`{{define "badge/travis" -}}
[![travis Status](https://travis-ci.org/{{.User}}/{{.Name}}.svg?branch={{.Branch}})](https://travis-ci.org/{{.User}}/{{.Name}})
{{- end}}`)

	g.AddTemplate(`{{define "badge/appveyor" -}}
[![appveyor Status](https://ci.appveyor.com/api/projects/status/{{.ProviderName}}/{{.User}}/{{.Name}}?branch={{.Branch}}&svg=true)](https://ci.appveyor.com/api/projects/status/{{.ProviderName}}/{{.User}}/{{.Name}}?branch={{.Branch}}&svg=true)
{{- end}}`)

	g.AddTemplate(`{{define "choco/install" -}}
` + "```sh" + `
choco install {{.Name}}
` + "```" + `
{{- end}}`)

	g.AddTemplate(`{{define "linux/gh_src_repo" -}}
` + "```sh" + `
wget -O - https://raw.githubusercontent.com/mh-cbon/latest/master/source.sh \
| GH={{.User}}/{{.Name}} sh -xe
# or
curl -L https://raw.githubusercontent.com/mh-cbon/latest/master/source.sh \
| GH={{.User}}/{{.Name}} sh -xe
` + "```" + `
{{- end}}`)

	g.AddTemplate(`{{define "linux/gh_pkg" -}}
` + "```sh" + `
curl -L https://raw.githubusercontent.com/mh-cbon/latest/master/install.sh \
| GH={{.User}}/{{.Name}} sh -xe
# or
wget -q -O - --no-check-certificate \
https://raw.githubusercontent.com/mh-cbon/latest/master/install.sh \
| GH={{.User}}/{{.Name}} sh -xe
` + "```" + `
{{- end}}`)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"time"
)

var (
	gistsApiEndpoint = "https://api.github.com/gists"
)

type gistReqBody struct {
	Public bool                         `json:"public"`
	Files  map[string]map[string]string `json:"files"`
}

type gistResBody struct {
	HtmlUrl string `json:"html_url"`
}

func main() {
	var content string
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		content += scanner.Text() + "\n"
	}

	var data bytes.Buffer
	fileName := fmt.Sprintf("stdgist-%s", strconv.FormatInt(time.Now().Unix(), 10))
	body := gistReqBody{
		Public: true,
		Files: map[string]map[string]string{
			fileName: map[string]string{
				"content": content,
			},
		},
	}

	encoder := json.NewEncoder(&data)
	err := encoder.Encode(body)
	if err != nil {
		panic(err)
	}

	res, err := http.Post(gistsApiEndpoint, "application/json", &data)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	var resBody gistResBody
	decoder := json.NewDecoder(res.Body)
	err = decoder.Decode(&resBody)
	if err != nil {
		panic(err)
	}

	if resBody.HtmlUrl != "" {
		fmt.Println(resBody.HtmlUrl)
	}
}
<commit_msg>Add file name support.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"
)

var (
	gistsApiEndpoint = "https://api.github.com/gists"
	fileName         = fmt.Sprintf("stdgist-%d", time.Now().Unix())
)

const usage = `Standard Gist

Usage: stdgist [options]

stdgist reads from standard input and posts the given
input as a gist.

Options:
  --name, -n  Name for the file.
`

type gistReqBody struct {
	Public bool                         `json:"public"`
	Files  map[string]map[string]string `json:"files"`
}

type gistResBody struct {
	HtmlUrl string `json:"html_url"`
}

func init() {
	flag.StringVar(&fileName, "name", fileName, "")
	flag.StringVar(&fileName, "n", fileName, "")
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, usage)
	}
}

func main() {
	flag.Parse()

	stat, err := os.Stdin.Stat()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Check if stdin is from a terminal (i.e. no input to read).
	if stat.Mode()&os.ModeCharDevice != 0 {
		fmt.Fprintln(os.Stderr, "Content needs to be given to stdin to use")
		os.Exit(1)
	}

	var content bytes.Buffer
	_, err = io.Copy(&content, os.Stdin)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	body := gistReqBody{
		Public: true,
		Files: map[string]map[string]string{
			fileName: map[string]string{
				"content": content.String(),
			},
		},
	}

	content.Reset()
	encoder := json.NewEncoder(&content)
	err = encoder.Encode(body)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	res, err := http.Post(gistsApiEndpoint, "application/json", &content)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	defer res.Body.Close()

	var resBody gistResBody
	decoder := json.NewDecoder(res.Body)
	err = decoder.Decode(&resBody)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if resBody.HtmlUrl != "" {
		fmt.Println(resBody.HtmlUrl)
	}
}
<|endoftext|>


<commit_before>package stealer

import "errors"

type Stealer struct {
	Datas map[string][]string
}

// Steal some variable data from path phpfile
func Steal(path string) (error, *Stealer) {
	var stealer Stealer
	var err error
	err, stealer.Datas = ReadFile(path)
	return err, &stealer
}

// save all the variables and its values to new path
// TODO : write test case where the path is not exist and need to create the path first
// lets say path/to/specific/file.go need to create the folder first
func (s *Stealer) Save(savePath, packageName string) error {
	var err error
	if s.Datas == nil {
		err = errors.New("Stealer Datas are empty")
		return err
	}
	err = WriteFile(savePath, packageName, s.Datas)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>handle empty parameters<commit_after>package stealer

import "errors"

type Stealer struct {
	Datas map[string][]string
}

// Steal some variable data from path phpfile
func Steal(path string) (error, *Stealer) {
	var stealer Stealer
	var err error

	if path == "" {
		err = errors.New("Given path is empty")
		return err, nil
	}

	err, stealer.Datas = ReadFile(path)
	return err, &stealer
}

// save all the variables and its values to new path
// TODO : write test case where the path is not exist and need to create the path first
// lets say path/to/specific/file.go need to create the folder first
func (s *Stealer) Save(savePath, packageName string) error {
	var err error

	if s.Datas == nil {
		err = errors.New("Stealer Datas are empty")
		return err
	}

	if savePath == "" || packageName == "" {
		err = errors.New("SavePath or PackageName must not be empty")
		return err
	}

	err = WriteFile(savePath, packageName, s.Datas)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package storageconsul

import (
	"context"
	"fmt"
	"io/fs"
	"net"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/caddyserver/certmagic"
	consul "github.com/hashicorp/consul/api"
	"github.com/pteich/errors"
	"go.uber.org/zap"
)

// ConsulStorage allows to store certificates and other TLS resources
// in a shared cluster environment using Consul's key/value-store.
// It uses distributed locks to ensure consistency.
type ConsulStorage struct {
	certmagic.Storage
	ConsulClient *consul.Client
	logger       *zap.SugaredLogger
	muLocks      *sync.RWMutex
	locks        map[string]*consul.Lock

	Address     string `json:"address"`
	Token       string `json:"token"`
	Timeout     int    `json:"timeout"`
	Prefix      string `json:"prefix"`
	ValuePrefix string `json:"value_prefix"`
	AESKey      []byte `json:"aes_key"`
	TlsEnabled  bool   `json:"tls_enabled"`
	TlsInsecure bool   `json:"tls_insecure"`
}

// New connects to Consul and returns a ConsulStorage
func New() *ConsulStorage {
	// create ConsulStorage and pre-set values
	s := ConsulStorage{
		locks:       make(map[string]*consul.Lock),
		AESKey:      []byte(DefaultAESKey),
		ValuePrefix: DefaultValuePrefix,
		Prefix:      DefaultPrefix,
		Timeout:     DefaultTimeout,
		muLocks:     &sync.RWMutex{},
	}

	return &s
}

func (cs *ConsulStorage) prefixKey(key string) string {
	return path.Join(cs.Prefix, key)
}

// Lock acquires a distributed lock for the given key or blocks until it gets one
func (cs *ConsulStorage) Lock(ctx context.Context, key string) error {
	cs.logger.Debugf("trying lock for %s", key)

	if _, isLocked := cs.GetLock(key); isLocked {
		return nil
	}

	// prepare the distributed lock
	cs.logger.Infof("creating Consul lock for %s", key)
	lock, err := cs.ConsulClient.LockOpts(&consul.LockOptions{
		Key:          cs.prefixKey(key),
		LockWaitTime: time.Duration(cs.Timeout) * time.Second,
		LockTryOnce:  true,
	})
	if err != nil {
		return errors.Wrapf(err, "could not create lock for %s", cs.prefixKey(key))
	}

	// acquire the lock and return a channel that is closed upon lost
	lockActive, err := lock.Lock(ctx.Done())
	if err != nil {
		return errors.Wrapf(err, "unable to lock %s", cs.prefixKey(key))
	}

	// auto-unlock and clean list of locks in case of lost
	go func() {
		<-lockActive
		err := cs.Unlock(ctx, key)
		if err != nil {
			cs.logger.Errorf("failed to release lock: %s", err)
		}
	}()

	// save the lock
	cs.muLocks.Lock()
	cs.locks[key] = lock
	cs.muLocks.Unlock()

	return nil
}

func (cs *ConsulStorage) GetLock(key string) (*consul.Lock, bool) {
	cs.muLocks.RLock()
	defer cs.muLocks.RUnlock()

	// if we already hold the lock, return early
	if lock, exists := cs.locks[key]; exists {
		return lock, true
	}

	return nil, false
}

// Unlock releases a specific lock
func (cs *ConsulStorage) Unlock(_ context.Context, key string) error {
	// check if we own it and unlock
	lock, exists := cs.GetLock(key)
	if !exists {
		return errors.Errorf("lock %s not found", cs.prefixKey(key))
	}

	err := lock.Unlock()
	if err != nil {
		return errors.Wrapf(err, "unable to unlock %s", cs.prefixKey(key))
	}

	cs.muLocks.Lock()
	delete(cs.locks, key)
	cs.muLocks.Unlock()

	return nil
}

// Store saves encrypted data value for a key in Consul KV
func (cs ConsulStorage) Store(ctx context.Context, key string, value []byte) error {
	kv := &consul.KVPair{Key: cs.prefixKey(key)}

	// prepare the stored data
	consulData := &StorageData{
		Value:    value,
		Modified: time.Now(),
	}

	encryptedValue, err := cs.EncryptStorageData(consulData)
	if err != nil {
		return errors.Wrapf(err, "unable to encode data for %s", cs.prefixKey(key))
	}

	kv.Value = encryptedValue

	opts := consul.WriteOptions{}
	if _, err = cs.ConsulClient.KV().Put(kv, opts.WithContext(ctx)); err != nil {
		return errors.Wrapf(err, "unable to store data for %s", cs.prefixKey(key))
	}

	return nil
}

// Load retrieves the value for a key from Consul KV
func (cs ConsulStorage) Load(ctx context.Context, key string) ([]byte, error) {
	cs.logger.Debugf("loading data from Consul for %s", key)

	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return nil, err
	}

	if kv == nil {
		return nil, fs.ErrNotExist
	}

	contents, err := cs.DecryptStorageData(kv.Value)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to decrypt data for %s", cs.prefixKey(key))
	}

	return contents.Value, nil
}

// Delete a key from Consul KV
func (cs ConsulStorage) Delete(ctx context.Context, key string) error {
	cs.logger.Infof("deleting key %s from Consul", key)

	// first obtain existing keypair
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return fmt.Errorf("%s: %w", err, fs.ErrNotExist)
	}

	if kv == nil {
		return fs.ErrNotExist
	}

	// now do a Check-And-Set operation to verify we really deleted the key
	if success, _, err := cs.ConsulClient.KV().DeleteCAS(kv, nil); err != nil {
		return errors.Wrapf(err, "unable to delete data for %s", cs.prefixKey(key))
	} else if !success {
		return errors.Errorf("failed to lock data delete for %s", cs.prefixKey(key))
	}

	return nil
}

// Exists checks if a key exists
func (cs ConsulStorage) Exists(ctx context.Context, key string) bool {
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if kv != nil && err == nil {
		return true
	}
	return false
}

// List returns a list with all keys under a given prefix
func (cs ConsulStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
	var keysFound []string

	// get a list of all keys at prefix
	keys, _, err := cs.ConsulClient.KV().Keys(cs.prefixKey(prefix), "", ConsulQueryDefaults(ctx))
	if err != nil {
		return keysFound, err
	}

	if len(keys) == 0 {
		return keysFound, fs.ErrNotExist
	}

	// remove default prefix from keys
	for _, key := range keys {
		if strings.HasPrefix(key, cs.prefixKey(prefix)) {
			key = strings.TrimPrefix(key, cs.Prefix+"/")
			keysFound = append(keysFound, key)
		}
	}

	// if recursive wanted, just return all keys
	if recursive {
		return keysFound, nil
	}

	// for non-recursive split path and look for unique keys just under given prefix
	keysMap := make(map[string]bool)
	for _, key := range keysFound {
		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
		keysMap[dir[0]] = true
	}

	keysFound = make([]string, 0)
	for key := range keysMap {
		keysFound = append(keysFound, path.Join(prefix, key))
	}

	return keysFound, nil
}

// Stat returns statistic data of a key
func (cs ConsulStorage) Stat(ctx context.Context, key string) (certmagic.KeyInfo, error) {
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return certmagic.KeyInfo{}, fmt.Errorf("unable to obtain data for %s: %w", cs.prefixKey(key), fs.ErrNotExist)
	}
	if kv == nil {
		return certmagic.KeyInfo{}, fs.ErrNotExist
	}

	contents, err := cs.DecryptStorageData(kv.Value)
	if err != nil {
		return certmagic.KeyInfo{}, errors.Errorf("unable to decrypt data for %s", cs.prefixKey(key))
	}

	return certmagic.KeyInfo{
		Key:        key,
		Modified:   contents.Modified,
		Size:       int64(len(contents.Value)),
		IsTerminal: false,
	}, nil
}

func (cs *ConsulStorage) createConsulClient() error {
	// get the default config
	consulCfg := consul.DefaultConfig()
	if cs.Address != "" {
		consulCfg.Address = cs.Address
	}
	if cs.Token != "" {
		consulCfg.Token = cs.Token
	}
	if cs.TlsEnabled {
		consulCfg.Scheme = "https"
	}
	consulCfg.TLSConfig.InsecureSkipVerify = cs.TlsInsecure

	// set a dial context to prevent default keepalive
	consulCfg.Transport.DialContext = (&net.Dialer{
		Timeout:   time.Duration(cs.Timeout) * time.Second,
		KeepAlive: time.Duration(cs.Timeout) * time.Second,
	}).DialContext

	// create the Consul API client
	consulClient, err := consul.NewClient(consulCfg)
	if err != nil {
		return errors.Wrap(err, "unable to create Consul client")
	}

	if _, err := consulClient.Agent().NodeName(); err != nil {
		return errors.Wrap(err, "unable to ping Consul")
	}

	cs.ConsulClient = consulClient
	return nil
}

func ConsulQueryDefaults(ctx context.Context) *consul.QueryOptions {
	opts := &consul.QueryOptions{
		UseCache:          true,
		RequireConsistent: true,
	}
	return opts.WithContext(ctx)
}
<commit_msg>Tweak Consul access performance<commit_after>package storageconsul

import (
	"context"
	"fmt"
	"io/fs"
	"net"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/caddyserver/certmagic"
	consul "github.com/hashicorp/consul/api"
	"github.com/pteich/errors"
	"go.uber.org/zap"
)

// ConsulStorage allows to store certificates and other TLS resources
// in a shared cluster environment using Consul's key/value-store.
// It uses distributed locks to ensure consistency.
type ConsulStorage struct {
	certmagic.Storage
	ConsulClient *consul.Client
	logger       *zap.SugaredLogger
	muLocks      *sync.RWMutex
	locks        map[string]*consul.Lock

	Address     string `json:"address"`
	Token       string `json:"token"`
	Timeout     int    `json:"timeout"`
	Prefix      string `json:"prefix"`
	ValuePrefix string `json:"value_prefix"`
	AESKey      []byte `json:"aes_key"`
	TlsEnabled  bool   `json:"tls_enabled"`
	TlsInsecure bool   `json:"tls_insecure"`
}

// New connects to Consul and returns a ConsulStorage
func New() *ConsulStorage {
	// create ConsulStorage and pre-set values
	s := ConsulStorage{
		locks:       make(map[string]*consul.Lock),
		AESKey:      []byte(DefaultAESKey),
		ValuePrefix: DefaultValuePrefix,
		Prefix:      DefaultPrefix,
		Timeout:     DefaultTimeout,
		muLocks:     &sync.RWMutex{},
	}

	return &s
}

func (cs *ConsulStorage) prefixKey(key string) string {
	return path.Join(cs.Prefix, key)
}

// Lock acquires a distributed lock for the given key or blocks until it gets one
func (cs *ConsulStorage) Lock(ctx context.Context, key string) error {
	cs.logger.Debugf("trying lock for %s", key)

	if _, isLocked := cs.GetLock(key); isLocked {
		return nil
	}

	// prepare the distributed lock
	cs.logger.Infof("creating Consul lock for %s", key)
	lock, err := cs.ConsulClient.LockOpts(&consul.LockOptions{
		Key:          cs.prefixKey(key),
		LockWaitTime: time.Duration(cs.Timeout) * time.Second,
		LockTryOnce:  true,
	})
	if err != nil {
		return errors.Wrapf(err, "could not create lock for %s", cs.prefixKey(key))
	}

	// acquire the lock and return a channel that is closed upon lost
	lockActive, err := lock.Lock(ctx.Done())
	if err != nil {
		return errors.Wrapf(err, "unable to lock %s", cs.prefixKey(key))
	}

	// auto-unlock and clean list of locks in case of lost
	go func() {
		<-lockActive
		err := cs.Unlock(ctx, key)
		if err != nil {
			cs.logger.Errorf("failed to release lock: %s", err)
		}
	}()

	// save the lock
	cs.muLocks.Lock()
	cs.locks[key] = lock
	cs.muLocks.Unlock()

	return nil
}

func (cs *ConsulStorage) GetLock(key string) (*consul.Lock, bool) {
	cs.muLocks.RLock()
	defer cs.muLocks.RUnlock()

	// if we already hold the lock, return early
	if lock, exists := cs.locks[key]; exists {
		return lock, true
	}

	return nil, false
}

// Unlock releases a specific lock
func (cs *ConsulStorage) Unlock(_ context.Context, key string) error {
	// check if we own it and unlock
	lock, exists := cs.GetLock(key)
	if !exists {
		return errors.Errorf("lock %s not found", cs.prefixKey(key))
	}

	err := lock.Unlock()
	if err != nil {
		return errors.Wrapf(err, "unable to unlock %s", cs.prefixKey(key))
	}

	cs.muLocks.Lock()
	delete(cs.locks, key)
	cs.muLocks.Unlock()

	return nil
}

// Store saves encrypted data value for a key in Consul KV
func (cs ConsulStorage) Store(ctx context.Context, key string, value []byte) error {
	kv := &consul.KVPair{Key: cs.prefixKey(key)}

	// prepare the stored data
	consulData := &StorageData{
		Value:    value,
		Modified: time.Now(),
	}

	encryptedValue, err := cs.EncryptStorageData(consulData)
	if err != nil {
		return errors.Wrapf(err, "unable to encode data for %s", cs.prefixKey(key))
	}

	kv.Value = encryptedValue

	opts := consul.WriteOptions{}
	if _, err = cs.ConsulClient.KV().Put(kv, opts.WithContext(ctx)); err != nil {
		return errors.Wrapf(err, "unable to store data for %s", cs.prefixKey(key))
	}

	return nil
}

// Load retrieves the value for a key from Consul KV
func (cs ConsulStorage) Load(ctx context.Context, key string) ([]byte, error) {
	cs.logger.Debugf("loading data from Consul for %s", key)

	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return nil, err
	}

	if kv == nil {
		return nil, fs.ErrNotExist
	}

	contents, err := cs.DecryptStorageData(kv.Value)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to decrypt data for %s", cs.prefixKey(key))
	}

	return contents.Value, nil
}

// Delete a key from Consul KV
func (cs ConsulStorage) Delete(ctx context.Context, key string) error {
	cs.logger.Infof("deleting key %s from Consul", key)

	// first obtain existing keypair
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return fmt.Errorf("%s: %w", err, fs.ErrNotExist)
	}

	if kv == nil {
		return fs.ErrNotExist
	}

	// now do a Check-And-Set operation to verify we really deleted the key
	if success, _, err := cs.ConsulClient.KV().DeleteCAS(kv, nil); err != nil {
		return errors.Wrapf(err, "unable to delete data for %s", cs.prefixKey(key))
	} else if !success {
		return errors.Errorf("failed to lock data delete for %s", cs.prefixKey(key))
	}

	return nil
}

// Exists checks if a key exists
func (cs ConsulStorage) Exists(ctx context.Context, key string) bool {
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if kv != nil && err == nil {
		return true
	}
	return false
}

// List returns a list with all keys under a given prefix
func (cs ConsulStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
	var keysFound []string

	// get a list of all keys at prefix
	keys, _, err := cs.ConsulClient.KV().Keys(cs.prefixKey(prefix), "", ConsulQueryDefaults(ctx))
	if err != nil {
		return keysFound, err
	}

	if len(keys) == 0 {
		return keysFound, fs.ErrNotExist
	}

	// remove default prefix from keys
	for _, key := range keys {
		if strings.HasPrefix(key, cs.prefixKey(prefix)) {
			key = strings.TrimPrefix(key, cs.Prefix+"/")
			keysFound = append(keysFound, key)
		}
	}

	// if recursive wanted, just return all keys
	if recursive {
		return keysFound, nil
	}

	// for non-recursive split path and look for unique keys just under given prefix
	keysMap := make(map[string]bool)
	for _, key := range keysFound {
		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
		keysMap[dir[0]] = true
	}

	keysFound = make([]string, 0)
	for key := range keysMap {
		keysFound = append(keysFound, path.Join(prefix, key))
	}

	return keysFound, nil
}

// Stat returns statistic data of a key
func (cs ConsulStorage) Stat(ctx context.Context, key string) (certmagic.KeyInfo, error) {
	kv, _, err := cs.ConsulClient.KV().Get(cs.prefixKey(key), ConsulQueryDefaults(ctx))
	if err != nil {
		return certmagic.KeyInfo{}, fmt.Errorf("unable to obtain data for %s: %w", cs.prefixKey(key), fs.ErrNotExist)
	}
	if kv == nil {
		return certmagic.KeyInfo{}, fs.ErrNotExist
	}

	contents, err := cs.DecryptStorageData(kv.Value)
	if err != nil {
		return certmagic.KeyInfo{}, errors.Errorf("unable to decrypt data for %s", cs.prefixKey(key))
	}

	return certmagic.KeyInfo{
		Key:        key,
		Modified:   contents.Modified,
		Size:       int64(len(contents.Value)),
		IsTerminal: false,
	}, nil
}

func (cs *ConsulStorage) createConsulClient() error {
	// get the default config
	consulCfg := consul.DefaultConfig()
	if cs.Address != "" {
		consulCfg.Address = cs.Address
	}
	if cs.Token != "" {
		consulCfg.Token = cs.Token
	}
	if cs.TlsEnabled {
		consulCfg.Scheme = "https"
	}
	consulCfg.TLSConfig.InsecureSkipVerify = cs.TlsInsecure

	// set a dial context to prevent default keepalive
	consulCfg.Transport.DialContext = (&net.Dialer{
		Timeout:   time.Duration(cs.Timeout) * time.Second,
		KeepAlive: time.Duration(cs.Timeout) * time.Second,
	}).DialContext

	// create the Consul API client
	consulClient, err := consul.NewClient(consulCfg)
	if err != nil {
		return errors.Wrap(err, "unable to create Consul client")
	}

	if _, err := consulClient.Agent().NodeName(); err != nil {
		return errors.Wrap(err, "unable to ping Consul")
	}

	cs.ConsulClient = consulClient
	return nil
}

func ConsulQueryDefaults(ctx context.Context) *consul.QueryOptions {
	opts := &consul.QueryOptions{
		UseCache:          false,
		RequireConsistent: false,
	}
	return opts.WithContext(ctx)
}
<|endoftext|>


<commit_before>package hello

import (
	"fmt"
	"net/http"
	"time"
)

func init() {
	http.HandleFunc("/svg/month", handler)
}

func handler(w http.ResponseWriter, r *http.Request) {

	text := r.URL.Query().Get("text")
	width := r.URL.Query().Get("width")
	if width == "" {
		width = "300"
	}

	height := r.URL.Query().Get("height")
	if height == "" {
		height = "20"
	}

	dx := r.URL.Query().Get("dx")

	if dx == "" {
		dx = "0"
	}

	dy := r.URL.Query().Get("dy")

	if dy == "" {
		dy = "-2.6em"
	}

	fill := r.URL.Query().Get("fill")

	if fill == "" {
		fill = "000000"
	}

	format := r.URL.Query().Get("format")

	if format == "" {
		format = "January 2006"
	}

	size := r.URL.Query().Get("size")

	if size == "" {
		size = "ANY SIZE"
	}

	if text == "" {
		text = time.Now().Format(format)
	} else {
		text = text + " " + time.Now().Format(format)
	}
	w.Header().Set("Content-Type", "image/svg+xml")

	output := fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"  viewBox="0 0 %s %s">
    <text text-anchor="left"  style="font-family: Arial, Helvetica, Verdana" font-weight="bold" fill="#%s" dx='%s' dy='%s' font-size="%s">%s</text>
</svg>`, width, height, fill, dx, dy, size, text)

	//  w.Header().Set("Content-Length", fmt.Sprintf("%d",len(output)))
	fmt.Fprint(w, output)
}
<commit_msg>setup<commit_after>package hello

import (
	"fmt"
	"net/http"
	"strconv"
	"time"
)

func init() {
	http.HandleFunc("/svg/month", mthHandler)
	http.HandleFunc("/svg/mth", mthHandler)
	http.HandleFunc("/svg/date", dateHandler)
}

func mthHandler(w http.ResponseWriter, r *http.Request) {

	text := r.URL.Query().Get("text")
	size := r.URL.Query().Get("size")

	if size == "" {
		size = "30"
	}

	y, err := strconv.ParseFloat(size, 64)
	if err != nil {
		y = 20
	} else {
		y = y / 1.5
	}
	width := r.URL.Query().Get("width")
	if width == "" {
		width = "600"
	}

	x, err := strconv.Atoi(width)
	if err != nil {
		x = 150
	} else {
		x = x / 2
	}

	height := r.URL.Query().Get("height")
	if height == "" {
		height = "20"
	}

	dx := r.URL.Query().Get("dx")

	if dx == "" {
		dx = "0"
	}

	dy := r.URL.Query().Get("dy")

	if dy == "" {
		dy = fmt.Sprintf("%f", y)
	}

	fill := r.URL.Query().Get("fill")

	if fill == "" {
		fill = "000000"
	}

	format := r.URL.Query().Get("format")

	if format == "" {
		format = "January 2006"
	}

	if text == "" {
		text = time.Now().Format(format)
	} else {
		text = text + " " + time.Now().Format(format)
	}
	w.Header().Set("Content-Type", "image/svg+xml")

	output := fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"  viewBox="0 0 %s %s" >
    <text dominant-baseline="top" text-anchor="middle" x='%s' y='%s' style="font-family: Arial, Helvetica, Verdana" font-weight="bold" fill="#%s" dx='%s' dy='%s' font-size="%s">%s</text>
</svg>`, width, height, fmt.Sprintf("%d", x), fmt.Sprintf("%d", y), fill, dx, dy, size, text)

	//  w.Header().Set("Content-Length", fmt.Sprintf("%f",len(output)))
	fmt.Fprint(w, output)
}

func dateHandler(w http.ResponseWriter, r *http.Request) {

	text := r.URL.Query().Get("text")
	size := r.URL.Query().Get("size")

	if size == "" {
		size = "20"
	}

	y, err := strconv.ParseFloat(size, 64)
	if err != nil {
		y = 20
	} else {
		y = y / 1.15
	}
	width := r.URL.Query().Get("width")
	if width == "" {
		width = "100"
	}

	x, err := strconv.Atoi(width)
	if err != nil {
		x = 100
	} else {

	}

	height := r.URL.Query().Get("height")
	if height == "" {
		height = "20"
	}

	dx := r.URL.Query().Get("dx")

	if dx == "" {
		dx = "0"
	}

	dy := r.URL.Query().Get("dy")

	if dy == "" {
		dy = fmt.Sprintf("%f", y)
	}

	fill := r.URL.Query().Get("fill")

	if fill == "" {
		fill = "000000"
	}

	format := r.URL.Query().Get("format")

	if format == "" {
		format = "2006-01-02"
	}

	if text == "" {
		text = time.Now().Format(format)
	} else {
		text = text + " " + time.Now().Format(format)
	}
	w.Header().Set("Content-Type", "image/svg+xml")

	output := fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"  viewBox="0 0 %s %s" >
    <text   text-anchor="end" x='%s' y='%s' style="font-family: Arial, Helvetica, Verdana" font-weight="bold" fill="#%s" dx='%s' dy='%s' font-size="%s">%s</text>
</svg>`, width, height, fmt.Sprintf("%d", x), fmt.Sprintf("%d", y), fill, dx, dy, size, text)

	//  w.Header().Set("Content-Length", fmt.Sprintf("%f",len(output)))
	fmt.Fprint(w, output)
}
<|endoftext|>


<commit_before>// Package svm provides a set of svm types and functions.
package svm

import (
	"fmt"
	"math"
	"math/rand"

	"github.com/santiaago/ml"
)

// SVM holds all the information needed to run a support vector machine algorithm.
//
type SVM struct {
	TrainingPoints int         // number of training points.
	Xn             [][]float64 // data set of points for training.
	Yn             []float64   // output, evaluation of each Xi based on input data/ training examples.
	Wn             []float64   // weight vector.
	VectorSize     int         // size of vectors Xi and Wi.
	Lambda         float64     // used in the learning algorithm.
	Eta            float64     // used in the learning algorithm.
	K              int         // used in the learning algorithm, it is the block size to use when running Pegasos
}

// NewSVM creates a support vector machine object.
//
func NewSVM() *SVM {
	return &SVM{K: 1}
}

// InitializeFromData reads a 2 dimentional array with the following format:
// x1 x2 y
// x1 x2 y
// x1 x2 y
// And sets Xn and Yn accordingly
//
func (svm *SVM) InitializeFromData(data [][]float64) error {

	n := 0
	svm.Yn = make([]float64, len(data))
	svm.Xn = make([][]float64, len(data))

	for i, sample := range data {

		svm.Xn[i] = make([]float64, len(sample))
		svm.Xn[i] = []float64{1}
		svm.Xn[i] = append(svm.Xn[i], sample[:len(sample)-1]...)

		svm.Yn[i] = sample[len(sample)-1]
		n++
	}

	svm.TrainingPoints = n
	svm.VectorSize = len(svm.Xn[0])
	svm.Wn = make([]float64, svm.VectorSize)

	return nil
}

// Learn will update the weight vector with the output of a svm algorithm,
// with respect of the training examples and labels (svm.Xn, svm.Yn).
//
// We use an implementation of the Mini-Batch Pegasos Algorithm.
//
// The implementation of the Mini-Batch Pegasos Algorithm is based on the work done by:
// Shalev-Shwartz, Shai and Singer, Yoram and Srebro, Nathan.
// Pegasos: Primal Estimated sub-GrAdient SOlver for SVM.
//
// Note: We might support other methods in the future like SMO, ...
//
func (svm *SVM) Learn() error {

	var err error
	svm.TrainingPoints = len(svm.Xn)
	svm.Wn = make([]float64, svm.TrainingPoints)

	for t := range svm.Xn {

		// choose At where |At| = k, uniformly at random
		var At []int // vector of the selected indexes of size K
		for i := 0; i < svm.K; i++ {
			at := rand.Intn(svm.TrainingPoints)
			At = append(At, at)
		}

		// set At+ = {i in At : yi<wt, xi> < 1}
		var Atplus []int
		for _, i := range At {

			xi, yi := ml.Vector(svm.Xn[i]), svm.Yn[i]
			wt := ml.Vector(svm.Wn)
			var dot float64

			if dot, err = wt.Dot(xi); err != nil {
				return err
			}

			if yi*dot < 1 {
				Atplus = append(Atplus, i)
			}
		}

		// update eta = 1 / lambda*t
		svm.Eta = float64(1) / (svm.Lambda * float64(t+1))

		// set wt+1 = (1 - eta*lambda)*wt + eta/k * sum(for i in At+ of yixi)
		term1 := ml.Vector(svm.Wn).Scale(1 - svm.Eta*svm.Lambda)
		term2 := ml.Vector(make([]float64, svm.VectorSize))
		for _, i := range Atplus {
			xi, yi := ml.Vector(svm.Xn[i]), svm.Yn[i]
			xiyi := xi.Scale(yi)
			if term2, err = term2.Add(xiyi); err != nil {
				return err
			}
		}
		if svm.Wn, err = term1.Add(term2); err != nil {
			return err
		}

		// wt+1 = min{1, (1/sqrt(lambda))/||wt+1||} wt+1
		var norm float64
		if norm, err = ml.Vector(svm.Wn).Norm(); err != nil {
			return err
		}
		projection := float64(1) / (math.Sqrt(svm.Lambda) * norm)
		if 1 > projection {
			svm.Wn = ml.Vector(svm.Wn).Scale(projection)
		}
	}
	return nil
}

// Ein returns the in sample error of the current svm model.
// It is the fraction of in sample points which got misclassified.
//
func (svm *SVM) Ein() float64 {

	// XnWn
	gInSample := make([]float64, len(svm.Xn))
	for i := 0; i < len(svm.Xn); i++ {
		gi, err := svm.Predict(svm.Xn[i])
		if err != nil {
			continue
		}
		gInSample[i] = ml.Sign(gi)
	}

	nEin := 0
	for i := 0; i < len(gInSample); i++ {
		if gInSample[i] != svm.Yn[i] {
			nEin++
		}
	}
	ein := float64(nEin) / float64(len(gInSample))
	svm.ComputedEin = true
	svm.ein = ein

	return ein
}

// Predict returns the result of the dot product between the x vector passed as param
// and the linear regression vector of weights.
//
func (svm *SVM) Predict(x []float64) (float64, error) {
	if len(x) != len(svm.Wn) {
		return 0, fmt.Errorf("SVM.Predict, size of x and Wn vector are different")
	}
	var p float64
	for j := 0; j < len(x); j++ {
		p += x[j] * svm.Wn[j]
	}
	return p, nil
}
<commit_msg>svm - fix bug<commit_after>// Package svm provides a set of svm types and functions.
package svm

import (
	"fmt"
	"math"
	"math/rand"

	"github.com/santiaago/ml"
)

// SVM holds all the information needed to run a support vector machine algorithm.
//
type SVM struct {
	TrainingPoints int         // number of training points.
	Xn             [][]float64 // data set of points for training.
	Yn             []float64   // output, evaluation of each Xi based on input data/ training examples.
	Wn             []float64   // weight vector.
	VectorSize     int         // size of vectors Xi and Wi.
	Lambda         float64     // used in the learning algorithm.
	Eta            float64     // used in the learning algorithm.
	K              int         // used in the learning algorithm, it is the block size to use when running Pegasos
}

// NewSVM creates a support vector machine object.
//
func NewSVM() *SVM {
	return &SVM{K: 1}
}

// InitializeFromData reads a 2 dimentional array with the following format:
// x1 x2 y
// x1 x2 y
// x1 x2 y
// And sets Xn and Yn accordingly
//
func (svm *SVM) InitializeFromData(data [][]float64) error {

	n := 0
	svm.Yn = make([]float64, len(data))
	svm.Xn = make([][]float64, len(data))

	for i, sample := range data {

		svm.Xn[i] = make([]float64, len(sample))
		svm.Xn[i] = []float64{1}
		svm.Xn[i] = append(svm.Xn[i], sample[:len(sample)-1]...)

		svm.Yn[i] = sample[len(sample)-1]
		n++
	}

	svm.TrainingPoints = n
	svm.VectorSize = len(svm.Xn[0])
	svm.Wn = make([]float64, svm.VectorSize)

	return nil
}

// Learn will update the weight vector with the output of a svm algorithm,
// with respect of the training examples and labels (svm.Xn, svm.Yn).
//
// We use an implementation of the Mini-Batch Pegasos Algorithm.
//
// The implementation of the Mini-Batch Pegasos Algorithm is based on the work done by:
// Shalev-Shwartz, Shai and Singer, Yoram and Srebro, Nathan.
// Pegasos: Primal Estimated sub-GrAdient SOlver for SVM.
//
// Note: We might support other methods in the future like SMO, ...
//
func (svm *SVM) Learn() error {

	var err error
	svm.TrainingPoints = len(svm.Xn)
	svm.Wn = make([]float64, svm.VectorSize)

	for t := range svm.Xn {

		// choose At where |At| = k, uniformly at random
		var At []int // vector of the selected indexes of size K
		for i := 0; i < svm.K; i++ {
			at := rand.Intn(svm.TrainingPoints)
			At = append(At, at)
		}

		// set At+ = {i in At : yi<wt, xi> < 1}
		var Atplus []int
		for _, i := range At {

			xi, yi := ml.Vector(svm.Xn[i]), svm.Yn[i]
			wt := ml.Vector(svm.Wn)
			var dot float64

			if dot, err = wt.Dot(xi); err != nil {
				return err
			}

			if yi*dot < 1 {
				Atplus = append(Atplus, i)
			}
		}

		// update eta = 1 / lambda*t
		svm.Eta = float64(1) / (svm.Lambda * float64(t+1))

		// set wt+1 = (1 - eta*lambda)*wt + eta/k * sum(for i in At+ of yixi)
		term1 := ml.Vector(svm.Wn).Scale(1 - svm.Eta*svm.Lambda)
		term2 := ml.Vector(make([]float64, svm.VectorSize))
		for _, i := range Atplus {
			xi, yi := ml.Vector(svm.Xn[i]), svm.Yn[i]
			xiyi := xi.Scale(yi)
			if term2, err = term2.Add(xiyi); err != nil {
				return err
			}
		}
		if svm.Wn, err = term1.Add(term2); err != nil {
			return err
		}

		// wt+1 = min{1, (1/sqrt(lambda))/||wt+1||} wt+1
		var norm float64
		if norm, err = ml.Vector(svm.Wn).Norm(); err != nil {
			return err
		}
		projection := float64(1) / (math.Sqrt(svm.Lambda) * norm)
		if 1 > projection {
			svm.Wn = ml.Vector(svm.Wn).Scale(projection)
		}
	}
	return nil
}

// Ein returns the in sample error of the current svm model.
// It is the fraction of in sample points which got misclassified.
//
func (svm *SVM) Ein() float64 {

	// XnWn
	gInSample := make([]float64, len(svm.Xn))
	for i := 0; i < len(svm.Xn); i++ {
		gi, err := svm.Predict(svm.Xn[i])
		if err != nil {
			continue
		}
		gInSample[i] = ml.Sign(gi)
	}

	nEin := 0
	for i := 0; i < len(gInSample); i++ {
		if gInSample[i] != svm.Yn[i] {
			nEin++
		}
	}
	ein := float64(nEin) / float64(len(gInSample))
	svm.ComputedEin = true
	svm.ein = ein

	return ein
}

// Predict returns the result of the dot product between the x vector passed as param
// and the linear regression vector of weights.
//
func (svm *SVM) Predict(x []float64) (float64, error) {
	if len(x) != len(svm.Wn) {
		return 0, fmt.Errorf("SVM.Predict, size of x and Wn vector are different")
	}
	var p float64
	for j := 0; j < len(x); j++ {
		p += x[j] * svm.Wn[j]
	}
	return p, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package serialize

import "math"

var (
	// defaultMaxNumberTags is the maximum number of tags that can be encoded.
	defaultMaxNumberTags uint16 = math.MaxUint16

	// defaultMaxTagLiteralLength is the maximum length of a tag Name/Value.
	defaultMaxTagLiteralLength uint16 = 4096
)

type limits struct {
	maxNumberTags       uint16
	maxTagLiteralLength uint16
}

// NewTagSerializationLimits returns a new TagSerializationLimits object.
func NewTagSerializationLimits() TagSerializationLimits {
	return &limits{
		maxNumberTags:       defaultMaxNumberTags,
		maxTagLiteralLength: defaultMaxTagLiteralLength,
	}
}

func (l *limits) SetMaxNumberTags(v uint16) TagSerializationLimits {
	lim := *l
	lim.maxNumberTags = v
	return &lim
}

func (l *limits) MaxNumberTags() uint16 {
	return l.maxNumberTags
}

func (l *limits) SetMaxTagLiteralLength(v uint16) TagSerializationLimits {
	lim := *l
	lim.maxTagLiteralLength = v
	return &lim
}

func (l *limits) MaxTagLiteralLength() uint16 {
	return l.maxTagLiteralLength
}
<commit_msg>Bump TagLiteralLength Limit to 64K (#1552)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package serialize

import "math"

var (
	// defaultMaxNumberTags is the maximum number of tags that can be encoded.
	defaultMaxNumberTags uint16 = math.MaxUint16

	// defaultMaxTagLiteralLength is the maximum length of a tag Name/Value.
	defaultMaxTagLiteralLength uint16 = math.MaxUint16
)

type limits struct {
	maxNumberTags       uint16
	maxTagLiteralLength uint16
}

// NewTagSerializationLimits returns a new TagSerializationLimits object.
func NewTagSerializationLimits() TagSerializationLimits {
	return &limits{
		maxNumberTags:       defaultMaxNumberTags,
		maxTagLiteralLength: defaultMaxTagLiteralLength,
	}
}

func (l *limits) SetMaxNumberTags(v uint16) TagSerializationLimits {
	lim := *l
	lim.maxNumberTags = v
	return &lim
}

func (l *limits) MaxNumberTags() uint16 {
	return l.maxNumberTags
}

func (l *limits) SetMaxTagLiteralLength(v uint16) TagSerializationLimits {
	lim := *l
	lim.maxTagLiteralLength = v
	return &lim
}

func (l *limits) MaxTagLiteralLength() uint16 {
	return l.maxTagLiteralLength
}
<|endoftext|>