<commit_before>package gce

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/emccode/rexray/core"
	"github.com/emccode/rexray/core/config"
	"github.com/emccode/rexray/core/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

const providerName = "gce"

// The GCE storage driver.
type driver struct {
	client  *compute.Service
	r       *core.RexRay
	zone    string
	project string
}

func ef() errors.Fields {
	return errors.Fields{
		"provider": providerName,
	}
}

func eff(fields errors.Fields) map[string]interface{} {
	errFields := map[string]interface{}{
		"provider": providerName,
	}
	if fields != nil {
		for k, v := range fields {
			errFields[k] = v
		}
	}
	return errFields
}

func init() {
	core.RegisterDriver(providerName, newDriver)
	config.Register(configRegistration())
}

func newDriver() core.Driver {
	return &driver{}
}

func (d *driver) Init(r *core.RexRay) error {
	d.r = r

	var err error

	d.zone = d.r.Config.GetString("gce.zone")
	d.project = d.r.Config.GetString("gce.project")
	serviceAccountJSON, err := ioutil.ReadFile(d.r.Config.GetString("gce.keyfile"))
	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not read service account credentials file, %s => {%s}", d.r.Config.GetString("gce.keyfile"), err)
	}

	config, err := google.JWTConfigFromJSON(serviceAccountJSON,
		compute.ComputeScope,
	)
	client, err := compute.New(config.Client(context.Background()))

	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not create compute client => {%s}", err)
	}
	d.client = client
	log.WithField("provider", providerName).Info("storage driver initialized")

	return nil
}

func (d *driver) Name() string {
	return providerName
}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
  log.WithField("provider", providerName).Debug("GetVolumeMapping")

    diskMap := make(map[string]*compute.Disk)
    disks, err := d.client.Disks.List(d.project, d.zone).Do()
    if err != nil {
	return []*core.BlockDevice{}, err
      }
  for _, disk := range disks.Items {
      log.WithField("provider", providerName).Debugf("%s",disk.SelfLink)
      diskMap[disk.SelfLink] = disk
    }

    instances, err := d.client.Instances.List(d.project, d.zone).Do()
    if err != nil {
	return []*core.BlockDevice{}, err
      }
  var ret []*core.BlockDevice
  for _, instance := range instances.Items {
      for _, disk := range instance.Disks {
      log.WithField("provider", providerName).Debugf("%s",disk.Source)
	  ret = append(ret, &core.BlockDevice {
	    ProviderName: "gce",
		InstanceID: strconv.FormatUint(instance.Id, 10),
		VolumeID: strconv.FormatUint(diskMap[disk.Source].Id, 10),
		DeviceName: disk.DeviceName,
		Region: diskMap[disk.Source].Zone,
		Status: diskMap[disk.Source].Status,
		NetworkName: disk.Source,
	    })

	}
    }
  return ret, nil
}

func (d *driver) GetInstance() (*core.Instance, error) {
	log.WithField("provider", providerName).Debug("GetInstance")
	return nil, nil
}

func (d *driver) CreateSnapshot(
	runAsync bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("CreateSnapshot")
	return nil, nil

}

func (d *driver) GetSnapshot(
	volumeID, snapshotID, snapshotName string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("GetSnapshot")
	return nil, nil
}

func (d *driver) RemoveSnapshot(snapshotID string) error {
	log.WithField("provider", providerName).Debug("RemoveSnapshot")
	return nil
}

func (d *driver) GetDeviceNextAvailable() (string, error) {
	letters := []string{
		"a", "b", "c", "d", "e", "f", "g", "h",
		"i", "j", "k", "l", "m", "n", "o", "p"}

	blockDeviceNames := make(map[string]bool)

	blockDeviceMapping, err := d.GetVolumeMapping()
	if err != nil {
		return "", err
	}

	for _, blockDevice := range blockDeviceMapping {
		re, _ := regexp.Compile(`^/dev/xvd([a-z])`)
		res := re.FindStringSubmatch(blockDevice.DeviceName)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	localDevices, err := getLocalDevices()
	if err != nil {
		return "", err
	}

	for _, localDevice := range localDevices {
		re, _ := regexp.Compile(`^xvd([a-z])`)
		res := re.FindStringSubmatch(localDevice)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	for _, letter := range letters {
		if !blockDeviceNames[letter] {
			nextDeviceName := "/dev/xvd" + letter
			log.Println("Got next device name: " + nextDeviceName)
			return nextDeviceName, nil
		}
	}
	return "", errors.New("No available device")
}

func getLocalDevices() (deviceNames []string, err error) {
	file := "/proc/partitions"
	contentBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return []string{}, err
	}

	content := string(contentBytes)

	lines := strings.Split(content, "\n")
	for _, line := range lines[2:] {
		fields := strings.Fields(line)
		if len(fields) == 4 {
			deviceNames = append(deviceNames, fields[3])
		}
	}

	return deviceNames, nil
}

func (d *driver) CreateVolume(
	runAsync bool, volumeName, volumeID, snapshotID, volumeType string,
	IOPS, size int64, availabilityZone string) (*core.Volume, error) {
	log.WithField("provider", providerName).Debug("CreateVolume")
	return nil, nil

}

func (d *driver) createVolumeCreateSnapshot(
	volumeID string, snapshotID string) (string, error) {
	log.WithField("provider", providerName).Debug("CreateVolumeCreateSnapshot")
	return "", nil

}

func (d *driver) GetVolume(
	volumeID, volumeName string) ([]*core.Volume, error) {
	log.WithField("provider", providerName).Debugf("GetVolume :%s %s", volumeID, volumeName)

	query := d.client.Disks.List(d.project, d.zone)
	if volumeID != "" {
		query.Filter(fmt.Sprintf("id eq %s", volumeID))
	}
	if volumeName != "" {
		query.Filter(fmt.Sprintf("name eq %s", volumeName))
	}
	var attachments []*core.VolumeAttachment
	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}

	disks, err := query.Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	var volumesSD []*core.Volume
	for _, disk := range disks.Items {
		var diskAttachments []*core.VolumeAttachment
		for _, attachment := range attachments {
			if attachment.VolumeID == disk.SelfLink {
				diskAttachments = append(diskAttachments, &core.VolumeAttachment{
					InstanceID: attachment.InstanceID,
					DeviceName: attachment.DeviceName,
					Status:     attachment.Status,
					VolumeID:   strconv.FormatUint(disk.Id, 10),
				})
			}
		}
		volumeSD := &core.Volume{
			Name:             disk.Name,
			VolumeID:         strconv.FormatUint(disk.Id, 10),
			AvailabilityZone: disk.Zone,
			Status:           disk.Status,
			VolumeType:       disk.Kind,
			NetworkName:      disk.SelfLink,
			IOPS:             0,
			Size:             strconv.FormatInt(disk.SizeGb, 10),
			Attachments:      diskAttachments,
		}
		volumesSD = append(volumesSD, volumeSD)

	}
	return volumesSD, nil
}

func (d *driver) GetVolumeAttach(
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debugf("GetVolumeAttach :%s %s",volumeID,instanceID)
	var attachments []*core.VolumeAttachment
	query := d.client.Instances.List(d.project, d.zone)
	if instanceID != "" {
		query.Filter(fmt.Sprintf("id eq %s", instanceID))
	}
	instances, err := query.Do()
	if err != nil {
		return []*core.VolumeAttachment{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}
	return attachments , nil
}

func (d *driver) waitSnapshotComplete(snapshotID string) error {
	return nil
}

func (d *driver) waitVolumeComplete(volumeID string) error {
	return nil
}

func (d *driver) waitVolumeAttach(volumeID, instanceID string) error {
	return nil
}

func (d *driver) waitVolumeDetach(volumeID string) error {
	return nil
}

func (d *driver) RemoveVolume(volumeID string) error {
	return nil
}

func (d *driver) AttachVolume(
	runAsync bool,
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debug("AttachVolume")
	return nil, nil

}

func (d *driver) DetachVolume(
	runAsync bool,
	volumeID, blank string) error {
	log.WithField("provider", providerName).Debug("DetachVolume")
	return nil
}

func (d *driver) CopySnapshot(runAsync bool,
	volumeID, snapshotID, snapshotName, destinationSnapshotName,
	destinationRegion string) (*core.Snapshot, error) {
	log.WithField("provider", providerName).Debug("CopySnapshot")
	return nil, nil
}

func configRegistration() *config.Registration {
	r := config.NewRegistration("Google GCE")
	r.Key(config.String, "", "", "", "gce.zone")
	r.Key(config.String, "", "", "", "gce.project")
	r.Key(config.String, "", "", "", "gce.keyfile")
	return r
}
<commit_msg>Work on current isntance<commit_after>package gce

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/emccode/rexray/core"
	"github.com/emccode/rexray/core/config"
	"github.com/emccode/rexray/core/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

const providerName = "gce"

// The GCE storage driver.
type driver struct {
	client  *compute.Service
	r       *core.RexRay
	zone    string
	project string
}

func ef() errors.Fields {
	return errors.Fields{
		"provider": providerName,
	}
}

func eff(fields errors.Fields) map[string]interface{} {
	errFields := map[string]interface{}{
		"provider": providerName,
	}
	if fields != nil {
		for k, v := range fields {
			errFields[k] = v
		}
	}
	return errFields
}

func init() {
	core.RegisterDriver(providerName, newDriver)
	config.Register(configRegistration())
}

func newDriver() core.Driver {
	return &driver{}
}

func (d *driver) Init(r *core.RexRay) error {
	d.r = r

	var err error

	d.zone = d.r.Config.GetString("gce.zone")
	d.project = d.r.Config.GetString("gce.project")
	serviceAccountJSON, err := ioutil.ReadFile(d.r.Config.GetString("gce.keyfile"))
	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not read service account credentials file, %s => {%s}", d.r.Config.GetString("gce.keyfile"), err)
	}

	config, err := google.JWTConfigFromJSON(serviceAccountJSON,
		compute.ComputeScope,
	)
	client, err := compute.New(config.Client(context.Background()))

	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not create compute client => {%s}", err)
	}
	d.client = client
	log.WithField("provider", providerName).Info("storage driver initialized")

	return nil
}

func (d *driver) Name() string {
	return providerName
}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
  log.WithField("provider", providerName).Debug("GetVolumeMapping")

    diskMap := make(map[string]*compute.Disk)
    disks, err := d.client.Disks.List(d.project, d.zone).Do()
    if err != nil {
	return []*core.BlockDevice{}, err
      }
  for _, disk := range disks.Items {
      log.WithField("provider", providerName).Debugf("%s",disk.SelfLink)
      diskMap[disk.SelfLink] = disk
    }

    instances, err := d.client.Instances.List(d.project, d.zone).Do()
    if err != nil {
	return []*core.BlockDevice{}, err
      }
  var ret []*core.BlockDevice
  for _, instance := range instances.Items {
      for _, disk := range instance.Disks {
      log.WithField("provider", providerName).Debugf("%s",disk.Source)
	  ret = append(ret, &core.BlockDevice {
	    ProviderName: "gce",
		InstanceID: strconv.FormatUint(instance.Id, 10),
		VolumeID: strconv.FormatUint(diskMap[disk.Source].Id, 10),
		DeviceName: disk.DeviceName,
		Region: diskMap[disk.Source].Zone,
		Status: diskMap[disk.Source].Status,
		NetworkName: disk.Source,
	    })

	}
    }
  return ret, nil
}

func (d *driver) GetInstance() (*core.Instance, error) {
  log.WithField("provider", providerName).Debug("GetInstance")
    var attachments []*core.VolumeAttachment
    query := d.client.Instances.List(d.project, d.zone)
    instances, err := query.Do()
    if err != nil {
     	return nil, err
      }
  var ret []*core.Instance
    for _, instance := range instances.Items {
	ret = append(ret, &core.Instance{
	  ProviderName: "gce",
	      InstanceID:   strconv.FormatUint(instance.Id, 10),
	      Region:       instance.Zone,
	      Name:  instance.Name,
	      })

      }
  return ret, nil
    }

func (d *driver) CreateSnapshot(
	runAsync bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("CreateSnapshot")
	return nil, nil

}

func (d *driver) GetSnapshot(
	volumeID, snapshotID, snapshotName string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("GetSnapshot")
	return nil, nil
}

func (d *driver) RemoveSnapshot(snapshotID string) error {
	log.WithField("provider", providerName).Debug("RemoveSnapshot")
	return nil
}

func (d *driver) GetDeviceNextAvailable() (string, error) {
	letters := []string{
		"a", "b", "c", "d", "e", "f", "g", "h",
		"i", "j", "k", "l", "m", "n", "o", "p"}

	blockDeviceNames := make(map[string]bool)

	blockDeviceMapping, err := d.GetVolumeMapping()
	if err != nil {
		return "", err
	}

	for _, blockDevice := range blockDeviceMapping {
		re, _ := regexp.Compile(`^/dev/xvd([a-z])`)
		res := re.FindStringSubmatch(blockDevice.DeviceName)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	localDevices, err := getLocalDevices()
	if err != nil {
		return "", err
	}

	for _, localDevice := range localDevices {
		re, _ := regexp.Compile(`^xvd([a-z])`)
		res := re.FindStringSubmatch(localDevice)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	for _, letter := range letters {
		if !blockDeviceNames[letter] {
			nextDeviceName := "/dev/xvd" + letter
			log.Println("Got next device name: " + nextDeviceName)
			return nextDeviceName, nil
		}
	}
	return "", errors.New("No available device")
}

func getLocalDevices() (deviceNames []string, err error) {
	file := "/proc/partitions"
	contentBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return []string{}, err
	}

	content := string(contentBytes)

	lines := strings.Split(content, "\n")
	for _, line := range lines[2:] {
		fields := strings.Fields(line)
		if len(fields) == 4 {
			deviceNames = append(deviceNames, fields[3])
		}
	}

	return deviceNames, nil
}

func (d *driver) CreateVolume(
	runAsync bool, volumeName, volumeID, snapshotID, volumeType string,
	IOPS, size int64, availabilityZone string) (*core.Volume, error) {
	log.WithField("provider", providerName).Debug("CreateVolume")
	return nil, nil

}

func (d *driver) createVolumeCreateSnapshot(
	volumeID string, snapshotID string) (string, error) {
	log.WithField("provider", providerName).Debug("CreateVolumeCreateSnapshot")
	return "", nil

}

func (d *driver) GetVolume(
	volumeID, volumeName string) ([]*core.Volume, error) {
	log.WithField("provider", providerName).Debugf("GetVolume :%s %s", volumeID, volumeName)

	query := d.client.Disks.List(d.project, d.zone)
	if volumeID != "" {
		query.Filter(fmt.Sprintf("id eq %s", volumeID))
	}
	if volumeName != "" {
		query.Filter(fmt.Sprintf("name eq %s", volumeName))
	}
	var attachments []*core.VolumeAttachment
	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}

	disks, err := query.Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	var volumesSD []*core.Volume
	for _, disk := range disks.Items {
		var diskAttachments []*core.VolumeAttachment
		for _, attachment := range attachments {
			if attachment.VolumeID == disk.SelfLink {
				diskAttachments = append(diskAttachments, &core.VolumeAttachment{
					InstanceID: attachment.InstanceID,
					DeviceName: attachment.DeviceName,
					Status:     attachment.Status,
					VolumeID:   strconv.FormatUint(disk.Id, 10),
				})
			}
		}
		volumeSD := &core.Volume{
			Name:             disk.Name,
			VolumeID:         strconv.FormatUint(disk.Id, 10),
			AvailabilityZone: disk.Zone,
			Status:           disk.Status,
			VolumeType:       disk.Kind,
			NetworkName:      disk.SelfLink,
			IOPS:             0,
			Size:             strconv.FormatInt(disk.SizeGb, 10),
			Attachments:      diskAttachments,
		}
		volumesSD = append(volumesSD, volumeSD)

	}
	return volumesSD, nil
}

func (d *driver) GetVolumeAttach(
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debugf("GetVolumeAttach :%s %s",volumeID,instanceID)
	var attachments []*core.VolumeAttachment
	query := d.client.Instances.List(d.project, d.zone)
	if instanceID != "" {
		query.Filter(fmt.Sprintf("id eq %s", instanceID))
	}
	instances, err := query.Do()
	if err != nil {
		return []*core.VolumeAttachment{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}
	return attachments , nil
}

func (d *driver) waitSnapshotComplete(snapshotID string) error {
	return nil
}

func (d *driver) waitVolumeComplete(volumeID string) error {
	return nil
}

func (d *driver) waitVolumeAttach(volumeID, instanceID string) error {
	return nil
}

func (d *driver) waitVolumeDetach(volumeID string) error {
	return nil
}

func (d *driver) RemoveVolume(volumeID string) error {
	return nil
}

func (d *driver) AttachVolume(
	runAsync bool,
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debug("AttachVolume")
	return nil, nil

}

func (d *driver) DetachVolume(
	runAsync bool,
	volumeID, blank string) error {
	log.WithField("provider", providerName).Debug("DetachVolume")
	return nil
}

func (d *driver) CopySnapshot(runAsync bool,
	volumeID, snapshotID, snapshotName, destinationSnapshotName,
	destinationRegion string) (*core.Snapshot, error) {
	log.WithField("provider", providerName).Debug("CopySnapshot")
	return nil, nil
}

func configRegistration() *config.Registration {
	r := config.NewRegistration("Google GCE")
	r.Key(config.String, "", "", "", "gce.zone")
	r.Key(config.String, "", "", "", "gce.project")
	r.Key(config.String, "", "", "", "gce.keyfile")
	return r
}
<|endoftext|>


<commit_before>package migrations

import (
	"fmt"
	"runtime"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/model/vfs/vfsswift"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/lock"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/swift"
	multierror "github.com/hashicorp/go-multierror"
)

const (
	swiftV1ToV2 = "swift-v1-to-v2"
	toSwiftV3   = "to-swift-v3"

	swiftV1ContainerPrefixCozy = "cozy-"
	swiftV1ContainerPrefixData = "data-"
	swiftV2ContainerPrefixCozy = "cozy-v2-"
	swiftV2ContainerPrefixData = "data-v2-"
	swiftV3ContainerPrefix     = "cozy-v3-"
)

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift
// made by a single migration.
const maxSimultaneousCalls = 8

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "migrations",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 2,
		WorkerFunc:   worker,
		WorkerCommit: commit,
	})
}

type message struct {
	Type string `json:"type"`
}

func worker(ctx *job.WorkerContext) error {
	var msg message
	if err := ctx.UnmarshalMessage(&msg); err != nil {
		return err
	}

	switch msg.Type {
	case toSwiftV3:
		return migrateToSwiftV3(ctx.Instance.Domain)
	case swiftV1ToV2:
		return fmt.Errorf("this migration type is no longer supported")
	default:
		return fmt.Errorf("unknown migration type %q", msg.Type)
	}
}

func commit(ctx *job.WorkerContext, err error) error {
	return err
}

func migrateToSwiftV3(domain string) error {
	c := config.GetSwiftConnection()
	inst, err := instance.GetFromCouch(domain)
	if err != nil {
		return err
	}

	var srcContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcContainer = swiftV1ContainerPrefixCozy + domain
	case 1: // layout v2
		srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix()
	case 2: // layout v3
		return nil // Nothing to do!
	default:
		return instance.ErrInvalidSwiftLayout
	}

	vfs := inst.VFS()
	root, err := vfs.DirByID(consts.RootDirID)
	if err != nil {
		return err
	}

	mutex := lock.ReadWrite(inst, "vfs")
	if err = mutex.Lock(); err != nil {
		return err
	}
	defer mutex.Unlock()

	dstContainer := swiftV3ContainerPrefix + inst.DBPrefix()
	if err = c.ContainerCreate(dstContainer, nil); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			_ = vfsswift.DeleteContainer(c, dstContainer)
		}
	}()

	if err = copyTheFilesToSwiftV3(inst, c, root, srcContainer, dstContainer); err != nil {
		return err
	}

	meta := &swift.Metadata{"cozy-migrated-from-v1": "1"}
	_ = c.ContainerUpdate(srcContainer, meta.ContainerHeaders())
	if in, err := instance.GetFromCouch(domain); err == nil {
		inst = in
	}
	inst.SwiftLayout = 2
	if err = couchdb.UpdateDoc(couchdb.GlobalDB, inst); err != nil {
		return err
	}
	_ = vfs.Delete()
	return nil
}

func copyTheFilesToSwiftV3(inst *instance.Instance, c *swift.Connection, root *vfs.DirDoc, src, dst string) error {
	nb := 0
	ch := make(chan error)
	log := logger.WithDomain(inst.Domain).
		WithField("nspace", "migration")

	var thumbsContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		thumbsContainer = swiftV1ContainerPrefixData + inst.Domain
	case 1: // layout v2
		thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix()
	default:
		return instance.ErrInvalidSwiftLayout
	}

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	fs := inst.VFS()
	errm := vfs.WalkAlreadyLocked(fs, root, func(fullpath string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error {
		if err != nil {
			return err
		}
		if f == nil {
			return nil
		}

		srcName := getSrcName(inst, f, fullpath)
		dstName := getDstName(inst, f)
		if srcName == "" || dstName == "" {
			return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName)
		}

		nb++
		go func() {
			k := <-tokens
			_, err := c.ObjectCopy(src, srcName, dst, dstName, nil)
			ch <- err
			tokens <- k
		}()

		// Copy the thumbnails
		if f.Class == "image" {
			srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f)
			dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f)
			nb += 3
			go func() {
				k := <-tokens
				_, err := c.ObjectCopy(thumbsContainer, srcSmall, dst, dstSmall, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail small from %s %s to %s %s",
						thumbsContainer, srcSmall, dst, dstSmall)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcMedium, dst, dstMedium, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail medium from %s %s to %s %s",
						thumbsContainer, srcMedium, dst, dstMedium)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcLarge, dst, dstLarge, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail large from %s %s to %s %s",
						thumbsContainer, srcLarge, dst, dstLarge)
				}
				ch <- nil
				tokens <- k
			}()
		}
		return nil
	})

	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}

func getSrcName(inst *instance.Instance, f *vfs.FileDoc, fullpath string) string {
	srcName := ""
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcName = fullpath
	case 1: // layout v2
		srcName = vfsswift.MakeObjectName(f.DocID)
	}
	return srcName
}

func getDstName(inst *instance.Instance, f *vfs.FileDoc) string {
	if f.InternalID == "" {
		f.InternalID = vfsswift.NewInternalID()
		if err := couchdb.UpdateDoc(inst, f); err != nil {
			return ""
		}
	}
	return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID)
}

func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	var small, medium, large string
	switch inst.SwiftLayout {
	case 0: // layout v1
		small = fmt.Sprintf("thumbs/%s-small", f.DocID)
		medium = fmt.Sprintf("thumbs/%s-medium", f.DocID)
		large = fmt.Sprintf("thumbs/%s-large", f.DocID)
	case 1: // layout v2
		obj := vfsswift.MakeObjectName(f.DocID)
		small = fmt.Sprintf("thumbs/%s-small", obj)
		medium = fmt.Sprintf("thumbs/%s-medium", obj)
		large = fmt.Sprintf("thumbs/%s-large", obj)
	}
	return small, medium, large
}

func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	obj := vfsswift.MakeObjectName(f.DocID)
	small := fmt.Sprintf("thumbs/%s-small", obj)
	medium := fmt.Sprintf("thumbs/%s-medium", obj)
	large := fmt.Sprintf("thumbs/%s-large", obj)
	return small, medium, large
}
<commit_msg>Add more logs for the migrations worker<commit_after>package migrations

import (
	"fmt"
	"runtime"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/model/vfs/vfsswift"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/lock"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/swift"
	multierror "github.com/hashicorp/go-multierror"
)

const (
	swiftV1ToV2 = "swift-v1-to-v2"
	toSwiftV3   = "to-swift-v3"

	swiftV1ContainerPrefixCozy = "cozy-"
	swiftV1ContainerPrefixData = "data-"
	swiftV2ContainerPrefixCozy = "cozy-v2-"
	swiftV2ContainerPrefixData = "data-v2-"
	swiftV3ContainerPrefix     = "cozy-v3-"
)

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift
// made by a single migration.
const maxSimultaneousCalls = 8

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "migrations",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 1,
		WorkerFunc:   worker,
		WorkerCommit: commit,
	})
}

type message struct {
	Type string `json:"type"`
}

func worker(ctx *job.WorkerContext) error {
	var msg message
	if err := ctx.UnmarshalMessage(&msg); err != nil {
		return err
	}

	logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration").
		Infof("Start the migration %s", msg.Type)

	switch msg.Type {
	case toSwiftV3:
		return migrateToSwiftV3(ctx.Instance.Domain)
	case swiftV1ToV2:
		return fmt.Errorf("this migration type is no longer supported")
	default:
		return fmt.Errorf("unknown migration type %q", msg.Type)
	}
}

func commit(ctx *job.WorkerContext, err error) error {
	log := logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration")
	if err == nil {
		log.Infof("Migration success")
	} else {
		log.Warningf("Migration error: %s", err)
	}
	return err
}

func migrateToSwiftV3(domain string) error {
	c := config.GetSwiftConnection()
	inst, err := instance.GetFromCouch(domain)
	if err != nil {
		return err
	}

	var srcContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcContainer = swiftV1ContainerPrefixCozy + domain
	case 1: // layout v2
		srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix()
	case 2: // layout v3
		return nil // Nothing to do!
	default:
		return instance.ErrInvalidSwiftLayout
	}

	vfs := inst.VFS()
	root, err := vfs.DirByID(consts.RootDirID)
	if err != nil {
		return err
	}

	mutex := lock.ReadWrite(inst, "vfs")
	if err = mutex.Lock(); err != nil {
		return err
	}
	defer mutex.Unlock()

	dstContainer := swiftV3ContainerPrefix + inst.DBPrefix()
	if err = c.ContainerCreate(dstContainer, nil); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			_ = vfsswift.DeleteContainer(c, dstContainer)
		}
	}()

	if err = copyTheFilesToSwiftV3(inst, c, root, srcContainer, dstContainer); err != nil {
		return err
	}

	meta := &swift.Metadata{"cozy-migrated-from-v1": "1"}
	_ = c.ContainerUpdate(srcContainer, meta.ContainerHeaders())
	if in, err := instance.GetFromCouch(domain); err == nil {
		inst = in
	}
	inst.SwiftLayout = 2
	if err = couchdb.UpdateDoc(couchdb.GlobalDB, inst); err != nil {
		return err
	}
	_ = vfs.Delete()
	return nil
}

func copyTheFilesToSwiftV3(inst *instance.Instance, c *swift.Connection, root *vfs.DirDoc, src, dst string) error {
	nb := 0
	ch := make(chan error)
	log := logger.WithDomain(inst.Domain).
		WithField("nspace", "migration")

	var thumbsContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		thumbsContainer = swiftV1ContainerPrefixData + inst.Domain
	case 1: // layout v2
		thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix()
	default:
		return instance.ErrInvalidSwiftLayout
	}

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	fs := inst.VFS()
	errm := vfs.WalkAlreadyLocked(fs, root, func(fullpath string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error {
		if err != nil {
			return err
		}
		if f == nil {
			return nil
		}

		srcName := getSrcName(inst, f, fullpath)
		dstName := getDstName(inst, f)
		if srcName == "" || dstName == "" {
			return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName)
		}

		nb++
		go func() {
			k := <-tokens
			_, err := c.ObjectCopy(src, srcName, dst, dstName, nil)
			ch <- err
			tokens <- k
		}()

		// Copy the thumbnails
		if f.Class == "image" {
			srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f)
			dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f)
			nb += 3
			go func() {
				k := <-tokens
				_, err := c.ObjectCopy(thumbsContainer, srcSmall, dst, dstSmall, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail small from %s %s to %s %s",
						thumbsContainer, srcSmall, dst, dstSmall)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcMedium, dst, dstMedium, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail medium from %s %s to %s %s",
						thumbsContainer, srcMedium, dst, dstMedium)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcLarge, dst, dstLarge, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail large from %s %s to %s %s",
						thumbsContainer, srcLarge, dst, dstLarge)
				}
				ch <- nil
				tokens <- k
			}()
		}
		return nil
	})

	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}

func getSrcName(inst *instance.Instance, f *vfs.FileDoc, fullpath string) string {
	srcName := ""
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcName = fullpath
	case 1: // layout v2
		srcName = vfsswift.MakeObjectName(f.DocID)
	}
	return srcName
}

func getDstName(inst *instance.Instance, f *vfs.FileDoc) string {
	if f.InternalID == "" {
		f.InternalID = vfsswift.NewInternalID()
		if err := couchdb.UpdateDoc(inst, f); err != nil {
			return ""
		}
	}
	return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID)
}

func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	var small, medium, large string
	switch inst.SwiftLayout {
	case 0: // layout v1
		small = fmt.Sprintf("thumbs/%s-small", f.DocID)
		medium = fmt.Sprintf("thumbs/%s-medium", f.DocID)
		large = fmt.Sprintf("thumbs/%s-large", f.DocID)
	case 1: // layout v2
		obj := vfsswift.MakeObjectName(f.DocID)
		small = fmt.Sprintf("thumbs/%s-small", obj)
		medium = fmt.Sprintf("thumbs/%s-medium", obj)
		large = fmt.Sprintf("thumbs/%s-large", obj)
	}
	return small, medium, large
}

func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	obj := vfsswift.MakeObjectName(f.DocID)
	small := fmt.Sprintf("thumbs/%s-small", obj)
	medium := fmt.Sprintf("thumbs/%s-medium", obj)
	large := fmt.Sprintf("thumbs/%s-large", obj)
	return small, medium, large
}
<|endoftext|>


<commit_before>package core

import ("fmt"
		"time"
		"github.com/op/go-nanomsg")
const MAXFRAMERATE = 60
const GAMEDELAY = time.Duration(time.Second / MAXFRAMERATE)
const GAMETICKFLOAT = float64(GAMEDELAY) / float64(time.Millisecond)

type GameThreadSockets struct {
	controlSocket *nanomsg.Socket
	inputMouseSub *nanomsg.SubSocket
	inputKbSub *nanomsg.SubSocket
	inputPush *nanomsg.Socket
	renderSocket *nanomsg.Socket
}

func gameThread(params GameThreadParams) {
	var gsockets GameThreadSockets
	var gs GameState
	var srs SharedRenderState
	var err error
	gsockets.controlSocket, err = nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
		panic(err)
    }
    _, err = gsockets.controlSocket.Connect("tcp://127.0.0.1:60206")
    if err != nil {
		panic(err)
    }
	gsockets.renderSocket, err = nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
		panic(err)
    }
    _, err = gsockets.renderSocket.Bind("tcp://127.0.0.1:60210")
    if err != nil {
		panic(err)
    }

	gsockets.inputMouseSub, err = nanomsg.NewSubSocket()
	if err != nil {
		panic(err)
	}
	gsockets.inputMouseSub.Subscribe("input.mouse:")
	_, err = gsockets.inputMouseSub.Connect("tcp://127.0.0.1:60208")
	if err != nil {
		panic(err)
	}

	gsockets.inputKbSub, err = nanomsg.NewSubSocket()
	gsockets.inputKbSub.Subscribe("input.kb:")
	if err != nil {
		panic(err)
	}
	_, err = gsockets.inputKbSub.Connect("tcp://127.0.0.1:60208")

	gameInit()
	baseLine := time.Since(params.start)
	var framenum uint64
	framenum = 0
	for true {
		now := time.Since(params.start)
		targetFrame := uint64 ((now - baseLine) / GAMEDELAY)
		if framenum <= targetFrame {
			framenum++
			// NOTE: build the state of the world at t = framenum * GAME_DELAY,
			// under normal conditions that's a time in the future
			// (the exception to that is if we are catching up on ticking game frames)
			gameTick(&gs, &srs, now);
			// Notify the render thread that a new game state is ready.
			// On the next render frame, it will start interpolating between the previous state and this new one
		} else {
			ahead := time.Duration(framenum) * GAMEDELAY - (now - baseLine)
			if ahead < 0 {
				panic(fmt.Sprintf("Ahead is less than 0: %d\n", ahead))
			}
			fmt.Printf("Game sleep %d ms\n", ahead)
			time.Sleep(ahead)
		}
		//cmd :=
	}
}
<commit_msg>Give time in milliseconds.<commit_after>package core

import ("fmt"
		"time"
		"github.com/op/go-nanomsg")
const MAXFRAMERATE = 60
const GAMEDELAY = time.Duration(time.Second / MAXFRAMERATE)
const GAMETICKFLOAT = float64(GAMEDELAY) / float64(time.Millisecond)

type GameThreadSockets struct {
	controlSocket *nanomsg.Socket
	inputMouseSub *nanomsg.SubSocket
	inputKbSub *nanomsg.SubSocket
	inputPush *nanomsg.Socket
	renderSocket *nanomsg.Socket
}

func gameThread(params GameThreadParams) {
	var gsockets GameThreadSockets
	var gs GameState
	var srs SharedRenderState
	var err error
	gsockets.controlSocket, err = nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
		panic(err)
    }
    _, err = gsockets.controlSocket.Connect("tcp://127.0.0.1:60206")
    if err != nil {
		panic(err)
    }
	gsockets.renderSocket, err = nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
		panic(err)
    }
    _, err = gsockets.renderSocket.Bind("tcp://127.0.0.1:60210")
    if err != nil {
		panic(err)
    }

	gsockets.inputMouseSub, err = nanomsg.NewSubSocket()
	if err != nil {
		panic(err)
	}
	gsockets.inputMouseSub.Subscribe("input.mouse:")
	_, err = gsockets.inputMouseSub.Connect("tcp://127.0.0.1:60208")
	if err != nil {
		panic(err)
	}

	gsockets.inputKbSub, err = nanomsg.NewSubSocket()
	gsockets.inputKbSub.Subscribe("input.kb:")
	if err != nil {
		panic(err)
	}
	_, err = gsockets.inputKbSub.Connect("tcp://127.0.0.1:60208")

	gameInit()
	baseLine := time.Since(params.start)
	var framenum uint64
	framenum = 0
	for true {
		now := time.Since(params.start)
		targetFrame := uint64 ((now - baseLine) / GAMEDELAY)
		if framenum <= targetFrame {
			framenum++
			// NOTE: build the state of the world at t = framenum * GAME_DELAY,
			// under normal conditions that's a time in the future
			// (the exception to that is if we are catching up on ticking game frames)
			gameTick(&gs, &srs, now);
			// Notify the render thread that a new game state is ready.
			// On the next render frame, it will start interpolating between the previous state and this new one
		} else {
			ahead := time.Duration(framenum) * GAMEDELAY - (now - baseLine)
			if ahead < 0 {
				panic(fmt.Sprintf("Ahead is less than 0: %d\n", ahead))
			}
			fmt.Printf("Game sleep %f ms\n", float64(ahead)/float64(time.Millisecond))
			time.Sleep(ahead)
		}
		//cmd :=
	}
}
<|endoftext|>


<commit_before>package checkerlution

import (
	"encoding/json"
	"github.com/couchbaselabs/logg"
)

// data structure that corresponds to the checkers:game json doc
type GameState struct {
	Teams        []Team `json:"teams"`
	ActiveTeam   int    `json:"activeTeam"`
	Number       int    `json:"number"`
	Turn         int    `json:"turn"`
	MoveInterval int    `json:""moveInterval""`
}

func NewGameStateFromString(jsonString string) GameState {
	gameState := &GameState{}
	jsonBytes := []byte(jsonString)
	err := json.Unmarshal(jsonBytes, gameState)
	if err != nil {
		logg.LogError(err)
	}
	return *gameState
}

type Piece struct {
	Location   int         `json:"location"`
	King       bool        `json:"king"`
	Captured   bool        `json:"captured"`
	ValidMoves []ValidMove `json:"validMoves"`
	PieceId    int
}

type Team struct {
	Score            int     `json:"score"`
	ParticipantCount int     `json:"participantCount"`
	Pieces           []Piece `json:"pieces"`
}

type ValidMove struct {
	Locations     []int     `json:"locations"`
	Captures      []Capture `json:"captures"`
	King          bool      `json:"king"`
	PieceId       int
	StartLocation int
}

type Capture struct {
	TeamID  int `json:"team"`
	PieceId int `json:"piece"`
}
<commit_msg>this had two pairs of double quotes, and test runner was complaining about it.  not sure why it was not noticed earlier.<commit_after>package checkerlution

import (
	"encoding/json"
	"github.com/couchbaselabs/logg"
)

// data structure that corresponds to the checkers:game json doc
type GameState struct {
	Teams        []Team `json:"teams"`
	ActiveTeam   int    `json:"activeTeam"`
	Number       int    `json:"number"`
	Turn         int    `json:"turn"`
	MoveInterval int    `json:"moveInterval"`
}

func NewGameStateFromString(jsonString string) GameState {
	gameState := &GameState{}
	jsonBytes := []byte(jsonString)
	err := json.Unmarshal(jsonBytes, gameState)
	if err != nil {
		logg.LogError(err)
	}
	return *gameState
}

type Piece struct {
	Location   int         `json:"location"`
	King       bool        `json:"king"`
	Captured   bool        `json:"captured"`
	ValidMoves []ValidMove `json:"validMoves"`
	PieceId    int
}

type Team struct {
	Score            int     `json:"score"`
	ParticipantCount int     `json:"participantCount"`
	Pieces           []Piece `json:"pieces"`
}

type ValidMove struct {
	Locations     []int     `json:"locations"`
	Captures      []Capture `json:"captures"`
	King          bool      `json:"king"`
	PieceId       int
	StartLocation int
}

type Capture struct {
	TeamID  int `json:"team"`
	PieceId int `json:"piece"`
}
<|endoftext|>


<commit_before>package gami

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"net/textproto"
	"reflect"
	"testing"
	"time"
)

//amiServer for mocking Asterisk AMI
type amiServer struct {
	Addr     string
	listener net.Listener
}

func TestLogin(t *testing.T) {
	srv := newAmiServer()
	defer srv.Close()
	ami, err := Dial(srv.Addr)
	if err != nil {
		t.Fatal(err)
	}
	go ami.Run()
	defer ami.Close()
	defaultInstaller(t, ami)
	ami.Login("admin", "admin")
}

func TestMultiAsyncActions(t *testing.T) {
	srv := newAmiServer()
	defer srv.Close()
	ami, err := Dial(srv.Addr)
	if err != nil {
		t.Fatal(err)
	}
	go ami.Run()
	defer ami.Close()
	defaultInstaller(t, ami)

	tests := 10
	workers := 3

	for ti := tests; ti > 0; ti-- {
		resWorkers := make(chan (<-chan *AMIResponse), workers)
		for i := 0; i < workers; i++ {
			chres, err := ami.AsyncAction("Test", nil)
			if err != nil {
				t.Error(err)
			}
			resWorkers <- chres
		}
		close(resWorkers)
		done := make(chan bool)
		go func() {
			select {
			case <-time.After(time.Second * 2):
				t.Error("asyncAction locked")
			case <-done:
				done <- true
				break
			}
		}()
		for res := range resWorkers {
			<-res
		}
		done <- true
		<-done
	}

}

func defaultInstaller(t *testing.T, ami *AMIClient) {
	go func() {
		for {
			select {
			//handle network errors
			case err := <-ami.NetError:
				t.Error("Network Error:", err)
			case err := <-ami.Error:
				t.Error("error:", err)
			//wait events and process
			case <-ami.Events:
				//t.Log("Event:", *ev)
			}
		}
	}()
}

func newAmiServer() *amiServer {
	addr := "localhost:0"
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(err)
	}
	srv := &amiServer{Addr: listener.Addr().String(), listener: listener}
	go srv.do(listener)
	return srv
}

//MockLogin it's a example for mocking responses
func (c *amiServer) MockLogin(params textproto.MIMEHeader) map[string]string {
	println("Llamado login")
	return map[string]string{
		"Response": "OK",
		"ActionID": params.Get("Actionid"),
	}
}

func (c *amiServer) do(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			return
		}
		fmt.Fprintf(conn, "Asterisk Call Manager\r\n")
		tconn := textproto.NewConn(conn)
		//install event HeartBeat
		go func(conn *textproto.Conn) {
			for now := range time.Tick(time.Second) {
				fmt.Fprintf(conn.W, "Event: HeartBeat\r\nTime: %d\r\n\r\n",
					now.Unix())
			}
		}(tconn)

		rval := reflect.ValueOf(c)
		go func(conn *textproto.Conn) {
			defer conn.Close()

			for {
				header, err := conn.ReadMIMEHeader()
				if err != nil {
					return
				}
				var output bytes.Buffer

				time.AfterFunc(time.Millisecond*time.Duration(rand.Intn(1000)), func() {
					fnc := rval.MethodByName("Mock" + header.Get("Action"))
					if fnc.IsValid() {
						rvals := fnc.Call([]reflect.Value{reflect.ValueOf(header)})
						ival := rvals[0].Interface()
						for k, vals := range ival.(map[string]string) {
							fmt.Fprintf(&output, "%s: %s\r\n", k, vals)
						}
						output.WriteString("\r\n")

						err := conn.PrintfLine(output.String())
						if err != nil {
							panic(err)
						}
					} else {
						//default response
						fmt.Fprintf(&output, "Response: TEST\r\nActionID: %s\r\n\r\n",
							header.Get("Actionid"))
						err := conn.PrintfLine(output.String())
						if err != nil {
							panic(err)
						}
					}
				})

			}
		}(tconn)
	}
}

func (c *amiServer) Close() {
	c.listener.Close()
}
<commit_msg>mock to callback<commit_after>package gami

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"net/textproto"
	"testing"
	"time"
)

type amiMockAction func(params textproto.MIMEHeader) map[string]string

//amiServer for mocking Asterisk AMI
type amiServer struct {
	Addr          string
	actionsMocked map[string]amiMockAction
	listener      net.Listener
}

func TestLogin(t *testing.T) {
	srv := newAmiServer()
	defer srv.Close()
	ami, err := Dial(srv.Addr)
	if err != nil {
		t.Fatal(err)
	}
	go ami.Run()
	defer ami.Close()
	defaultInstaller(t, ami)
	srv.Mock("Login", func(params textproto.MIMEHeader) map[string]string {
		println("mock login")
		return map[string]string{
			"Response": "OK",
			"ActionID": params.Get("Actionid"),
		}
	})
	ami.Login("admin", "admin")
}

func TestMultiAsyncActions(t *testing.T) {
	srv := newAmiServer()
	defer srv.Close()
	ami, err := Dial(srv.Addr)
	if err != nil {
		t.Fatal(err)
	}
	go ami.Run()
	defer ami.Close()
	defaultInstaller(t, ami)

	tests := 10
	workers := 3

	for ti := tests; ti > 0; ti-- {
		resWorkers := make(chan (<-chan *AMIResponse), workers)
		for i := 0; i < workers; i++ {
			chres, err := ami.AsyncAction("Test", nil)
			if err != nil {
				t.Error(err)
			}
			resWorkers <- chres
		}
		close(resWorkers)
		done := make(chan bool)
		go func() {
			select {
			case <-time.After(time.Second * 2):
				t.Error("asyncAction locked")
			case <-done:
				done <- true
				break
			}
		}()
		for res := range resWorkers {
			<-res
		}
		done <- true
		<-done
	}

}

func defaultInstaller(t *testing.T, ami *AMIClient) {
	go func() {
		for {
			select {
			//handle network errors
			case err := <-ami.NetError:
				t.Error("Network Error:", err)
			case err := <-ami.Error:
				t.Error("error:", err)
			//wait events and process
			case <-ami.Events:
				//t.Log("Event:", *ev)
			}
		}
	}()
}

func newAmiServer() *amiServer {
	addr := "localhost:0"
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(err)
	}
	srv := &amiServer{Addr: listener.Addr().String(),
		listener:      listener,
		actionsMocked: make(map[string]amiMockAction)}
	go srv.do(listener)
	return srv
}

//Mock
func (c *amiServer) Mock(action string, cb amiMockAction) {
	c.actionsMocked[action] = cb
}

func (c *amiServer) do(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			return
		}
		fmt.Fprintf(conn, "Asterisk Call Manager\r\n")
		tconn := textproto.NewConn(conn)
		//install event HeartBeat
		go func(conn *textproto.Conn) {
			for now := range time.Tick(time.Second) {
				fmt.Fprintf(conn.W, "Event: HeartBeat\r\nTime: %d\r\n\r\n",
					now.Unix())
			}
		}(tconn)

		go func(conn *textproto.Conn) {
			defer conn.Close()

			for {
				header, err := conn.ReadMIMEHeader()
				if err != nil {
					return
				}
				var output bytes.Buffer

				time.AfterFunc(time.Millisecond*time.Duration(rand.Intn(1000)), func() {

					if _, ok := c.actionsMocked[header.Get("Action")]; ok {
						rvals := c.actionsMocked[header.Get("Action")](header)
						for k, vals := range rvals {
							fmt.Fprintf(&output, "%s: %s\r\n", k, vals)
						}
						output.WriteString("\r\n")

						err := conn.PrintfLine(output.String())
						if err != nil {
							panic(err)
						}
					} else {
						//default response
						fmt.Fprintf(&output, "Response: TEST\r\nActionID: %s\r\n\r\n",
							header.Get("Actionid"))
						err := conn.PrintfLine(output.String())
						if err != nil {
							panic(err)
						}
					}
				})

			}
		}(tconn)
	}
}

func (c *amiServer) Close() {
	c.listener.Close()
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"regexp"
	"strconv"

	"github.com/wanelo/image-server/core"
)

func NameToConfiguration(sc *core.ServerConfiguration, filename string) (*core.ImageConfiguration, error) {
	var w, h, q, f string
	var quality uint

	reR := regexp.MustCompile(`^([0-9]+)x([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reS := regexp.MustCompile(`^x([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reW := regexp.MustCompile(`^w([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reF := regexp.MustCompile(`^full_size(?:-q([0-9]+))?\.(\w{3,4})$`)

	if reR.MatchString(filename) {
		m := reR.FindStringSubmatch(filename)
		w, h, q, f = m[1], m[2], m[3], m[4]
	} else if reS.MatchString(filename) {
		m := reS.FindStringSubmatch(filename)
		w, h, q, f = m[1], m[1], m[2], m[3]
	} else if reW.MatchString(filename) {
		m := reW.FindStringSubmatch(filename)
		w, h, q, f = m[1], "0", m[2], m[3]
	} else if reF.MatchString(filename) {
		m := reF.FindStringSubmatch(filename)
		w, h, q, f = "0", "0", m[1], m[2]
	} else {
		// return error
		return &core.ImageConfiguration{}, fmt.Errorf("unsupported")
	}

	width, _ := strconv.Atoi(w)
	height, _ := strconv.Atoi(h)
	quality64, _ := strconv.ParseUint(q, 10, 0)

	if quality64 > 0 {
		quality = uint(quality64)
	} else {
		quality = sc.DefaultQuality
	}

	return &core.ImageConfiguration{Width: width, Height: height, Quality: quality, Format: f, Filename: filename}, nil
}
<commit_msg>Name parser gives a better error message<commit_after>package parser

import (
	"fmt"
	"regexp"
	"strconv"

	"github.com/wanelo/image-server/core"
)

func NameToConfiguration(sc *core.ServerConfiguration, filename string) (*core.ImageConfiguration, error) {
	var w, h, q, f string
	var quality uint

	reR := regexp.MustCompile(`^([0-9]+)x([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reS := regexp.MustCompile(`^x([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reW := regexp.MustCompile(`^w([0-9]+)(?:-q([0-9]+))?\.(\w{3,4})$`)
	reF := regexp.MustCompile(`^full_size(?:-q([0-9]+))?\.(\w{3,4})$`)

	if reR.MatchString(filename) {
		m := reR.FindStringSubmatch(filename)
		w, h, q, f = m[1], m[2], m[3], m[4]
	} else if reS.MatchString(filename) {
		m := reS.FindStringSubmatch(filename)
		w, h, q, f = m[1], m[1], m[2], m[3]
	} else if reW.MatchString(filename) {
		m := reW.FindStringSubmatch(filename)
		w, h, q, f = m[1], "0", m[2], m[3]
	} else if reF.MatchString(filename) {
		m := reF.FindStringSubmatch(filename)
		w, h, q, f = "0", "0", m[1], m[2]
	} else {
		// return error
		return &core.ImageConfiguration{}, fmt.Errorf("unsupported output: %s", filename)
	}

	width, _ := strconv.Atoi(w)
	height, _ := strconv.Atoi(h)
	quality64, _ := strconv.ParseUint(q, 10, 0)

	if quality64 > 0 {
		quality = uint(quality64)
	} else {
		quality = sc.DefaultQuality
	}

	return &core.ImageConfiguration{Width: width, Height: height, Quality: quality, Format: f, Filename: filename}, nil
}
<|endoftext|>


<commit_before>package handler

import (
	"context"
	"fmt"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/strfmt"
	"github.com/goharbor/harbor/src/common/rbac"
	"github.com/goharbor/harbor/src/common/utils"
	"github.com/goharbor/harbor/src/controller/robot"
	"github.com/goharbor/harbor/src/lib"
	"github.com/goharbor/harbor/src/lib/config"
	"github.com/goharbor/harbor/src/lib/errors"
	pkg "github.com/goharbor/harbor/src/pkg/robot/model"
	"github.com/goharbor/harbor/src/server/v2.0/handler/model"
	"github.com/goharbor/harbor/src/server/v2.0/models"
	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/robot"
	"math"
	"regexp"
	"strconv"
	"strings"
)

func newRobotAPI() *robotAPI {
	return &robotAPI{
		robotCtl: robot.Ctl,
	}
}

type robotAPI struct {
	BaseAPI
	robotCtl robot.Controller
}

func (rAPI *robotAPI) CreateRobot(ctx context.Context, params operation.CreateRobotParams) middleware.Responder {
	if err := validateName(params.Robot.Name); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.validate(params.Robot.Duration, params.Robot.Level, params.Robot.Permissions); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, params.Robot.Level, params.Robot.Permissions[0].Namespace, rbac.ActionCreate); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r := &robot.Robot{
		Robot: pkg.Robot{
			Name:        params.Robot.Name,
			Description: params.Robot.Description,
			Duration:    params.Robot.Duration,
			Visible:     true,
		},
		Level: params.Robot.Level,
	}

	lib.JSONCopy(&r.Permissions, params.Robot.Permissions)

	rid, pwd, err := rAPI.robotCtl.Create(ctx, r)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	created, err := rAPI.robotCtl.Get(ctx, rid, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), created.ID)
	return operation.NewCreateRobotCreated().WithLocation(location).WithPayload(&models.RobotCreated{
		ID:           created.ID,
		Name:         created.Name,
		Secret:       pwd,
		CreationTime: strfmt.DateTime(created.CreationTime),
		ExpiresAt:    created.ExpiresAt,
	})
}

func (rAPI *robotAPI) DeleteRobot(ctx context.Context, params operation.DeleteRobotParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionDelete); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.robotCtl.Delete(ctx, params.RobotID); err != nil {
		// for the version 1 robot account, has to ignore the no permissions error.
		if !r.Editable && errors.IsNotFoundErr(err) {
			return operation.NewDeleteRobotOK()
		}
		return rAPI.SendError(ctx, err)
	}
	return operation.NewDeleteRobotOK()
}

func (rAPI *robotAPI) ListRobot(ctx context.Context, params operation.ListRobotParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	query, err := rAPI.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	var projectID int64
	var level string
	// GET /api/v2.0/robots or GET /api/v2.0/robots?level=system to get all of system level robots.
	// GET /api/v2.0/robots?level=project&project_id=1
	if _, ok := query.Keywords["Level"]; ok {
		if !isValidLevel(query.Keywords["Level"].(string)) {
			return rAPI.SendError(ctx, errors.New(nil).WithMessage("bad request error level input").WithCode(errors.BadRequestCode))
		}
		level = query.Keywords["Level"].(string)
		if level == robot.LEVELPROJECT {
			if _, ok := query.Keywords["ProjectID"]; !ok {
				return rAPI.SendError(ctx, errors.BadRequestError(nil).WithMessage("must with project ID when to query project robots"))
			}
			pid, err := strconv.ParseInt(query.Keywords["ProjectID"].(string), 10, 64)
			if err != nil {
				return rAPI.SendError(ctx, errors.BadRequestError(nil).WithMessage("Project ID must be int type."))
			}
			projectID = pid
		}

	} else {
		level = robot.LEVELSYSTEM
		query.Keywords["ProjectID"] = 0
	}
	query.Keywords["Visible"] = true

	if err := rAPI.requireAccess(ctx, level, projectID, rbac.ActionList); err != nil {
		return rAPI.SendError(ctx, err)
	}

	total, err := rAPI.robotCtl.Count(ctx, query)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	robots, err := rAPI.robotCtl.List(ctx, query, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	var results []*models.Robot
	for _, r := range robots {
		results = append(results, model.NewRobot(r).ToSwagger())
	}

	return operation.NewListRobotOK().
		WithXTotalCount(total).
		WithLink(rAPI.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
		WithPayload(results)
}

func (rAPI *robotAPI) GetRobotByID(ctx context.Context, params operation.GetRobotByIDParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionRead); err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewGetRobotByIDOK().WithPayload(model.NewRobot(r).ToSwagger())
}

func (rAPI *robotAPI) UpdateRobot(ctx context.Context, params operation.UpdateRobotParams) middleware.Responder {
	var err error
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}
	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if !r.Editable {
		err = rAPI.updateV1Robot(ctx, params, r)
	} else {
		err = rAPI.updateV2Robot(ctx, params, r)
	}
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewUpdateRobotOK()
}

func (rAPI *robotAPI) RefreshSec(ctx context.Context, params operation.RefreshSecParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionUpdate); err != nil {
		return rAPI.SendError(ctx, err)
	}

	var secret string
	robotSec := &models.RobotSec{}
	if params.RobotSec.Secret != "" {
		if !isValidSec(params.RobotSec.Secret) {
			return rAPI.SendError(ctx, errors.New("the secret must longer than 8 chars with at least 1 uppercase letter, 1 lowercase letter and 1 number").WithCode(errors.BadRequestCode))
		}
		secret = utils.Encrypt(params.RobotSec.Secret, r.Salt, utils.SHA256)
		robotSec.Secret = ""
	} else {
		pwd := utils.GenerateRandomString()
		secret = utils.Encrypt(pwd, r.Salt, utils.SHA256)
		robotSec.Secret = pwd
	}

	r.Secret = secret
	if err := rAPI.robotCtl.Update(ctx, r, nil); err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewRefreshSecOK().WithPayload(robotSec)
}

func (rAPI *robotAPI) requireAccess(ctx context.Context, level string, projectIDOrName interface{}, action rbac.Action) error {
	if level == robot.LEVELSYSTEM {
		return rAPI.RequireSystemAccess(ctx, action, rbac.ResourceRobot)
	} else if level == robot.LEVELPROJECT {
		return rAPI.RequireProjectAccess(ctx, projectIDOrName, action, rbac.ResourceRobot)
	}
	return errors.ForbiddenError(nil)
}

// more validation
func (rAPI *robotAPI) validate(d int64, level string, permissions []*models.RobotPermission) error {
	if !isValidDuration(d) {
		return errors.New(nil).WithMessage("bad request error duration input: %d", d).WithCode(errors.BadRequestCode)
	}

	if !isValidLevel(level) {
		return errors.New(nil).WithMessage("bad request error level input: %s", level).WithCode(errors.BadRequestCode)
	}

	if len(permissions) == 0 {
		return errors.New(nil).WithMessage("bad request empty permission").WithCode(errors.BadRequestCode)
	}

	for _, perm := range permissions {
		if len(perm.Access) == 0 {
			return errors.New(nil).WithMessage("bad request empty access").WithCode(errors.BadRequestCode)
		}
	}

	// to create a project robot, the permission must be only one project scope.
	if level == robot.LEVELPROJECT && len(permissions) > 1 {
		return errors.New(nil).WithMessage("bad request permission").WithCode(errors.BadRequestCode)
	}
	return nil
}

// only disable can be updated for v1 robot
func (rAPI *robotAPI) updateV1Robot(ctx context.Context, params operation.UpdateRobotParams, r *robot.Robot) error {
	if err := rAPI.requireAccess(ctx, params.Robot.Level, r.ProjectID, rbac.ActionUpdate); err != nil {
		return err
	}
	r.Disabled = params.Robot.Disable
	r.Description = params.Robot.Description
	if err := rAPI.robotCtl.Update(ctx, r, &robot.Option{
		WithPermission: false,
	}); err != nil {
		return err
	}
	return nil
}

func (rAPI *robotAPI) updateV2Robot(ctx context.Context, params operation.UpdateRobotParams, r *robot.Robot) error {
	if err := rAPI.validate(params.Robot.Duration, params.Robot.Level, params.Robot.Permissions); err != nil {
		return err
	}

	if err := rAPI.requireAccess(ctx, params.Robot.Level, params.Robot.Permissions[0].Namespace, rbac.ActionUpdate); err != nil {
		return err
	}

	if params.Robot.Level != r.Level || params.Robot.Name != r.Name {
		return errors.BadRequestError(nil).WithMessage("cannot update the level or name of robot")
	}

	if r.Duration != params.Robot.Duration {
		r.Duration = params.Robot.Duration
		if params.Robot.Duration == -1 {
			r.ExpiresAt = -1
		} else if params.Robot.Duration == 0 {
			r.Duration = int64(config.RobotTokenDuration(ctx))
			r.ExpiresAt = r.CreationTime.AddDate(0, 0, config.RobotTokenDuration(ctx)).Unix()
		} else {
			r.ExpiresAt = r.CreationTime.AddDate(0, 0, int(params.Robot.Duration)).Unix()
		}
	}

	r.Description = params.Robot.Description
	r.Disabled = params.Robot.Disable
	if len(params.Robot.Permissions) != 0 {
		lib.JSONCopy(&r.Permissions, params.Robot.Permissions)
	}

	if err := rAPI.robotCtl.Update(ctx, r, &robot.Option{
		WithPermission: true,
	}); err != nil {
		return err
	}
	return nil
}

func isValidLevel(l string) bool {
	return l == robot.LEVELSYSTEM || l == robot.LEVELPROJECT
}

func isValidDuration(d int64) bool {
	return d >= int64(-1) && d < math.MaxInt32
}

func isValidSec(sec string) bool {
	hasLower := regexp.MustCompile(`[a-z]`)
	hasUpper := regexp.MustCompile(`[A-Z]`)
	hasNumber := regexp.MustCompile(`[0-9]`)
	if len(sec) >= 8 && hasLower.MatchString(sec) && hasUpper.MatchString(sec) && hasNumber.MatchString(sec) {
		return true
	}
	return false
}

// validateName validates the robot name, especially '+' cannot be a valid character
func validateName(name string) error {
	robotNameReg := `^[a-z0-9]+(?:[._-][a-z0-9]+)*$`
	legal := regexp.MustCompile(robotNameReg).MatchString(name)
	if !legal {
		return errors.BadRequestError(nil).WithMessage("robot name is not in lower case or contains illegal characters")
	}
	return nil
}
<commit_msg>fix legacy robot edit issue (#15709)<commit_after>package handler

import (
	"context"
	"fmt"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/strfmt"
	"github.com/goharbor/harbor/src/common/rbac"
	"github.com/goharbor/harbor/src/common/utils"
	"github.com/goharbor/harbor/src/controller/robot"
	"github.com/goharbor/harbor/src/lib"
	"github.com/goharbor/harbor/src/lib/config"
	"github.com/goharbor/harbor/src/lib/errors"
	pkg "github.com/goharbor/harbor/src/pkg/robot/model"
	"github.com/goharbor/harbor/src/server/v2.0/handler/model"
	"github.com/goharbor/harbor/src/server/v2.0/models"
	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/robot"
	"math"
	"regexp"
	"strconv"
	"strings"
)

func newRobotAPI() *robotAPI {
	return &robotAPI{
		robotCtl: robot.Ctl,
	}
}

type robotAPI struct {
	BaseAPI
	robotCtl robot.Controller
}

func (rAPI *robotAPI) CreateRobot(ctx context.Context, params operation.CreateRobotParams) middleware.Responder {
	if err := validateName(params.Robot.Name); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.validate(params.Robot.Duration, params.Robot.Level, params.Robot.Permissions); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, params.Robot.Level, params.Robot.Permissions[0].Namespace, rbac.ActionCreate); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r := &robot.Robot{
		Robot: pkg.Robot{
			Name:        params.Robot.Name,
			Description: params.Robot.Description,
			Duration:    params.Robot.Duration,
			Visible:     true,
		},
		Level: params.Robot.Level,
	}

	lib.JSONCopy(&r.Permissions, params.Robot.Permissions)

	rid, pwd, err := rAPI.robotCtl.Create(ctx, r)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	created, err := rAPI.robotCtl.Get(ctx, rid, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	location := fmt.Sprintf("%s/%d", strings.TrimSuffix(params.HTTPRequest.URL.Path, "/"), created.ID)
	return operation.NewCreateRobotCreated().WithLocation(location).WithPayload(&models.RobotCreated{
		ID:           created.ID,
		Name:         created.Name,
		Secret:       pwd,
		CreationTime: strfmt.DateTime(created.CreationTime),
		ExpiresAt:    created.ExpiresAt,
	})
}

func (rAPI *robotAPI) DeleteRobot(ctx context.Context, params operation.DeleteRobotParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionDelete); err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.robotCtl.Delete(ctx, params.RobotID); err != nil {
		// for the version 1 robot account, has to ignore the no permissions error.
		if !r.Editable && errors.IsNotFoundErr(err) {
			return operation.NewDeleteRobotOK()
		}
		return rAPI.SendError(ctx, err)
	}
	return operation.NewDeleteRobotOK()
}

func (rAPI *robotAPI) ListRobot(ctx context.Context, params operation.ListRobotParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	query, err := rAPI.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	var projectID int64
	var level string
	// GET /api/v2.0/robots or GET /api/v2.0/robots?level=system to get all of system level robots.
	// GET /api/v2.0/robots?level=project&project_id=1
	if _, ok := query.Keywords["Level"]; ok {
		if !isValidLevel(query.Keywords["Level"].(string)) {
			return rAPI.SendError(ctx, errors.New(nil).WithMessage("bad request error level input").WithCode(errors.BadRequestCode))
		}
		level = query.Keywords["Level"].(string)
		if level == robot.LEVELPROJECT {
			if _, ok := query.Keywords["ProjectID"]; !ok {
				return rAPI.SendError(ctx, errors.BadRequestError(nil).WithMessage("must with project ID when to query project robots"))
			}
			pid, err := strconv.ParseInt(query.Keywords["ProjectID"].(string), 10, 64)
			if err != nil {
				return rAPI.SendError(ctx, errors.BadRequestError(nil).WithMessage("Project ID must be int type."))
			}
			projectID = pid
		}

	} else {
		level = robot.LEVELSYSTEM
		query.Keywords["ProjectID"] = 0
	}
	query.Keywords["Visible"] = true

	if err := rAPI.requireAccess(ctx, level, projectID, rbac.ActionList); err != nil {
		return rAPI.SendError(ctx, err)
	}

	total, err := rAPI.robotCtl.Count(ctx, query)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	robots, err := rAPI.robotCtl.List(ctx, query, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	var results []*models.Robot
	for _, r := range robots {
		results = append(results, model.NewRobot(r).ToSwagger())
	}

	return operation.NewListRobotOK().
		WithXTotalCount(total).
		WithLink(rAPI.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
		WithPayload(results)
}

func (rAPI *robotAPI) GetRobotByID(ctx context.Context, params operation.GetRobotByIDParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionRead); err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewGetRobotByIDOK().WithPayload(model.NewRobot(r).ToSwagger())
}

func (rAPI *robotAPI) UpdateRobot(ctx context.Context, params operation.UpdateRobotParams) middleware.Responder {
	var err error
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}
	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, &robot.Option{
		WithPermission: true,
	})
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if !r.Editable {
		err = errors.DeniedError(nil).WithMessage("editing of legacy robot is not allowed")
	} else {
		err = rAPI.updateV2Robot(ctx, params, r)
	}
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewUpdateRobotOK()
}

func (rAPI *robotAPI) RefreshSec(ctx context.Context, params operation.RefreshSecParams) middleware.Responder {
	if err := rAPI.RequireAuthenticated(ctx); err != nil {
		return rAPI.SendError(ctx, err)
	}

	r, err := rAPI.robotCtl.Get(ctx, params.RobotID, nil)
	if err != nil {
		return rAPI.SendError(ctx, err)
	}

	if err := rAPI.requireAccess(ctx, r.Level, r.ProjectID, rbac.ActionUpdate); err != nil {
		return rAPI.SendError(ctx, err)
	}

	var secret string
	robotSec := &models.RobotSec{}
	if params.RobotSec.Secret != "" {
		if !isValidSec(params.RobotSec.Secret) {
			return rAPI.SendError(ctx, errors.New("the secret must longer than 8 chars with at least 1 uppercase letter, 1 lowercase letter and 1 number").WithCode(errors.BadRequestCode))
		}
		secret = utils.Encrypt(params.RobotSec.Secret, r.Salt, utils.SHA256)
		robotSec.Secret = ""
	} else {
		pwd := utils.GenerateRandomString()
		secret = utils.Encrypt(pwd, r.Salt, utils.SHA256)
		robotSec.Secret = pwd
	}

	r.Secret = secret
	if err := rAPI.robotCtl.Update(ctx, r, nil); err != nil {
		return rAPI.SendError(ctx, err)
	}

	return operation.NewRefreshSecOK().WithPayload(robotSec)
}

func (rAPI *robotAPI) requireAccess(ctx context.Context, level string, projectIDOrName interface{}, action rbac.Action) error {
	if level == robot.LEVELSYSTEM {
		return rAPI.RequireSystemAccess(ctx, action, rbac.ResourceRobot)
	} else if level == robot.LEVELPROJECT {
		return rAPI.RequireProjectAccess(ctx, projectIDOrName, action, rbac.ResourceRobot)
	}
	return errors.ForbiddenError(nil)
}

// more validation
func (rAPI *robotAPI) validate(d int64, level string, permissions []*models.RobotPermission) error {
	if !isValidDuration(d) {
		return errors.New(nil).WithMessage("bad request error duration input: %d", d).WithCode(errors.BadRequestCode)
	}

	if !isValidLevel(level) {
		return errors.New(nil).WithMessage("bad request error level input: %s", level).WithCode(errors.BadRequestCode)
	}

	if len(permissions) == 0 {
		return errors.New(nil).WithMessage("bad request empty permission").WithCode(errors.BadRequestCode)
	}

	for _, perm := range permissions {
		if len(perm.Access) == 0 {
			return errors.New(nil).WithMessage("bad request empty access").WithCode(errors.BadRequestCode)
		}
	}

	// to create a project robot, the permission must be only one project scope.
	if level == robot.LEVELPROJECT && len(permissions) > 1 {
		return errors.New(nil).WithMessage("bad request permission").WithCode(errors.BadRequestCode)
	}
	return nil
}

func (rAPI *robotAPI) updateV2Robot(ctx context.Context, params operation.UpdateRobotParams, r *robot.Robot) error {
	if err := rAPI.validate(params.Robot.Duration, params.Robot.Level, params.Robot.Permissions); err != nil {
		return err
	}

	if err := rAPI.requireAccess(ctx, params.Robot.Level, params.Robot.Permissions[0].Namespace, rbac.ActionUpdate); err != nil {
		return err
	}

	if params.Robot.Level != r.Level || params.Robot.Name != r.Name {
		return errors.BadRequestError(nil).WithMessage("cannot update the level or name of robot")
	}

	if r.Duration != params.Robot.Duration {
		r.Duration = params.Robot.Duration
		if params.Robot.Duration == -1 {
			r.ExpiresAt = -1
		} else if params.Robot.Duration == 0 {
			r.Duration = int64(config.RobotTokenDuration(ctx))
			r.ExpiresAt = r.CreationTime.AddDate(0, 0, config.RobotTokenDuration(ctx)).Unix()
		} else {
			r.ExpiresAt = r.CreationTime.AddDate(0, 0, int(params.Robot.Duration)).Unix()
		}
	}

	r.Description = params.Robot.Description
	r.Disabled = params.Robot.Disable
	if len(params.Robot.Permissions) != 0 {
		lib.JSONCopy(&r.Permissions, params.Robot.Permissions)
	}

	if err := rAPI.robotCtl.Update(ctx, r, &robot.Option{
		WithPermission: true,
	}); err != nil {
		return err
	}
	return nil
}

func isValidLevel(l string) bool {
	return l == robot.LEVELSYSTEM || l == robot.LEVELPROJECT
}

func isValidDuration(d int64) bool {
	return d >= int64(-1) && d < math.MaxInt32
}

func isValidSec(sec string) bool {
	hasLower := regexp.MustCompile(`[a-z]`)
	hasUpper := regexp.MustCompile(`[A-Z]`)
	hasNumber := regexp.MustCompile(`[0-9]`)
	if len(sec) >= 8 && hasLower.MatchString(sec) && hasUpper.MatchString(sec) && hasNumber.MatchString(sec) {
		return true
	}
	return false
}

// validateName validates the robot name, especially '+' cannot be a valid character
func validateName(name string) error {
	robotNameReg := `^[a-z0-9]+(?:[._-][a-z0-9]+)*$`
	legal := regexp.MustCompile(robotNameReg).MatchString(name)
	if !legal {
		return errors.BadRequestError(nil).WithMessage("robot name is not in lower case or contains illegal characters")
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>generated<commit_after><|endoftext|>


<commit_before>package upload

import (
	"net/http"
	"net/url"
	"testing"

	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/stretchr/testify/require"
)

func TestCreateV4Request(t *testing.T) {
	assert := require.New(t)
	header := "test-header"
	signed := http.Header{}
	reqHeaders := http.Header{}
	signed.Add(header, "val")
	reqHeaders.Add(header, "val2")
	req := &request.Request{SignedHeaderVals: signed}
	u := url.URL{Path: "url", RawQuery: "a=b&c=d"}
	hr := http.Request{URL: &u, Header: reqHeaders, Method: "PUT"}
	req.HTTPRequest = &hr
	params := UploadParameters{}
	v4 := CreateV4Request(params, req)
	assert.NotNil(v4)
	assert.Equal("us-east-1", v4.Region)
	assert.Equal("PUT", v4.Method)
	assert.Equal("url", v4.Path)
	assert.Equal("a=b&c=d", v4.RawQuery)
	assert.Equal("", v4.Headers[header])
	assert.Equal("val2", v4.Headers["Test-Header"])
	built := v4.BuildRequest()
	assert.Equal(v4.Method, built.Method)
	assert.Equal(v4.RawQuery, built.URL.RawQuery)
	assert.Equal(v4.Path, built.URL.Path)
	assert.Equal("val2", built.Header.Get("Test-Header"))
}
<commit_msg>add Sign test<commit_after>package upload

import (
	"net/http"
	"net/url"
	"testing"

	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/stretchr/testify/require"
)

func TestCreateV4Request(t *testing.T) {
	assert := require.New(t)
	header := "test-header"
	signed := http.Header{}
	reqHeaders := http.Header{}
	signed.Add(header, "val")
	reqHeaders.Add(header, "val2")
	req := &request.Request{SignedHeaderVals: signed}
	u := url.URL{Path: "url", RawQuery: "a=b&c=d"}
	hr := http.Request{URL: &u, Header: reqHeaders, Method: "PUT"}
	req.HTTPRequest = &hr
	params := UploadParameters{}
	v4 := CreateV4Request(params, req)
	assert.NotNil(v4)
	assert.Equal("us-east-1", v4.Region)
	assert.Equal("PUT", v4.Method)
	assert.Equal("url", v4.Path)
	assert.Equal("a=b&c=d", v4.RawQuery)
	assert.Equal("", v4.Headers[header])
	assert.Equal("val2", v4.Headers["Test-Header"])
	built := v4.BuildRequest()
	assert.Equal(v4.Method, built.Method)
	assert.Equal(v4.RawQuery, built.URL.RawQuery)
	assert.Equal(v4.Path, built.URL.Path)
	assert.Equal("val2", built.Header.Get("Test-Header"))
}

func TestSign(t *testing.T) {
	assert := require.New(t)
	headers := make(map[string]string)
	headers["test-header"] = "val"
	req := V4Request{Region: "us-east-1", Headers: headers}
	resp, err := req.Sign("a", "b")
	assert.Nil(err)
	assert.NotEmpty(resp.Date)
	assert.Contains(resp.Authorization, "AWS4-HMAC-SHA256 Credential=a/20180223/us-east-1/s3/aws4_request, SignedHeaders=host;test-header;x-amz-content-sha256;x-amz-date")
}
<|endoftext|>


<commit_before>package upload

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codeclimate/test-reporter/formatters"
	"github.com/codeclimate/test-reporter/version"
	"github.com/pkg/errors"
)

type Uploader struct {
	ReporterID  string
	EndpointURL string
	BatchSize   int
	Input       io.Reader
}

func (u Uploader) Upload() error {
	if u.ReporterID == "" {
		return errors.New("you must supply a CC_TEST_REPORTER_ID ENV variable or pass it via the -r flag")
	}

	rep, err := formatters.NewReport()
	if err != nil {
		return errors.WithStack(err)
	}

	err = json.NewDecoder(u.Input).Decode(&rep)
	if err != nil {
		return errors.WithStack(err)
	}

	testReport := NewTestReport(rep)

	pr, pw := io.Pipe()
	go func() {
		defer pw.Close()
		bb := &bytes.Buffer{}
		w := io.MultiWriter(pw, bb)
		err := json.NewEncoder(w).Encode(JSONWraper{Data: testReport})
		if err != nil {
			logrus.Error(err)
			return
		}
		logrus.Debug(bb.String())
	}()

	res, err := u.doRequest(pr, u.EndpointURL)
	if err != nil {
		return errors.WithStack(err)
	}

	batchLinks := struct {
		Links struct {
			PostBatch string `json:"post_batch"`
		} `json:"links"`
	}{}

	err = json.NewDecoder(res.Body).Decode(&batchLinks)
	if err != nil {
		return errors.WithStack(err)
	}

	return u.SendBatches(testReport, batchLinks.Links.PostBatch)
}

func (u Uploader) SendBatches(rep *TestReport, url string) error {
	batch := [][]SourceFile{}

	pos := 0
	count := len(rep.SourceFiles) / u.BatchSize
	remainder := len(rep.SourceFiles) % u.BatchSize
	for i := 0; i < count; i++ {
		end := pos + u.BatchSize
		batch = append(batch, rep.SourceFiles[pos:end])
		pos = end
	}
	if remainder > 0 {
		batch = append(batch, rep.SourceFiles[pos:])
	}

	for i, b := range batch {
		pr, pw := io.Pipe()
		go func() {
			defer pw.Close()
			bb := &bytes.Buffer{}
			w := io.MultiWriter(pw, bb)
			err := json.NewEncoder(w).Encode(JSONWraper{
				Data: b,
				Meta: map[string]int{
					"current": i + 1,
					"total":   len(batch),
				},
			})
			if err != nil {
				logrus.Error(err)
				return
			}
			logrus.Debug(bb.String())
		}()
		_, err := u.doRequest(pr, url)
		if err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

func (u Uploader) doRequest(in io.Reader, url string) (*http.Response, error) {
	c := http.Client{
		Timeout: 30 * time.Second,
	}

	req, err := u.newRequest(in, url)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	logrus.Debugf("posting request to %s", url)
	res, err := c.Do(req)
	if err != nil {
		return res, errors.WithStack(err)
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return res, fmt.Errorf("response from %s was %d", url, res.StatusCode)
	}
	return res, nil
}

func (u Uploader) newRequest(in io.Reader, url string) (*http.Request, error) {
	req, err := http.NewRequest("POST", url, in)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	req.Header.Set("User-Agent", fmt.Sprintf("TestReporter/%s (Code Climate, Inc.)", version.Version))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-CC-Test-Reporter-Id", u.ReporterID)
	req.Header.Set("Accept", "application/vnd.api+json")
	return req, err
}
<commit_msg>Remove git dependency on git upload-coverage (#163)<commit_after>package upload

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codeclimate/test-reporter/formatters"
	"github.com/codeclimate/test-reporter/version"
	"github.com/pkg/errors"
)

type Uploader struct {
	ReporterID  string
	EndpointURL string
	BatchSize   int
	Input       io.Reader
}

func (u Uploader) Upload() error {
	if u.ReporterID == "" {
		return errors.New("you must supply a CC_TEST_REPORTER_ID ENV variable or pass it via the -r flag")
	}

	rep := formatters.Report{
		SourceFiles: formatters.SourceFiles{},
	}

	err := json.NewDecoder(u.Input).Decode(&rep)
	if err != nil {
		return errors.WithStack(err)
	}

	testReport := NewTestReport(rep)

	pr, pw := io.Pipe()
	go func() {
		defer pw.Close()
		bb := &bytes.Buffer{}
		w := io.MultiWriter(pw, bb)
		err := json.NewEncoder(w).Encode(JSONWraper{Data: testReport})
		if err != nil {
			logrus.Error(err)
			return
		}
		logrus.Debug(bb.String())
	}()

	res, err := u.doRequest(pr, u.EndpointURL)
	if err != nil {
		return errors.WithStack(err)
	}

	batchLinks := struct {
		Links struct {
			PostBatch string `json:"post_batch"`
		} `json:"links"`
	}{}

	err = json.NewDecoder(res.Body).Decode(&batchLinks)
	if err != nil {
		return errors.WithStack(err)
	}

	return u.SendBatches(testReport, batchLinks.Links.PostBatch)
}

func (u Uploader) SendBatches(rep *TestReport, url string) error {
	batch := [][]SourceFile{}

	pos := 0
	count := len(rep.SourceFiles) / u.BatchSize
	remainder := len(rep.SourceFiles) % u.BatchSize
	for i := 0; i < count; i++ {
		end := pos + u.BatchSize
		batch = append(batch, rep.SourceFiles[pos:end])
		pos = end
	}
	if remainder > 0 {
		batch = append(batch, rep.SourceFiles[pos:])
	}

	for i, b := range batch {
		pr, pw := io.Pipe()
		go func() {
			defer pw.Close()
			bb := &bytes.Buffer{}
			w := io.MultiWriter(pw, bb)
			err := json.NewEncoder(w).Encode(JSONWraper{
				Data: b,
				Meta: map[string]int{
					"current": i + 1,
					"total":   len(batch),
				},
			})
			if err != nil {
				logrus.Error(err)
				return
			}
			logrus.Debug(bb.String())
		}()
		_, err := u.doRequest(pr, url)
		if err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

func (u Uploader) doRequest(in io.Reader, url string) (*http.Response, error) {
	c := http.Client{
		Timeout: 30 * time.Second,
	}

	req, err := u.newRequest(in, url)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	logrus.Debugf("posting request to %s", url)
	res, err := c.Do(req)
	if err != nil {
		return res, errors.WithStack(err)
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return res, fmt.Errorf("response from %s was %d", url, res.StatusCode)
	}
	return res, nil
}

func (u Uploader) newRequest(in io.Reader, url string) (*http.Request, error) {
	req, err := http.NewRequest("POST", url, in)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	req.Header.Set("User-Agent", fmt.Sprintf("TestReporter/%s (Code Climate, Inc.)", version.Version))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-CC-Test-Reporter-Id", u.ReporterID)
	req.Header.Set("Accept", "application/vnd.api+json")
	return req, err
}
<|endoftext|>


<commit_before>package loggen

import (
	"bufio"
	"fmt"
	"math"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	useragentList []string
	categoryList  []string
	endTime       = time.Now()
	currentTime   = endTime
)

func causeErr(errRate float64) bool {
	rand.Seed(time.Now().UnixNano())
	for i := 1.0; ; i *= 10 {
		if errRate*i >= 1.0 {
			n := rand.Intn(100 * int(i))
			if int(errRate*i) > n {
				return true
			} else {
				return false
			}
		}
	}
}

func floatToIntString(input float64) string {
	return strconv.Itoa(int(input))
}

func intToString(input int) string {
	return strconv.Itoa(input)
}

func outputMultipleRecord(days int, errRate float64) {
	for i := 0; i < randInt(1, 3); i++ {
		fmt.Println(GetRecord(days, errRate))
	}
}

func outputToFile(days int, errRate float64, filename string) {
	file, _ := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	defer file.Close()

	writer := bufio.NewWriter(file)
	for i := 0; i < randInt(1, 3); i++ {
		record := ([]byte)(GetRecord(days, errRate) + "\n")
		writer.Write(record)
		writer.Flush()
	}
}

func randInt(min int, max int) int {
	return min + rand.Intn(max-min)
}

// Generate random number based on log-normal distribution
func randLogNormal(mu, sigma float64) float64 {
	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1).Float64()
	r2 := rand.New(s1).Float64()
	z := mu + sigma*math.Sqrt(-2.0*math.Log(r1))*math.Sin(2.0*math.Pi*r2)
	return math.Exp(z)
}

func returnNewList(path string) []string {
	var newList []string

	absPath, _ := filepath.Abs(path)
	fp, err := os.Open(absPath)
	if err != nil {
		panic(err)
	}
	scanner := bufio.NewScanner(fp)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		newList = append(newList, scanner.Text())
	}

	return newList
}

func zitter(i int) int {
	rand.Seed(time.Now().UnixNano())
	min := i - randInt(1, 2)
	max := i + randInt(1, 2)
	if min < 0 {
		min = 1
	}
	return min + rand.Intn(max-min)
}

// TODO: exclude private IP address
func Ipv4Address() string {
	var ipStr string
	ipStr = intToString(randInt(1, 223)) + "."
	ipStr += intToString(randInt(0, 255)) + "."
	ipStr += intToString(randInt(0, 255)) + "."
	ipStr += intToString(randInt(0, 255))
	return ipStr
}

func RequestTime(i int) string {
	returnTime := currentTime.Add(time.Second * time.Duration(i))
	return returnTime.Format("02/Jan/2006:15:04:05 -0700")
}

func RequestType() string {
	s := []string{"GET", "POST", "PUT", "DELETE"}
	return s[rand.Intn(len(s))]
}

func HttpStatusCode(errRate float64) string {
	rand.NewSource(time.Now().UnixNano())
	if causeErr(errRate) == false {
		return "200"
	} else {
		s := []string{"301", "403", "404", "500"}
		return s[rand.Intn(len(s))]
	}
}

func UserAgent() string {
	rand.Seed(time.Now().UTC().UnixNano())
	if len(useragentList) == 0 {
		useragentList = returnNewList(os.Getenv("GOPATH") + "/src/github.com/acroquest/apache-loggen-go/resources/useragents.txt")
	}
	useragent := useragentList[rand.Intn(len(useragentList))]
	return useragent
}

func Request() string {
	if len(categoryList) == 0 {
		categoryList = returnNewList(os.Getenv("GOPATH") + "/src/github.com/acroquest/apache-loggen-go/resources/categories.txt")
	}
	category := categoryList[rand.Intn(len(categoryList))]

	i := rand.Intn(10)
	if i < 7 {
		return "\"" + RequestType() + " /category/" + category + " HTTP/1.1\" "
	} else {
		return "\"" + RequestType() + " /" + category + "/" + intToString(randInt(1, 999)) + " HTTP/1.1\" "
	}
}

func Referer() string {
	referer := "-"
	return "\"" + referer + "\""
}

func SizeofBytes(bytes int) string {
	return floatToIntString(randLogNormal(0.0, 0.5) * float64(bytes))
}

func ResponseTime(millisecond int) string {
	return floatToIntString(randLogNormal(0.0, 0.5) * float64(millisecond))
}

func GetRecord(i int, errRate float64) string {
	return Ipv4Address() + " - - [" + RequestTime(i) + "] " + Request() + HttpStatusCode(errRate) + " " + SizeofBytes(2000) + " " + Referer() + " \"" + UserAgent() + "\" " + ResponseTime(20000)
}

// TODO change the amount of log data every day.
func GenerateLog(days int, errRate float64) {
	var weight int
	currentTime = endTime.Add(-24 * time.Hour * time.Duration(days))
	beforeHour := endTime.Hour()

	// generating log data every 1 second
	for i := 0; endTime.Sub(currentTime.Add(time.Second*time.Duration(i))) >= 0; i += 1 {
		hour := currentTime.Add(time.Second * time.Duration(i)).Hour()
		rand.Seed(time.Now().UnixNano())
		j := rand.Intn(10)

		// add weight for adding variation to the amount of data
		if hour != beforeHour {
			weight = zitter(randInt(1, 3))
			beforeHour = hour
		}

		switch {
		case hour >= 1 && hour <= 5:
			if j <= 2+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 6 && hour <= 9:
			if j <= 4+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 10 && hour <= 17:
			if j <= 6+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 18 && hour <= 23:
			if j <= 6+weight {
				outputMultipleRecord(i, errRate)
			}
		default:
			if j <= 4+weight {
				outputMultipleRecord(i, errRate)
			}
		}
	}
}

func GenerateLogToFile(days int, errRate float64, filename string) {
	var weight int
	currentTime = endTime.Add(-24 * time.Hour * time.Duration(days))
	beforeHour := endTime.Hour()

	// generating log data every 1 second
	for i := 0; endTime.Sub(currentTime.Add(time.Second*time.Duration(i))) >= 0; i += 1 {
		hour := currentTime.Add(time.Second * time.Duration(i)).Hour()
		rand.Seed(time.Now().UnixNano())
		j := rand.Intn(10)

		// add weight for adding variation to the amount of data
		if hour != beforeHour {
			weight = zitter(randInt(1, 3))
			beforeHour = hour
		}

		switch {
		case hour >= 1 && hour <= 5:
			if j <= 2+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 6 && hour <= 9:
			if j <= 4+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 10 && hour <= 17:
			if j <= 6+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 18 && hour <= 23:
			if j <= 6+weight {
				outputToFile(i, errRate, filename)
			}
		default:
			if j <= 4+weight {
				outputToFile(i, errRate, filename)
			}
		}
	}
}
<commit_msg>rename variable 'currentTime' to 'marker'<commit_after>package loggen

import (
	"bufio"
	"fmt"
	"math"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	useragentList []string
	categoryList  []string
	endTime       = time.Now()
	marker        = endTime
)

func causeErr(errRate float64) bool {
	rand.Seed(time.Now().UnixNano())
	for i := 1.0; ; i *= 10 {
		if errRate*i >= 1.0 {
			n := rand.Intn(100 * int(i))
			if int(errRate*i) > n {
				return true
			} else {
				return false
			}
		}
	}
}

func floatToIntString(input float64) string {
	return strconv.Itoa(int(input))
}

func intToString(input int) string {
	return strconv.Itoa(input)
}

func outputMultipleRecord(days int, errRate float64) {
	for i := 0; i < randInt(1, 3); i++ {
		fmt.Println(GetRecord(days, errRate))
	}
}

func outputToFile(days int, errRate float64, filename string) {
	file, _ := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	defer file.Close()

	writer := bufio.NewWriter(file)
	for i := 0; i < randInt(1, 3); i++ {
		record := ([]byte)(GetRecord(days, errRate) + "\n")
		writer.Write(record)
		writer.Flush()
	}
}

func randInt(min int, max int) int {
	return min + rand.Intn(max-min)
}

// Generate random number based on log-normal distribution
func randLogNormal(mu, sigma float64) float64 {
	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1).Float64()
	r2 := rand.New(s1).Float64()
	z := mu + sigma*math.Sqrt(-2.0*math.Log(r1))*math.Sin(2.0*math.Pi*r2)
	return math.Exp(z)
}

func returnNewList(path string) []string {
	var newList []string

	absPath, _ := filepath.Abs(path)
	fp, err := os.Open(absPath)
	if err != nil {
		panic(err)
	}
	scanner := bufio.NewScanner(fp)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		newList = append(newList, scanner.Text())
	}

	return newList
}

func zitter(i int) int {
	rand.Seed(time.Now().UnixNano())
	min := i - randInt(1, 2)
	max := i + randInt(1, 2)
	if min < 0 {
		min = 1
	}
	return min + rand.Intn(max-min)
}

// TODO: exclude private IP address
func Ipv4Address() string {
	var ipStr string
	ipStr = intToString(randInt(1, 223)) + "."
	ipStr += intToString(randInt(0, 255)) + "."
	ipStr += intToString(randInt(0, 255)) + "."
	ipStr += intToString(randInt(0, 255))
	return ipStr
}

func RequestTime(i int) string {
	returnTime := marker.Add(time.Second * time.Duration(i))
	return returnTime.Format("02/Jan/2006:15:04:05 -0700")
}

func RequestType() string {
	s := []string{"GET", "POST", "PUT", "DELETE"}
	return s[rand.Intn(len(s))]
}

func HttpStatusCode(errRate float64) string {
	rand.NewSource(time.Now().UnixNano())
	if causeErr(errRate) == false {
		return "200"
	} else {
		s := []string{"301", "403", "404", "500"}
		return s[rand.Intn(len(s))]
	}
}

func UserAgent() string {
	rand.Seed(time.Now().UTC().UnixNano())
	if len(useragentList) == 0 {
		useragentList = returnNewList(os.Getenv("GOPATH") + "/src/github.com/acroquest/apache-loggen-go/resources/useragents.txt")
	}
	useragent := useragentList[rand.Intn(len(useragentList))]
	return useragent
}

func Request() string {
	if len(categoryList) == 0 {
		categoryList = returnNewList(os.Getenv("GOPATH") + "/src/github.com/acroquest/apache-loggen-go/resources/categories.txt")
	}
	category := categoryList[rand.Intn(len(categoryList))]

	i := rand.Intn(10)
	if i < 7 {
		return "\"" + RequestType() + " /category/" + category + " HTTP/1.1\" "
	} else {
		return "\"" + RequestType() + " /" + category + "/" + intToString(randInt(1, 999)) + " HTTP/1.1\" "
	}
}

func Referer() string {
	referer := "-"
	return "\"" + referer + "\""
}

func SizeofBytes(bytes int) string {
	return floatToIntString(randLogNormal(0.0, 0.5) * float64(bytes))
}

func ResponseTime(millisecond int) string {
	return floatToIntString(randLogNormal(0.0, 0.5) * float64(millisecond))
}

func GetRecord(i int, errRate float64) string {
	return Ipv4Address() + " - - [" + RequestTime(i) + "] " + Request() + HttpStatusCode(errRate) + " " + SizeofBytes(2000) + " " + Referer() + " \"" + UserAgent() + "\" " + ResponseTime(20000)
}

// TODO change the amount of log data every day.
func GenerateLog(days int, errRate float64) {
	var weight int
	marker = endTime.Add(-24 * time.Hour * time.Duration(days))
	beforeHour := endTime.Hour()

	// generating log data every 1 second
	for i := 0; endTime.Sub(marker.Add(time.Second*time.Duration(i))) >= 0; i += 1 {
		hour := marker.Add(time.Second * time.Duration(i)).Hour()
		rand.Seed(time.Now().UnixNano())
		j := rand.Intn(10)

		// add weight for adding variation to the amount of data
		if hour != beforeHour {
			weight = zitter(randInt(1, 3))
			beforeHour = hour
		}

		switch {
		case hour >= 1 && hour <= 5:
			if j <= 2+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 6 && hour <= 9:
			if j <= 4+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 10 && hour <= 17:
			if j <= 6+weight {
				outputMultipleRecord(i, errRate)
			}
		case hour >= 18 && hour <= 23:
			if j <= 6+weight {
				outputMultipleRecord(i, errRate)
			}
		default:
			if j <= 4+weight {
				outputMultipleRecord(i, errRate)
			}
		}
	}
}

func GenerateLogToFile(days int, errRate float64, filename string) {
	var weight int
	marker = endTime.Add(-24 * time.Hour * time.Duration(days))
	beforeHour := endTime.Hour()

	// generating log data every 1 second
	for i := 0; endTime.Sub(marker.Add(time.Second*time.Duration(i))) >= 0; i += 1 {
		hour := marker.Add(time.Second * time.Duration(i)).Hour()
		rand.Seed(time.Now().UnixNano())
		j := rand.Intn(10)

		// add weight for adding variation to the amount of data
		if hour != beforeHour {
			weight = zitter(randInt(1, 3))
			beforeHour = hour
		}

		switch {
		case hour >= 1 && hour <= 5:
			if j <= 2+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 6 && hour <= 9:
			if j <= 4+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 10 && hour <= 17:
			if j <= 6+weight {
				outputToFile(i, errRate, filename)
			}
		case hour >= 18 && hour <= 23:
			if j <= 6+weight {
				outputToFile(i, errRate, filename)
			}
		default:
			if j <= 4+weight {
				outputToFile(i, errRate, filename)
			}
		}
	}
}
<|endoftext|>