<commit_before>package main

import (
	"encoding/hex"
	"fmt"
	"github.com/godbus/dbus"
	"github.com/paypal/gatt"
	"log"
	"strings"
)

type BleDbusWrapper struct {
	bus               *dbus.Conn
	device            gatt.Device
	connected         bool
	devicesDiscovered map[string]*DiscoveredDeviceInfo
}

type DiscoveredDeviceInfo struct {
	name            string
	rssi            int
	peripheral      gatt.Peripheral
	characteristics map[string]*gatt.Characteristic
	ready           bool
}

type gattCommandHandler func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error)

func newDHError(message string) *dbus.Error {
	return dbus.NewError("com.devicehive.Error",
		[]interface{}{message})
}

func normalizeHex(s string) (res string, err error) {
	trimmed := strings.Map(func(r rune) rune {
		if !strings.ContainsRune(":- ", r) {
			return r
		}
		return -1
	}, s)

	b, err := hex.DecodeString(trimmed)

	if err != nil {
		return "", err
	}

	return hex.EncodeToString(b), nil
}

func NewBleDbusWrapper(bus *dbus.Conn) *BleDbusWrapper {
	d, err := gatt.NewDevice([]gatt.Option{
		gatt.LnxDeviceID(0, false),
	}...)

	if err != nil {
		log.Fatalf("Failed to open device, err: %s\n", err)
		return nil
	}

	wrapper := new(BleDbusWrapper)
	wrapper.bus = bus
	wrapper.device = d
	wrapper.devicesDiscovered = make(map[string]*DiscoveredDeviceInfo)

	d.Handle(gatt.PeripheralDiscovered(func(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
		id, _ := normalizeHex(p.ID())
		if _, ok := wrapper.devicesDiscovered[id]; !ok {
			wrapper.devicesDiscovered[id] = &DiscoveredDeviceInfo{name: p.Name(), rssi: rssi, peripheral: p}
			log.Printf("Adding mac: %s", id)
		}
		bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDiscovered", id, p.Name(), int16(rssi))
	}))

	d.Handle(gatt.PeripheralConnected(func(p gatt.Peripheral, err error) {
		id, _ := normalizeHex(p.ID())
		if _, ok := wrapper.devicesDiscovered[id]; ok {
			dev := wrapper.devicesDiscovered[id]
			dev.characteristics = make(map[string]*gatt.Characteristic)
			dev.ready = false
			dev.peripheral = p
			dev.explorePeripheral(dev.peripheral)
			dev.ready = true
			bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceConnected", id)
		}
	}))

	d.Handle(gatt.PeripheralDisconnected(func(p gatt.Peripheral, err error) {
		id, _ := normalizeHex(p.ID())
		if dev, ok := wrapper.devicesDiscovered[id]; ok {
			dev.ready = false
			bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDisconnected", id)
		}
	}))

	d.Init(func(dev gatt.Device, s gatt.State) {
		switch s {
		case gatt.StatePoweredOn:
			log.Print("HCI device powered on")
			wrapper.connected = true
			return
		default:
			log.Printf("StateChanged handler received: %v", s)
			wrapper.connected = false
		}
	})
	wrapper.device = d
	return wrapper
}

func (w *BleDbusWrapper) ScanStart() *dbus.Error {
	if !w.connected {
		return newDHError("Disconnected")
	}

	w.device.Scan(nil, false)

	// Just let them know devices that are cached, as they might be connected and
	// no longer advertising. Put RSSI to 0.
	go func() {
		for k, v := range w.devicesDiscovered {
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDiscovered", k, v.name, int16(0))
		}
	}()

	return nil
}

func (w *BleDbusWrapper) ScanStop() *dbus.Error {
	if !w.connected {
		return newDHError("Disconnected")
	}

	w.device.StopScanning()
	return nil
}

func (w *BleDbusWrapper) Connect(mac string) *dbus.Error {
	mac, err := normalizeHex(mac)

	if err != nil {
		return newDHError("Invalid MAC provided")
	}

	log.Printf("Connecting to: %s", mac)

	if val, ok := w.devicesDiscovered[mac]; ok {
		w.device.Connect(val.peripheral)
		return nil
	}

	log.Print("MAC wasn't descovered")
	return newDHError("MAC wasn't descovered, use Scan Start/Stop first")
}

func (w *BleDbusWrapper) Disconnect(mac string) *dbus.Error {
	if val, ok := w.devicesDiscovered[mac]; ok {
		w.device.CancelConnection(val.peripheral)
		return nil
	}

	log.Print("MAC wasn't descovered")
	return newDHError("MAC wasn't descovered, use Scan Start/Stop first")
}

func (w *BleDbusWrapper) handleGattCommand(mac string, uuid string, message string, handler gattCommandHandler) (string, *dbus.Error) {
	mac, _ = normalizeHex(mac)
	uuid, _ = normalizeHex(uuid)

	res := ""

	if val, ok := w.devicesDiscovered[mac]; ok {
		if !w.devicesDiscovered[mac].ready {
			log.Printf("Device %s is not ready (probably still connecting, or discoverig services and characteristics)", mac)
			return "", newDHError("Device not ready")
		}

		log.Printf("GATT COMMAND to mac %v char %v", mac, uuid)
		var b []byte
		var err error

		if message != "" {
			b, err = hex.DecodeString(message)
			if err != nil {
				log.Printf("Invalid message: %s", message)
				return "", newDHError(err.Error())
			}
		}

		if c, ok := val.characteristics[uuid]; ok {
			b, err = handler(val.peripheral, c, b)

			if b != nil {
				res = hex.EncodeToString(b)
			}

			if err != nil {
				log.Printf("Error writing/reading characteristic: %s", err)
				return "", newDHError(err.Error())
			}
		} else {
			s := fmt.Sprintf("Characteristic %s not found. Please try full name and check the device spec.", uuid)
			log.Print(s)
			return "", newDHError(s)
		}

	} else {
		log.Printf("Invalid peripheral ID: %s", mac)
		return "", newDHError("Invalid peripheral ID")
	}

	return res, nil
}

func (w *BleDbusWrapper) GattWrite(mac string, uuid string, message string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		error := p.WriteCharacteristic(c, b, false)
		return nil, error
	}

	_, error := w.handleGattCommand(mac, uuid, message, h)
	return error
}

func (w *BleDbusWrapper) GattRead(mac string, uuid string) (string, *dbus.Error) {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		return p.ReadCharacteristic(c)
	}

	return w.handleGattCommand(mac, uuid, "", h)
}

func (w *BleDbusWrapper) GattNotifications(mac string, uuid string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		err := p.SetNotifyValue(c, func(_ *gatt.Characteristic, b []byte, e error) {
			if e != nil {
				log.Printf("Notification handler received error: %s", e)
				return
			}

			m := hex.EncodeToString(b)
			log.Printf("Received notification: %s", m)
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.NotificationReceived", mac, uuid, m)
		})
		return nil, err
	}

	_, err := w.handleGattCommand(mac, uuid, "", h)
	return err
}

func (w *BleDbusWrapper) GattIndications(mac string, uuid string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		err := p.SetIndicateValue(c, func(_ *gatt.Characteristic, b []byte, e error) {
			if e != nil {
				log.Printf("Indications handler received error: %s", e)
				return
			}

			m := hex.EncodeToString(b)
			log.Printf("Received indication: %s", m)
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.IndicationReceived", mac, uuid, m)
		})
		return nil, err
	}

	_, err := w.handleGattCommand(mac, uuid, "", h)
	return err
}

func (b *DiscoveredDeviceInfo) explorePeripheral(p gatt.Peripheral) error {
	log.Println("Connected")

	ss, err := p.DiscoverServices(nil)
	if err != nil {
		log.Printf("Failed to discover services, err: %s\n", err)
		return err
	}

	for _, s := range ss {
		msg := "Service: " + s.UUID().String()
		if len(s.Name()) > 0 {
			msg += " (" + s.Name() + ")"
		}
		log.Println(msg)

		// Discovery characteristics
		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			log.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			msg := "  Characteristic  " + c.UUID().String()
			if len(c.Name()) > 0 {
				msg += " (" + c.Name() + ")"
			}
			msg += "\n    properties    " + c.Properties().String()
			log.Println(msg)

			// Discovery descriptors
			ds, err := p.DiscoverDescriptors(nil, c)
			if err != nil {
				log.Printf("Failed to discover descriptors, err: %s\n", err)
				continue
			}

			for _, d := range ds {
				msg := "  Descriptor      " + d.UUID().String()
				if len(d.Name()) > 0 {
					msg += " (" + d.Name() + ")"
				}
				log.Println(msg)
			}

			id, _ := normalizeHex(c.UUID().String())
			b.characteristics[id] = c
		}
		log.Println("Done exploring peripheral")
		b.ready = true
	}

	return nil
}

func main() {
	var err error
	var bus *dbus.Conn
	bus, err = dbus.SystemBus()
	if err != nil {
		log.Panic(err)
	}

	reply, err := bus.RequestName("com.devicehive.bluetooth",
		dbus.NameFlagDoNotQueue)
	if err != nil {
		log.Panic(err)
	}

	if reply != dbus.RequestNameReplyPrimaryOwner {
		log.Fatal("name already taken")
	}

	w := NewBleDbusWrapper(bus)

	log.Printf("Exporting BleDbusWrapper...")

	c := make(chan *dbus.Signal)
	bus.Signal(c)

	go func(ch chan *dbus.Signal) {
		for c := range ch {
			log.Printf("Signal: %+v", c)
		}
	}(c)

	bus.Export(w, "/com/devicehive/bluetooth", "com.devicehive.bluetooth")

	select {}
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/hex"
	"fmt"
	"github.com/godbus/dbus"
	"github.com/paypal/gatt"
	"log"
	"strings"
)

type BleDbusWrapper struct {
	bus               *dbus.Conn
	device            gatt.Device
	connected         bool
	devicesDiscovered map[string]*DiscoveredDeviceInfo
}

type DiscoveredDeviceInfo struct {
	name            string
	rssi            int
	peripheral      gatt.Peripheral
	characteristics map[string]*gatt.Characteristic
	ready           bool
}

type gattCommandHandler func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error)

func newDHError(message string) *dbus.Error {
	return dbus.NewError("com.devicehive.Error",
		[]interface{}{message})
}

func normalizeHex(s string) (res string, err error) {
	trimmed := strings.Map(func(r rune) rune {
		if !strings.ContainsRune(":- ", r) {
			return r
		}
		return -1
	}, s)

	b, err := hex.DecodeString(trimmed)

	if err != nil {
		return "", err
	}

	return hex.EncodeToString(b), nil
}

func NewBleDbusWrapper(bus *dbus.Conn) *BleDbusWrapper {
	d, err := gatt.NewDevice([]gatt.Option{
		gatt.LnxDeviceID(0, false),
	}...)

	if err != nil {
		log.Fatalf("Failed to open device, err: %s\n", err)
		return nil
	}

	wrapper := new(BleDbusWrapper)
	wrapper.bus = bus
	wrapper.device = d
	wrapper.devicesDiscovered = make(map[string]*DiscoveredDeviceInfo)

	d.Handle(gatt.PeripheralDiscovered(func(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
		id, _ := normalizeHex(p.ID())
		if _, ok := wrapper.devicesDiscovered[id]; !ok {
			wrapper.devicesDiscovered[id] = &DiscoveredDeviceInfo{name: p.Name(), rssi: rssi, peripheral: p}
			log.Printf("Adding mac: %s", id)
		}
		bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDiscovered", id, p.Name(), int16(rssi))
	}))

	d.Handle(gatt.PeripheralConnected(func(p gatt.Peripheral, err error) {
		id, _ := normalizeHex(p.ID())
		if _, ok := wrapper.devicesDiscovered[id]; ok {
			dev := wrapper.devicesDiscovered[id]
			dev.characteristics = make(map[string]*gatt.Characteristic)
			dev.ready = false
			dev.peripheral = p
			dev.explorePeripheral(dev.peripheral)
			dev.ready = true
			bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceConnected", id)
		}
	}))

	d.Handle(gatt.PeripheralDisconnected(func(p gatt.Peripheral, err error) {
		id, _ := normalizeHex(p.ID())
		if dev, ok := wrapper.devicesDiscovered[id]; ok {
			dev.ready = false
			bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDisconnected", id)
		}
	}))

	d.Init(func(dev gatt.Device, s gatt.State) {
		switch s {
		case gatt.StatePoweredOn:
			log.Print("HCI device powered on")
			wrapper.connected = true
			return
		default:
			log.Printf("StateChanged handler received: %v", s)
			wrapper.connected = false
		}
	})
	wrapper.device = d
	return wrapper
}

func (w *BleDbusWrapper) ScanStart() *dbus.Error {
	if !w.connected {
		return newDHError("Disconnected")
	}

	w.device.Scan(nil, false)

	// Just let them know devices that are cached, as they might be connected and
	// no longer advertising. Put RSSI to 0.
	go func() {
		for k, v := range w.devicesDiscovered {
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.DeviceDiscovered", k, v.name, int16(0))
		}
	}()

	return nil
}

func (w *BleDbusWrapper) ScanStop() *dbus.Error {
	if !w.connected {
		return newDHError("Disconnected")
	}

	w.device.StopScanning()
	return nil
}

func (w *BleDbusWrapper) Connect(mac string) *dbus.Error {
	mac, err := normalizeHex(mac)

	if err != nil {
		return newDHError("Invalid MAC provided")
	}

	log.Printf("Connecting to: %s", mac)

	if val, ok := w.devicesDiscovered[mac]; ok {
		w.device.Connect(val.peripheral)
		return nil
	}

	log.Print("MAC wasn't descovered")
	return newDHError("MAC wasn't descovered, use Scan Start/Stop first")
}

func (w *BleDbusWrapper) Disconnect(mac string) *dbus.Error {
	if val, ok := w.devicesDiscovered[mac]; ok {
		w.device.CancelConnection(val.peripheral)
		return nil
	}

	log.Print("MAC wasn't descovered")
	return newDHError("MAC wasn't descovered, use Scan Start/Stop first")
}

func (w *BleDbusWrapper) handleGattCommand(mac string, uuid string, message string, handler gattCommandHandler) (string, *dbus.Error) {
	mac, _ = normalizeHex(mac)
	uuid, _ = normalizeHex(uuid)

	res := ""

	if val, ok := w.devicesDiscovered[mac]; ok {
		if !w.devicesDiscovered[mac].ready {
			log.Printf("Device %s is not ready (probably still connecting, or discoverig services and characteristics)", mac)
			return "", newDHError("Device not ready")
		}

		log.Printf("GATT COMMAND to mac %v char %v", mac, uuid)
		var b []byte
		var err error

		if message != "" {
			b, err = hex.DecodeString(message)
			if err != nil {
				log.Printf("Invalid message: %s", message)
				return "", newDHError(err.Error())
			}
		}

		if c, ok := val.characteristics[uuid]; ok {
			b, err = handler(val.peripheral, c, b)

			if b != nil {
				res = hex.EncodeToString(b)
			}

			if err != nil {
				log.Printf("Error writing/reading characteristic: %s", err)
				return "", newDHError(err.Error())
			}
		} else {
			s := fmt.Sprintf("Characteristic %s not found. Please try full name and check the device spec.", uuid)
			log.Print(s)
			return "", newDHError(s)
		}

	} else {
		log.Printf("Invalid peripheral ID: %s", mac)
		return "", newDHError("Invalid peripheral ID")
	}

	return res, nil
}

func (w *BleDbusWrapper) GattWrite(mac string, uuid string, message string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		error := p.WriteCharacteristic(c, b, false)
		return nil, error
	}

	_, error := w.handleGattCommand(mac, uuid, message, h)
	return error
}

func (w *BleDbusWrapper) GattRead(mac string, uuid string) (string, *dbus.Error) {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		return p.ReadCharacteristic(c)
	}

	return w.handleGattCommand(mac, uuid, "", h)
}

func (w *BleDbusWrapper) GattNotifications(mac string, uuid string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		err := p.SetNotifyValue(c, func(_ *gatt.Characteristic, b []byte, e error) {
			if e != nil {
				log.Printf("Notification handler received error: %s", e)
				return
			}

			m := hex.EncodeToString(b)
			log.Printf("Received notification: %s", m)
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.NotificationReceived", mac, uuid, m)
		})
		return nil, err
	}

	_, err := w.handleGattCommand(mac, uuid, "", h)
	return err
}

func (w *BleDbusWrapper) GattIndications(mac string, uuid string) *dbus.Error {
	h := func(p gatt.Peripheral, c *gatt.Characteristic, b []byte) ([]byte, error) {
		err := p.SetIndicateValue(c, func(_ *gatt.Characteristic, b []byte, e error) {
			if e != nil {
				log.Printf("Indications handler received error: %s", e)
				return
			}

			m := hex.EncodeToString(b)
			log.Printf("Received indication: %s", m)
			w.bus.Emit("/com/devicehive/bluetooth", "com.devicehive.bluetooth.IndicationReceived", mac, uuid, m)
		})
		return nil, err
	}

	_, err := w.handleGattCommand(mac, uuid, "", h)
	return err
}

func (b *DiscoveredDeviceInfo) explorePeripheral(p gatt.Peripheral) error {
	log.Println("Connected")

	ss, err := p.DiscoverServices(nil)
	if err != nil {
		log.Printf("Failed to discover services, err: %s\n", err)
		return err
	}

	for _, s := range ss {
		msg := "Service: " + s.UUID().String()
		if len(s.Name()) > 0 {
			msg += " (" + s.Name() + ")"
		}
		log.Println(msg)

		// Discovery characteristics
		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			log.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			msg := "  Characteristic  " + c.UUID().String()
			if len(c.Name()) > 0 {
				msg += " (" + c.Name() + ")"
			}
			msg += "\n    properties    " + c.Properties().String()
			log.Println(msg)

			// Discovery descriptors
			ds, err := p.DiscoverDescriptors(nil, c)
			if err != nil {
				log.Printf("Failed to discover descriptors, err: %s\n", err)
				continue
			}

			for _, d := range ds {
				msg := "  Descriptor      " + d.UUID().String()
				if len(d.Name()) > 0 {
					msg += " (" + d.Name() + ")"
				}
				log.Println(msg)
			}

			id, _ := normalizeHex(c.UUID().String())
			b.characteristics[id] = c
		}
		log.Println("Done exploring peripheral")
		b.ready = true
	}

	return nil
}

func main() {
	var err error
	var bus *dbus.Conn
	bus, err = dbus.SystemBus()
	if err != nil {
		log.Panic(err)
	}

	reply, err := bus.RequestName("com.devicehive.bluetooth",
		dbus.NameFlagDoNotQueue)
	if err != nil {
		log.Panic(err)
	}

	if reply != dbus.RequestNameReplyPrimaryOwner {
		log.Fatal("name already taken")
	}

	w := NewBleDbusWrapper(bus)
	bus.Export(w, "/com/devicehive/bluetooth", "com.devicehive.bluetooth")

	select {}
}
<|endoftext|>


<commit_before>package validation

import (
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation/field"

	oapi "github.com/openshift/origin/pkg/api"
	projectapi "github.com/openshift/origin/pkg/project/apis/project"
)

func TestValidateProject(t *testing.T) {
	testCases := []struct {
		name    string
		project projectapi.Project
		numErrs int
	}{
		{
			name: "missing id",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the ID is missing.
			numErrs: 1,
		},
		{
			name: "invalid id",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "141-.124.$",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the ID is invalid.
			numErrs: 1,
		},
		{
			name: "invalid id uppercase",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "AA",
				},
			},
			numErrs: 1,
		},
		{
			name: "valid id leading number",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "11",
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid id for create (< 2 characters)",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "h",
				},
			},
			numErrs: 1,
		},
		{
			name: "valid id for create (2+ characters)",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "hi",
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid id internal dots",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "1.a.1",
				},
			},
			numErrs: 1,
		},
		{
			name: "has namespace",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "foo",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the namespace is supplied.
			numErrs: 1,
		},
		{
			name: "invalid display name",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "h\t\ni",
					},
				},
			},
			// Should fail because the display name has \t \n
			numErrs: 1,
		},
		{
			name: "valid node selector",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						projectapi.ProjectNodeSelector: "infra=true, env = test",
					},
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid node selector",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						projectapi.ProjectNodeSelector: "infra, env = $test",
					},
				},
			},
			// Should fail because infra and $test doesn't satisfy the format
			numErrs: 1,
		},
	}

	for _, tc := range testCases {
		errs := ValidateProject(&tc.project)
		if len(errs) != tc.numErrs {
			t.Errorf("Unexpected error list for case %q: %+v", tc.name, errs)
		}
	}

	project := projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name: "foo",
			Annotations: map[string]string{
				oapi.OpenShiftDescription: "This is a description",
				oapi.OpenShiftDisplayName: "hi",
			},
		},
	}
	errs := ValidateProject(&project)
	if len(errs) != 0 {
		t.Errorf("Unexpected non-zero error list: %#v", errs)
	}
}

func TestValidateProjectUpdate(t *testing.T) {
	// Ensure we can update projects with short names, to make sure we can
	// proxy updates to namespaces created outside project validation
	project := &projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "project-name",
			ResourceVersion: "1",
			Annotations: map[string]string{
				oapi.OpenShiftDescription:      "This is a description",
				oapi.OpenShiftDisplayName:      "display name",
				projectapi.ProjectNodeSelector: "infra=true, env = test",
			},
			Labels: map[string]string{"label-name": "value"},
		},
	}
	updateDisplayname := &projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "project-name",
			ResourceVersion: "1",
			Annotations: map[string]string{
				oapi.OpenShiftDescription:      "This is a description",
				oapi.OpenShiftDisplayName:      "display name change",
				projectapi.ProjectNodeSelector: "infra=true, env = test",
			},
			Labels: map[string]string{"label-name": "value"},
		},
	}

	errs := ValidateProjectUpdate(updateDisplayname, project)
	if len(errs) > 0 {
		t.Fatalf("Expected no errors, got %v", errs)
	}

	errorCases := map[string]struct {
		A projectapi.Project
		T field.ErrorType
		F string
	}{
		"change name": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "different",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
					Labels:          project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.name",
		},
		"invalid displayname": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription:      "This is a description",
						oapi.OpenShiftDisplayName:      "display name\n",
						projectapi.ProjectNodeSelector: "infra=true, env = test",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[" + oapi.OpenShiftDisplayName + "]",
		},
		"updating disallowed annotation": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription:      "This is a description",
						oapi.OpenShiftDisplayName:      "display name",
						projectapi.ProjectNodeSelector: "infra=true, env = test2",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[openshift.io/node-selector]",
		},
		"delete annotation": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "display name",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[openshift.io/node-selector]",
		},
		"updating label": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
					Labels:          map[string]string{"label-name": "diff"},
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.labels[label-name]",
		},
		"deleting label": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.labels[label-name]",
		},
	}
	for k, v := range errorCases {
		errs := ValidateProjectUpdate(&v.A, project)
		if len(errs) == 0 {
			t.Errorf("expected failure %s for %v", k, v.A)
			continue
		}
		for i := range errs {
			if errs[i].Type != v.T {
				t.Errorf("%s: expected errors to have type %s: %v", k, v.T, errs[i])
			}
			if errs[i].Field != v.F {
				t.Errorf("%s: expected errors to have field %s: %v", k, v.F, errs[i])
			}
		}
	}

}
<commit_msg>Add an error text check for project validation<commit_after>package validation

import (
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation/field"

	oapi "github.com/openshift/origin/pkg/api"
	projectapi "github.com/openshift/origin/pkg/project/apis/project"
)

func TestValidateProject(t *testing.T) {
	testCases := []struct {
		name    string
		project projectapi.Project
		numErrs int
		errText string
	}{
		{
			name: "missing id",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the ID is missing.
			numErrs: 1,
		},
		{
			name: "invalid id",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "141-.124.$",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the ID is invalid.
			numErrs: 1,
		},
		{
			name: "invalid id uppercase",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "AA",
				},
			},
			numErrs: 1,
		},
		{
			name: "valid id leading number",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "11",
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid id for create (< 2 characters)",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "h",
				},
			},
			numErrs: 1,
		},
		{
			name: "valid id for create (2+ characters)",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "hi",
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid id internal dots",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name: "1.a.1",
				},
			},
			numErrs: 1,
		},
		{
			name: "has namespace",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "foo",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "hi",
					},
				},
			},
			// Should fail because the namespace is supplied.
			numErrs: 1,
		},
		{
			name: "invalid display name",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "h\t\ni",
					},
				},
			},
			// Should fail because the display name has \t \n
			numErrs: 1,
		},
		{
			name: "valid node selector",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						projectapi.ProjectNodeSelector: "infra=true, env = test",
					},
				},
			},
			numErrs: 0,
		},
		{
			name: "invalid node selector",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						projectapi.ProjectNodeSelector: "infra, env = $test",
					},
				},
			},
			// Should fail because infra and $test doesn't satisfy the format
			numErrs: 1,
		},
		{
			name: "valid node selector",
			project: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "foo",
					Namespace: "",
					Annotations: map[string]string{
						projectapi.ProjectNodeSelector: "env,qa",
					},
				},
			},
			// Should fail because ProjectNodeSelector is invalid
			errText: `nodeSelector: Invalid value: "env,qa": must be a valid label selector`,
			numErrs: 1,
		},
	}

	for _, tc := range testCases {
		errs := ValidateProject(&tc.project)
		if len(errs) != tc.numErrs {
			t.Errorf("Unexpected error list for case %q: %+v", tc.name, errs)
		}
		for _, v := range errs {
			if tc.errText != "" && (tc.errText != string(v.Error())) {
				t.Errorf("Unexpected error text %q for %q", string(v.Error()), tc.errText)
			}
		}
	}

	project := projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name: "foo",
			Annotations: map[string]string{
				oapi.OpenShiftDescription: "This is a description",
				oapi.OpenShiftDisplayName: "hi",
			},
		},
	}
	errs := ValidateProject(&project)
	if len(errs) != 0 {
		t.Errorf("Unexpected non-zero error list: %#v", errs)
	}
}

func TestValidateProjectUpdate(t *testing.T) {
	// Ensure we can update projects with short names, to make sure we can
	// proxy updates to namespaces created outside project validation
	project := &projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "project-name",
			ResourceVersion: "1",
			Annotations: map[string]string{
				oapi.OpenShiftDescription:      "This is a description",
				oapi.OpenShiftDisplayName:      "display name",
				projectapi.ProjectNodeSelector: "infra=true, env = test",
			},
			Labels: map[string]string{"label-name": "value"},
		},
	}
	updateDisplayname := &projectapi.Project{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "project-name",
			ResourceVersion: "1",
			Annotations: map[string]string{
				oapi.OpenShiftDescription:      "This is a description",
				oapi.OpenShiftDisplayName:      "display name change",
				projectapi.ProjectNodeSelector: "infra=true, env = test",
			},
			Labels: map[string]string{"label-name": "value"},
		},
	}

	errs := ValidateProjectUpdate(updateDisplayname, project)
	if len(errs) > 0 {
		t.Fatalf("Expected no errors, got %v", errs)
	}

	errorCases := map[string]struct {
		A projectapi.Project
		T field.ErrorType
		F string
	}{
		"change name": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "different",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
					Labels:          project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.name",
		},
		"invalid displayname": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription:      "This is a description",
						oapi.OpenShiftDisplayName:      "display name\n",
						projectapi.ProjectNodeSelector: "infra=true, env = test",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[" + oapi.OpenShiftDisplayName + "]",
		},
		"updating disallowed annotation": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription:      "This is a description",
						oapi.OpenShiftDisplayName:      "display name",
						projectapi.ProjectNodeSelector: "infra=true, env = test2",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[openshift.io/node-selector]",
		},
		"delete annotation": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations: map[string]string{
						oapi.OpenShiftDescription: "This is a description",
						oapi.OpenShiftDisplayName: "display name",
					},
					Labels: project.Labels,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.annotations[openshift.io/node-selector]",
		},
		"updating label": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
					Labels:          map[string]string{"label-name": "diff"},
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.labels[label-name]",
		},
		"deleting label": {
			A: projectapi.Project{
				ObjectMeta: metav1.ObjectMeta{
					Name:            "project-name",
					ResourceVersion: "1",
					Annotations:     project.Annotations,
				},
			},
			T: field.ErrorTypeInvalid,
			F: "metadata.labels[label-name]",
		},
	}
	for k, v := range errorCases {
		errs := ValidateProjectUpdate(&v.A, project)
		if len(errs) == 0 {
			t.Errorf("expected failure %s for %v", k, v.A)
			continue
		}
		for i := range errs {
			if errs[i].Type != v.T {
				t.Errorf("%s: expected errors to have type %s: %v", k, v.T, errs[i])
			}
			if errs[i].Field != v.F {
				t.Errorf("%s: expected errors to have field %s: %v", k, v.F, errs[i])
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	MQTT "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"log"
	"time"
)

type MQTTPublisher struct {
	config *MqttProtocol
	client *MQTT.MqttClient
	dataCh chan AgentResponse
}

func newMQTTPublisher(conf *Config) *MQTTPublisher {
	// Check if we need to publish to MQTT
	config, ok := conf.Protocols[ProtocolTypeMQTT].(MqttProtocol)
	if !ok {
		return nil
	}

	requiresMqtt := false
	for _, d := range conf.Devices {
		for _, r := range d.Resources {
			for _, p := range r.Protocols {
				if p.Type == ProtocolTypeMQTT {
					requiresMqtt = true
					break
				}
				if requiresMqtt {
					break
				}
			}
			if requiresMqtt {
				break
			}
		}
		if requiresMqtt {
			break
		}
	}

	if !requiresMqtt {
		return nil
	}

	// Prepare MQTT connection opts
	broker := fmt.Sprintf("tcp://%s:%v", config.Host, config.Port)
	clientId := conf.Id
	connOpts := MQTT.NewClientOptions().AddBroker(broker).SetClientId(clientId).SetCleanSession(true).SetOnConnectionLost(onConnectionLost)

	// Create and return publisher
	publisher := &MQTTPublisher{
		config: &config,
		client: MQTT.NewClient(connOpts),
		dataCh: make(chan AgentResponse),
	}
	return publisher
}

func (self *MQTTPublisher) dataInbox() chan<- AgentResponse {
	return self.dataCh
}

func (self *MQTTPublisher) start() {
	log.Println("MQTTPublisher.start()")
	// start the connection routine
	log.Printf("MQTTPublisher: Will connect to the broker tcp://%s:%v", self.config.Host, self.config.Port)
	go connect(self.client, 0)

	qos := 1
	prefix := self.config.Prefix
	for resp := range self.dataCh {
		if !self.client.IsConnected() {
			log.Println("MQTTPublisher: got data while not connected to the broker. **discarded**")
			continue
		}
		if resp.IsError {
			log.Println("MQTTPublisher: data ERROR from agent manager:", string(resp.Payload))
			continue
		}
		topic := fmt.Sprintf("%s/%s", prefix, resp.ResourceId)
		self.client.Publish(MQTT.QoS(qos), topic, resp.Payload)
		// We dont' wait for confirmation from broker (avoid blocking here!)
		//<-r
		log.Println("MQTTPublisher: published to", topic)
	}
}

func (self *MQTTPublisher) stop() {
	log.Println("MQTTPublisher.stop()")
	if self.client != nil && self.client.IsConnected() {
		self.client.Disconnect(500)
	}
}

func connect(client *MQTT.MqttClient, backOff int) {
	log.Println("MQTTPublisher.connect() with backOff (sec): ", backOff)
	// sleep for backOff seconds
	time.Sleep(time.Duration(backOff) * time.Second)
	_, err := client.Start()

	if err != nil {
		log.Printf("Failed to connected to MQTT broker: %v\n", err.Error())
		// intial backOff 10 sec, every further retry backOff*2 unless <= 10 min
		if backOff == 0 {
			backOff = 10
		} else if backOff <= 600 {
			backOff *= 2
		}
		go connect(client, backOff)
		return
	}

	log.Printf("MQTTPublisher: Connected to the broker")
	return
}

func onConnectionLost(client *MQTT.MqttClient, reason error) {
	log.Println("MQTTPulbisher: lost connection to the broker: ", reason.Error())
	// FIXME: bug in mqtt library (panic on reconnect)?
	// go connect(client, 0)
}
<commit_msg>DGW/mqtt: fixed reconnection to the broker (#21)<commit_after>package main

import (
	"fmt"
	MQTT "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"log"
	"time"
)

type MQTTPublisher struct {
	config   *MqttProtocol
	clientId string
	client   *MQTT.MqttClient
	dataCh   chan AgentResponse
}

func newMQTTPublisher(conf *Config) *MQTTPublisher {
	// Check if we need to publish to MQTT
	config, ok := conf.Protocols[ProtocolTypeMQTT].(MqttProtocol)
	if !ok {
		return nil
	}

	requiresMqtt := false
	for _, d := range conf.Devices {
		for _, r := range d.Resources {
			for _, p := range r.Protocols {
				if p.Type == ProtocolTypeMQTT {
					requiresMqtt = true
					break
				}
				if requiresMqtt {
					break
				}
			}
			if requiresMqtt {
				break
			}
		}
		if requiresMqtt {
			break
		}
	}

	if !requiresMqtt {
		return nil
	}

	// Create and return publisher
	publisher := &MQTTPublisher{
		config:   &config,
		clientId: conf.Id,
		dataCh:   make(chan AgentResponse),
	}

	// Prepare MQTT connection opts
	broker := fmt.Sprintf("tcp://%s:%v", config.Host, config.Port)
	connOpts := MQTT.NewClientOptions().AddBroker(broker).SetClientId(publisher.clientId).
		SetCleanSession(true).SetOnConnectionLost(publisher.onConnectionLost)

	publisher.client = MQTT.NewClient(connOpts)
	return publisher
}

func (self *MQTTPublisher) dataInbox() chan<- AgentResponse {
	return self.dataCh
}

func (self *MQTTPublisher) start() {
	log.Println("MQTTPublisher.start()")
	// start the connection routine
	log.Printf("MQTTPublisher: Will connect to the broker tcp://%s:%v", self.config.Host, self.config.Port)
	go self.connect(0)

	qos := 1
	prefix := self.config.Prefix
	for resp := range self.dataCh {
		if !self.client.IsConnected() {
			log.Println("MQTTPublisher: got data while not connected to the broker. **discarded**")
			continue
		}
		if resp.IsError {
			log.Println("MQTTPublisher: data ERROR from agent manager:", string(resp.Payload))
			continue
		}
		topic := fmt.Sprintf("%s/%s", prefix, resp.ResourceId)
		self.client.Publish(MQTT.QoS(qos), topic, resp.Payload)
		// We dont' wait for confirmation from broker (avoid blocking here!)
		//<-r
		log.Println("MQTTPublisher: published to", topic)
	}
}

func (self *MQTTPublisher) stop() {
	log.Println("MQTTPublisher.stop()")
	if self.client != nil && self.client.IsConnected() {
		self.client.Disconnect(500)
	}
}

func (self *MQTTPublisher) connect(backOff int) {
	log.Printf("MQTTPublisher: connecting to the broker %s:%v, backOff: %v sec\n", self.config.Host, self.config.Port, backOff)
	// sleep for backOff seconds
	time.Sleep(time.Duration(backOff) * time.Second)
	_, err := self.client.Start()

	if err != nil {
		log.Printf("MQTTPublisher: failed to connect: %v\n", err.Error())
		// intial backOff 10 sec, every further retry backOff*2 unless <= 10 min
		if backOff == 0 {
			backOff = 10
		} else if backOff <= 600 {
			backOff *= 2
		}
		go self.connect(backOff)
		return
	}

	log.Printf("MQTTPublisher: connected to the broker %s:%v", self.config.Host, self.config.Port)
	return
}

func (self *MQTTPublisher) onConnectionLost(client *MQTT.MqttClient, reason error) {
	log.Println("MQTTPulbisher: lost connection to the broker: ", reason.Error())

	// Initialize a new client and reconnect
	broker := fmt.Sprintf("tcp://%s:%v", self.config.Host, self.config.Port)
	connOpts := MQTT.NewClientOptions().AddBroker(broker).SetClientId(self.clientId).
		SetCleanSession(true).SetOnConnectionLost(self.onConnectionLost)

	self.client = MQTT.NewClient(connOpts)
	go self.connect(0)
}
<|endoftext|>


<commit_before>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

func config(workingDir, user, email string) error {
	for k, v := range map[string]string{
		"user.name":  user,
		"user.email": email,
	} {
		if err := execGitCmd(workingDir, "", nil, "config", k, v); err != nil {
			return errors.Wrap(err, "setting git config")
		}
	}
	return nil
}

func clone(workingDir, keyPath, repoURL, repoBranch string) (path string, err error) {
	repoPath := filepath.Join(workingDir, "repo")
	args := []string{"clone"}
	if repoBranch != "" {
		args = append(args, "--branch", repoBranch)
	}
	args = append(args, repoURL, repoPath)
	if err := execGitCmd(workingDir, keyPath, nil, args...); err != nil {
		return "", errors.Wrap(err, "git clone")
	}
	return repoPath, nil
}

func commit(workingDir, commitMessage string) error {
	if err := execGitCmd(
		workingDir, "", nil,
		"commit",
		"--no-verify", "-a", "-m", commitMessage, ".",
	); err != nil {
		return errors.Wrap(err, "git commit")
	}
	return nil
}

// push the refs given to the upstream repo
func push(keyPath, workingDir, upstream string, refs ...string) error {
	args := append([]string{"push", upstream}, refs...)
	if err := execGitCmd(workingDir, keyPath, nil, args...); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git push %s %s", upstream, refs))
	}
	return nil
}

// pull the specific ref from upstream. Usually this would
func pull(keyPath, workingDir, upstream, ref string) error {
	if err := execGitCmd(workingDir, keyPath, nil, "pull", "--ff-only", upstream, ref); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git pull --ff-only %s %s", upstream, ref))
	}
	return nil
}

func fetch(keyPath, workingDir, upstream, refspec string) error {
	if err := execGitCmd(workingDir, keyPath, nil, "fetch", upstream, refspec); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git fetch %s %s", upstream, refspec))
	}
	return nil
}

// Get the full ref for a shorthand notes ref
func getNotesRef(workingDir, ref string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(workingDir, "", out, "notes", "--ref", ref, "get-ref"); err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func addNote(workingDir, rev, notesRef, note string) error {
	return execGitCmd(workingDir, "", nil, "notes", "--ref", notesRef, "add", "-m", note, rev)
}

func getNote(workingDir, notesRef, rev string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(workingDir, "", out, "notes", "--ref", notesRef, "show", rev); err != nil {
		return "", err
	}
	return out.String(), nil
}

// Get the commit hash for HEAD
func headRevision(path string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(path, "", out, "rev-list", "--max-count", "1", "HEAD"); err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func revlist(path, ref1, ref2 string) ([]string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(path, "", out, "rev-list", fmt.Sprintf("%s..%s", ref1, ref2)); err != nil {
		return nil, err
	}
	return strings.Split(out.String(), "\n"), nil
}

// Move the tag to the ref given and push that tag upstream
func moveTagAndPush(path, key, tag, ref, msg, upstream string) error {
	if err := execGitCmd(path, "", nil, "tag", "--force", "-a", "-m", msg, tag, ref); err != nil {
		return errors.Wrap(err, "moving tag "+tag)
	}
	if err := execGitCmd(path, key, nil, "push", "--force", upstream, "tag", tag); err != nil {
		return errors.Wrap(err, "pushing tag to origin")
	}
	return nil
}

func execGitCmd(dir, keyPath string, out io.Writer, args ...string) error {
	//	println("git", strings.Join(args, " "))
	c := exec.Command("git", args...)
	if dir != "" {
		c.Dir = dir
	}
	c.Env = env(keyPath)
	c.Stdout = ioutil.Discard
	if out != nil {
		c.Stdout = out
	}
	errOut := &bytes.Buffer{}
	c.Stderr = errOut
	err := c.Run()
	if err != nil {
		println(errOut.String())
		msg := findFatalMessage(errOut)
		if msg != "" {
			err = errors.New(msg)
		}
	}
	return err
}

func env(keyPath string) []string {
	base := `GIT_SSH_COMMAND=ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no`
	if keyPath == "" {
		return []string{base}
	}
	return []string{fmt.Sprintf("%s -i %q", base, keyPath), "GIT_TERMINAL_PROMPT=0"}
}

// check returns true if there are changes locally.
func check(workingDir, subdir string) bool {
	// `--quiet` means "exit with 1 if there are changes"
	return execGitCmd(workingDir, "", nil, "diff", "--quiet", "--", subdir) != nil
}

func narrowKeyPerms(keyPath string) error {
	return os.Chmod(keyPath, 0400)
}

func findFatalMessage(output io.Reader) string {
	sc := bufio.NewScanner(output)
	for sc.Scan() {
		if strings.HasPrefix(sc.Text(), "fatal:") {
			return sc.Text()
		}
	}
	return ""
}
<commit_msg>Fixup: don't use a . path after all<commit_after>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

func config(workingDir, user, email string) error {
	for k, v := range map[string]string{
		"user.name":  user,
		"user.email": email,
	} {
		if err := execGitCmd(workingDir, "", nil, "config", k, v); err != nil {
			return errors.Wrap(err, "setting git config")
		}
	}
	return nil
}

func clone(workingDir, keyPath, repoURL, repoBranch string) (path string, err error) {
	repoPath := filepath.Join(workingDir, "repo")
	args := []string{"clone"}
	if repoBranch != "" {
		args = append(args, "--branch", repoBranch)
	}
	args = append(args, repoURL, repoPath)
	if err := execGitCmd(workingDir, keyPath, nil, args...); err != nil {
		return "", errors.Wrap(err, "git clone")
	}
	return repoPath, nil
}

func commit(workingDir, commitMessage string) error {
	if err := execGitCmd(
		workingDir, "", nil,
		"commit",
		"--no-verify", "-a", "-m", commitMessage,
	); err != nil {
		return errors.Wrap(err, "git commit")
	}
	return nil
}

// push the refs given to the upstream repo
func push(keyPath, workingDir, upstream string, refs ...string) error {
	args := append([]string{"push", upstream}, refs...)
	if err := execGitCmd(workingDir, keyPath, nil, args...); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git push %s %s", upstream, refs))
	}
	return nil
}

// pull the specific ref from upstream. Usually this would
func pull(keyPath, workingDir, upstream, ref string) error {
	if err := execGitCmd(workingDir, keyPath, nil, "pull", "--ff-only", upstream, ref); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git pull --ff-only %s %s", upstream, ref))
	}
	return nil
}

func fetch(keyPath, workingDir, upstream, refspec string) error {
	if err := execGitCmd(workingDir, keyPath, nil, "fetch", upstream, refspec); err != nil {
		return errors.Wrap(err, fmt.Sprintf("git fetch %s %s", upstream, refspec))
	}
	return nil
}

// Get the full ref for a shorthand notes ref
func getNotesRef(workingDir, ref string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(workingDir, "", out, "notes", "--ref", ref, "get-ref"); err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func addNote(workingDir, rev, notesRef, note string) error {
	return execGitCmd(workingDir, "", nil, "notes", "--ref", notesRef, "add", "-m", note, rev)
}

func getNote(workingDir, notesRef, rev string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(workingDir, "", out, "notes", "--ref", notesRef, "show", rev); err != nil {
		return "", err
	}
	return out.String(), nil
}

// Get the commit hash for HEAD
func headRevision(path string) (string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(path, "", out, "rev-list", "--max-count", "1", "HEAD"); err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func revlist(path, ref1, ref2 string) ([]string, error) {
	out := &bytes.Buffer{}
	if err := execGitCmd(path, "", out, "rev-list", fmt.Sprintf("%s..%s", ref1, ref2)); err != nil {
		return nil, err
	}
	return strings.Split(out.String(), "\n"), nil
}

// Move the tag to the ref given and push that tag upstream
func moveTagAndPush(path, key, tag, ref, msg, upstream string) error {
	if err := execGitCmd(path, "", nil, "tag", "--force", "-a", "-m", msg, tag, ref); err != nil {
		return errors.Wrap(err, "moving tag "+tag)
	}
	if err := execGitCmd(path, key, nil, "push", "--force", upstream, "tag", tag); err != nil {
		return errors.Wrap(err, "pushing tag to origin")
	}
	return nil
}

func execGitCmd(dir, keyPath string, out io.Writer, args ...string) error {
	//	println("git", strings.Join(args, " "))
	c := exec.Command("git", args...)
	if dir != "" {
		c.Dir = dir
	}
	c.Env = env(keyPath)
	c.Stdout = ioutil.Discard
	if out != nil {
		c.Stdout = out
	}
	errOut := &bytes.Buffer{}
	c.Stderr = errOut
	err := c.Run()
	if err != nil {
		println(errOut.String())
		msg := findFatalMessage(errOut)
		if msg != "" {
			err = errors.New(msg)
		}
	}
	return err
}

func env(keyPath string) []string {
	base := `GIT_SSH_COMMAND=ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no`
	if keyPath == "" {
		return []string{base}
	}
	return []string{fmt.Sprintf("%s -i %q", base, keyPath), "GIT_TERMINAL_PROMPT=0"}
}

// check returns true if there are changes locally.
func check(workingDir, subdir string) bool {
	// `--quiet` means "exit with 1 if there are changes"
	return execGitCmd(workingDir, "", nil, "diff", "--quiet", "--", subdir) != nil
}

func narrowKeyPerms(keyPath string) error {
	return os.Chmod(keyPath, 0400)
}

func findFatalMessage(output io.Reader) string {
	sc := bufio.NewScanner(output)
	for sc.Scan() {
		if strings.HasPrefix(sc.Text(), "fatal:") {
			return sc.Text()
		}
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmcmd

import (
	"io"
	"net/http"
	"os"

	"github.com/juju/errors"
	"github.com/juju/persistent-cookiejar"
	"gopkg.in/juju/charmrepo.v2-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable/csclient"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
)

// TODO(ericsnow) Factor out code from cmd/juju/commands/common.go and
// cmd/envcmd/base.go into cmd/charmstore.go and cmd/apicontext.go. Then
// use those here instead of copy-and-pasting here.

// CharmstoreClient exposes the functionality of the charm store client.
type CharmstoreClient interface {
	io.Closer
}

///////////////////
// The charmstoreSpec code is based loosely on code in cmd/juju/commands/deploy.go.

// CharmstoreSpec provides the functionality needed to open a charm
// store client.
type CharmstoreSpec interface {
	// Connect connects to the specified charm store.
	Connect() (CharmstoreClient, error)
}

type charmstoreSpec struct {
	params charmrepo.NewCharmStoreParams
}

// newCharmstoreSpec creates a new charm store spec with default
// settings.
func newCharmstoreSpec() CharmstoreSpec {
	return &charmstoreSpec{
		params: charmrepo.NewCharmStoreParams{
			//URL:        We use the default.
			//HTTPClient: We set it later.
			VisitWebPage: httpbakery.OpenWebBrowser,
		},
	}
}

// Connect implements CharmstoreSpec.
func (cs charmstoreSpec) Connect() (CharmstoreClient, error) {
	params, apiContext, err := cs.connect()
	if err != nil {
		return nil, errors.Trace(err)
	}

	baseClient := csclient.New(csclient.Params{
		URL:          params.URL,
		HTTPClient:   params.HTTPClient,
		VisitWebPage: params.VisitWebPage,
	})

	csClient := &charmstoreClient{
		Client:     baseClient,
		apiContext: apiContext,
	}
	return csClient, nil
}

// TODO(ericsnow) Also add charmstoreSpec.Repo() -> charmrepo.Interface?

func (cs charmstoreSpec) connect() (charmrepo.NewCharmStoreParams, *apiContext, error) {
	apiContext, err := newAPIContext()
	if err != nil {
		return charmrepo.NewCharmStoreParams{}, nil, errors.Trace(err)
	}

	params := cs.params // a copy
	params.HTTPClient = apiContext.HTTPClient()
	return params, apiContext, nil
}

///////////////////
// charmstoreClient is based loosely on cmd/juju/commands/common.go.

type charmstoreClient struct {
	*csclient.Client
	*apiContext
}

// Close implements io.Closer.
func (cs *charmstoreClient) Close() error {
	return cs.apiContext.Close()
}

///////////////////
// For the most part, apiContext is copied directly from cmd/envcmd/base.go.

// newAPIContext returns a new api context, which should be closed
// when done with.
func newAPIContext() (*apiContext, error) {
	jar, err := cookiejar.New(&cookiejar.Options{
		Filename: cookieFile(),
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	client := httpbakery.NewClient()
	client.Jar = jar
	client.VisitWebPage = httpbakery.OpenWebBrowser

	return &apiContext{
		jar:    jar,
		client: client,
	}, nil
}

// apiContext is a convenience type that can be embedded wherever
// we need an API connection.
// It also stores a bakery bakery client allowing the API
// to be used using macaroons to authenticate. It stores
// obtained macaroons and discharges in a cookie jar file.
type apiContext struct {
	jar    *cookiejar.Jar
	client *httpbakery.Client
}

// Close saves the embedded cookie jar.
func (c *apiContext) Close() error {
	if err := c.jar.Save(); err != nil {
		return errors.Annotatef(err, "cannot save cookie jar")
	}
	return nil
}

// HTTPClient returns an http.Client that contains the loaded
// persistent cookie jar.
func (ctx *apiContext) HTTPClient() *http.Client {
	return ctx.client.Client
}

// cookieFile returns the path to the cookie used to store authorization
// macaroons. The returned value can be overridden by setting the
// JUJU_COOKIEFILE or GO_COOKIEFILE environment variables.
func cookieFile() string {
	if file := os.Getenv("JUJU_COOKIEFILE"); file != "" {
		return file
	}
	return cookiejar.DefaultCookieFile()
}
<commit_msg>Add a TODO.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmcmd

import (
	"io"
	"net/http"
	"os"

	"github.com/juju/errors"
	"github.com/juju/persistent-cookiejar"
	"gopkg.in/juju/charmrepo.v2-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable/csclient"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
)

// TODO(ericsnow) Factor out code from cmd/juju/commands/common.go and
// cmd/envcmd/base.go into cmd/charmstore.go and cmd/apicontext.go. Then
// use those here instead of copy-and-pasting here.

// CharmstoreClient exposes the functionality of the charm store client.
type CharmstoreClient interface {
	// TODO(ericsnow) Embed github.com/juju/juju/charmstore.Client.
	io.Closer
}

///////////////////
// The charmstoreSpec code is based loosely on code in cmd/juju/commands/deploy.go.

// CharmstoreSpec provides the functionality needed to open a charm
// store client.
type CharmstoreSpec interface {
	// Connect connects to the specified charm store.
	Connect() (CharmstoreClient, error)
}

type charmstoreSpec struct {
	params charmrepo.NewCharmStoreParams
}

// newCharmstoreSpec creates a new charm store spec with default
// settings.
func newCharmstoreSpec() CharmstoreSpec {
	return &charmstoreSpec{
		params: charmrepo.NewCharmStoreParams{
			//URL:        We use the default.
			//HTTPClient: We set it later.
			VisitWebPage: httpbakery.OpenWebBrowser,
		},
	}
}

// Connect implements CharmstoreSpec.
func (cs charmstoreSpec) Connect() (CharmstoreClient, error) {
	params, apiContext, err := cs.connect()
	if err != nil {
		return nil, errors.Trace(err)
	}

	baseClient := csclient.New(csclient.Params{
		URL:          params.URL,
		HTTPClient:   params.HTTPClient,
		VisitWebPage: params.VisitWebPage,
	})

	csClient := &charmstoreClient{
		Client:     baseClient,
		apiContext: apiContext,
	}
	return csClient, nil
}

// TODO(ericsnow) Also add charmstoreSpec.Repo() -> charmrepo.Interface?

func (cs charmstoreSpec) connect() (charmrepo.NewCharmStoreParams, *apiContext, error) {
	apiContext, err := newAPIContext()
	if err != nil {
		return charmrepo.NewCharmStoreParams{}, nil, errors.Trace(err)
	}

	params := cs.params // a copy
	params.HTTPClient = apiContext.HTTPClient()
	return params, apiContext, nil
}

///////////////////
// charmstoreClient is based loosely on cmd/juju/commands/common.go.

type charmstoreClient struct {
	*csclient.Client
	*apiContext
}

// Close implements io.Closer.
func (cs *charmstoreClient) Close() error {
	return cs.apiContext.Close()
}

///////////////////
// For the most part, apiContext is copied directly from cmd/envcmd/base.go.

// newAPIContext returns a new api context, which should be closed
// when done with.
func newAPIContext() (*apiContext, error) {
	jar, err := cookiejar.New(&cookiejar.Options{
		Filename: cookieFile(),
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	client := httpbakery.NewClient()
	client.Jar = jar
	client.VisitWebPage = httpbakery.OpenWebBrowser

	return &apiContext{
		jar:    jar,
		client: client,
	}, nil
}

// apiContext is a convenience type that can be embedded wherever
// we need an API connection.
// It also stores a bakery bakery client allowing the API
// to be used using macaroons to authenticate. It stores
// obtained macaroons and discharges in a cookie jar file.
type apiContext struct {
	jar    *cookiejar.Jar
	client *httpbakery.Client
}

// Close saves the embedded cookie jar.
func (c *apiContext) Close() error {
	if err := c.jar.Save(); err != nil {
		return errors.Annotatef(err, "cannot save cookie jar")
	}
	return nil
}

// HTTPClient returns an http.Client that contains the loaded
// persistent cookie jar.
func (ctx *apiContext) HTTPClient() *http.Client {
	return ctx.client.Client
}

// cookieFile returns the path to the cookie used to store authorization
// macaroons. The returned value can be overridden by setting the
// JUJU_COOKIEFILE or GO_COOKIEFILE environment variables.
func cookieFile() string {
	if file := os.Getenv("JUJU_COOKIEFILE"); file != "" {
		return file
	}
	return cookiejar.DefaultCookieFile()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/cmd/load-generator/wfe"
)

func usage(fs []*flag.FlagSet) {
	fmt.Fprintln(os.Stdout, "Usage:")
	for _, s := range fs {
		fmt.Fprint(os.Stderr, "\t[Section]\n")
		s.PrintDefaults()
	}
}

func main() {
	wfeArgs := flag.NewFlagSet("wfe", flag.ExitOnError)
	wfeConfigPath := wfeArgs.String("config", "", "Path to configuration file for WFE load-generator")

	ocspArgs := flag.NewFlagSet("ocsp", flag.ExitOnError)
	// ocspBase := ocspArgs.String("ocspBase", "http://localhost:4000", "The base URI of the OCSP responder")
	// ocspGetPlan := ocspArgs.String("getPlan", "", "Execution plan for OCSP GET generator,  Format is '{throughput},{duration for}:{next throughput},{duration for}' etc. E.g. '1,10m' or '2,5m:3,5m:4,5m:5,5m'")
	// ocspPostPlan := ocspArgs.String("postPlan", "", "Execution plan for OCSP GET generator,  Format is '{throughput},{duration for}:{next throughput},{duration for}' etc. E.g. '1,10m' or '2,5m:3,5m:4,5m:5,5m'")
	// ocspIssuer := ocspArgs.String("issuer", "", "Path to issuer to use to generate OCSP requests")
	// ocspResultsPath := ocspArgs.String("results", "", "Path to write results file to")
	// ocspSerialsPath := ocspArgs.String("serials", "", "Path to CSV file of serial numbers of use")

	if len(os.Args) <= 1 {
		fmt.Fprint(os.Stderr, "A sub-command is required\n\n")
		usage([]*flag.FlagSet{wfeArgs, ocspArgs})
		os.Exit(1)
	}

	switch os.Args[1] {
	case "wfe":
		err := wfeArgs.Parse(os.Args[2:])
		cmd.FailOnError(err, "Failed to parse arguments")

		configBytes, err := ioutil.ReadFile(*wfeConfigPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read wfe config file %q: %s\n", &wfeConfigPath, err)
			os.Exit(1)
		}
		var config wfe.Config
		err = json.Unmarshal(configBytes, &config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to parse wfe config file: %s\n", err)
			os.Exit(1)
		}

		s, err := wfe.New(
			config.APIBase,
			config.CertKeySize,
			config.DomainBase,
			config.RealIP,
			config.MaxRegs,
			config.Results,
			config.RegEmail,
			config.Plan.Actions,
		)
		cmd.FailOnError(err, "Failed to create WFE generator")

		if config.ExternalState != "" {
			err = s.Restore(config.ExternalState)
			cmd.FailOnError(err, "Failed to load registration snapshot")
		}

		runtime, err := time.ParseDuration(config.Plan.Runtime)
		cmd.FailOnError(err, "Failed to parse plan runtime")

		plan := wfe.Plan{
			Runtime: runtime,
			Rate:    config.Plan.Rate,
			// Delta: &wfe.RateDelta{
			// 	Inc:    5,
			// 	Period: time.Minute,
			// },
		}
		if config.Plan.RateDelta != "" {
			// do stuff
		}
		err = s.Run(config.HTTPOneAddr, config.TLSOneAddr, plan)
		cmd.FailOnError(err, "Failed to run WFE load generator")

		if config.ExternalState != "" && !config.DontSaveState {
			err = s.Snapshot(config.ExternalState)
			cmd.FailOnError(err, "Failed to save registration snapshot")
		}
	// case "ocsp":
	// 	err := ocspArgs.Parse(os.Args[2:])
	// 	cmd.FailOnError(err, "Failed to parse arguments")

	// 	getPlan, getRuntime, err := eventParser(*ocspGetPlan)
	// 	postPlan, postRuntime, err := eventParser(*ocspPostPlan)
	// 	var runtime time.Duration
	// 	if getRuntime < postRuntime {
	// 		runtime = postRuntime
	// 	} else {
	// 		runtime = getRuntime
	// 	}

	// 	cont, err := ioutil.ReadFile(*ocspSerialsPath)
	// 	cmd.FailOnError(err, "Failed to read serials file")
	// 	serials := strings.Split(string(cont), "\n")

	// 	s, err := responder.New(
	// 		*ocspBase,
	// 		getPlan,
	// 		postPlan,
	// 		*ocspIssuer,
	// 		*ocspResultsPath,
	// 		runtime,
	// 		serials,
	// 	)
	// 	cmd.FailOnError(err, "Failed to create OCSP-Responder generator")

	// 	s.Run()
	default:
		fmt.Fprintf(os.Stderr, "%s is a invalid command\n\n", os.Args[1])
		usage([]*flag.FlagSet{wfeArgs, ocspArgs})
		os.Exit(1)
	}
	fmt.Println("[+] All done, bye bye ^_^")
}
<commit_msg>Basic rate delta support<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/cmd/load-generator/wfe"
)

func usage(fs []*flag.FlagSet) {
	fmt.Fprintln(os.Stdout, "Usage:")
	for _, s := range fs {
		fmt.Fprint(os.Stderr, "\t[Section]\n")
		s.PrintDefaults()
	}
}

func main() {
	wfeArgs := flag.NewFlagSet("wfe", flag.ExitOnError)
	wfeConfigPath := wfeArgs.String("config", "", "Path to configuration file for WFE load-generator")

	ocspArgs := flag.NewFlagSet("ocsp", flag.ExitOnError)
	// ocspBase := ocspArgs.String("ocspBase", "http://localhost:4000", "The base URI of the OCSP responder")
	// ocspGetPlan := ocspArgs.String("getPlan", "", "Execution plan for OCSP GET generator,  Format is '{throughput},{duration for}:{next throughput},{duration for}' etc. E.g. '1,10m' or '2,5m:3,5m:4,5m:5,5m'")
	// ocspPostPlan := ocspArgs.String("postPlan", "", "Execution plan for OCSP GET generator,  Format is '{throughput},{duration for}:{next throughput},{duration for}' etc. E.g. '1,10m' or '2,5m:3,5m:4,5m:5,5m'")
	// ocspIssuer := ocspArgs.String("issuer", "", "Path to issuer to use to generate OCSP requests")
	// ocspResultsPath := ocspArgs.String("results", "", "Path to write results file to")
	// ocspSerialsPath := ocspArgs.String("serials", "", "Path to CSV file of serial numbers of use")

	if len(os.Args) <= 1 {
		fmt.Fprint(os.Stderr, "A sub-command is required\n\n")
		usage([]*flag.FlagSet{wfeArgs, ocspArgs})
		os.Exit(1)
	}

	switch os.Args[1] {
	case "wfe":
		err := wfeArgs.Parse(os.Args[2:])
		cmd.FailOnError(err, "Failed to parse arguments")

		configBytes, err := ioutil.ReadFile(*wfeConfigPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read wfe config file %q: %s\n", &wfeConfigPath, err)
			os.Exit(1)
		}
		var config wfe.Config
		err = json.Unmarshal(configBytes, &config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to parse wfe config file: %s\n", err)
			os.Exit(1)
		}

		s, err := wfe.New(
			config.APIBase,
			config.CertKeySize,
			config.DomainBase,
			config.RealIP,
			config.MaxRegs,
			config.Results,
			config.RegEmail,
			config.Plan.Actions,
		)
		cmd.FailOnError(err, "Failed to create WFE generator")

		if config.ExternalState != "" {
			err = s.Restore(config.ExternalState)
			cmd.FailOnError(err, "Failed to load registration snapshot")
		}

		runtime, err := time.ParseDuration(config.Plan.Runtime)
		cmd.FailOnError(err, "Failed to parse plan runtime")

		var delta *wfe.RateDelta
		if config.Plan.RateDelta != "" {
			parts := strings.Split(config.Plan.RateDelta, "/")
			if len(parts) != 2 {
				fmt.Fprintf(os.Stderr, "RateDelta is malformed")
				os.Exit(1)
			}
			rate, err := strconv.Atoi(parts[0])
			cmd.FailOnError(err, "Failed to parse increase portion of RateDelta")
			period, err := time.ParseDuration(parts[1])
			cmd.FailOnError(err, "Failed to parse period portion of RateDelta")
			delta = &wfe.RateDelta{Inc: int64(rate), Period: period}
		}

		err = s.Run(config.HTTPOneAddr, config.TLSOneAddr, wfe.Plan{
			Runtime: runtime,
			Rate:    config.Plan.Rate,
			Delta:   delta,
		})
		cmd.FailOnError(err, "Failed to run WFE load generator")

		if config.ExternalState != "" && !config.DontSaveState {
			err = s.Snapshot(config.ExternalState)
			cmd.FailOnError(err, "Failed to save registration snapshot")
		}
	// case "ocsp":
	// 	err := ocspArgs.Parse(os.Args[2:])
	// 	cmd.FailOnError(err, "Failed to parse arguments")

	// 	getPlan, getRuntime, err := eventParser(*ocspGetPlan)
	// 	postPlan, postRuntime, err := eventParser(*ocspPostPlan)
	// 	var runtime time.Duration
	// 	if getRuntime < postRuntime {
	// 		runtime = postRuntime
	// 	} else {
	// 		runtime = getRuntime
	// 	}

	// 	cont, err := ioutil.ReadFile(*ocspSerialsPath)
	// 	cmd.FailOnError(err, "Failed to read serials file")
	// 	serials := strings.Split(string(cont), "\n")

	// 	s, err := responder.New(
	// 		*ocspBase,
	// 		getPlan,
	// 		postPlan,
	// 		*ocspIssuer,
	// 		*ocspResultsPath,
	// 		runtime,
	// 		serials,
	// 	)
	// 	cmd.FailOnError(err, "Failed to create OCSP-Responder generator")

	// 	s.Run()
	default:
		fmt.Fprintf(os.Stderr, "%s is a invalid command\n\n", os.Args[1])
		usage([]*flag.FlagSet{wfeArgs, ocspArgs})
		os.Exit(1)
	}
	fmt.Println("[+] All done, bye bye ^_^")
}
<|endoftext|>


<commit_before>package compose

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/mutagen-io/mutagen/cmd"
)

func runPassthrough(command *cobra.Command, arguments []string) {
	cmd.Warning("The \"run\" command isn't fully supported by Mutagen")
	fmt.Println()
	passthrough(command, arguments)
}

var runCommand = &cobra.Command{
	Use:                "run",
	Run:                runPassthrough,
	SilenceUsage:       true,
	DisableFlagParsing: true,
}

var runConfiguration struct {
	// help indicates the presence of the -h/--help flag.
	help bool
	// detach indicates the presence of the -d/--detach flag.
	detach bool
	// name stores the value of the --name flag.
	name string
	// entrypoint stores the value of the --entrypoint flag.
	entrypoint string
	// e stores the value(s) of the -e flag(s).
	e []string
	// label stores the value(s) of the -l/--label flag(s).
	label []string
	// user stores the value of the -u/--user flag.
	user string
	// noDeps indicates the presence of the --no-deps flag.
	noDeps bool
	// rm indicates the presence of the --rm flag.
	rm bool
	// publish stores the value(s) of the -p/--publish flag(s).
	publish []string
	// servicePorts indicates the presence of the --service-ports flag.
	servicePorts bool
	// useAliases indicates the presence of the --use-aliases flag.
	useAliases bool
	// volume stores the value(s) of the -v/--volume flag(s).
	volume []string
	// T indicates the presence of the -T flag.
	t bool
	// workdir stores the value of the -w/--workdir flag.
	workdir string
}

func init() {
	// We don't set an explicit help function since we disable flag parsing for
	// this command and simply pass arguments directly through to the underlying
	// command. We still explicitly register a -h/--help flag below for shell
	// completion support.

	// Grab a handle for the command line flags.
	flags := runCommand.Flags()

	// Wire up flags. We don't bother specifying usage information since we'll
	// shell out to Docker Compose if we need to display help information. In
	// the case of this command, we also disable flag parsing and shell out
	// directly, so we only register these flags to support shell completion.
	flags.BoolVarP(&runConfiguration.help, "help", "h", false, "")
	flags.StringVar(&runConfiguration.name, "name", "", "")
	flags.StringVar(&runConfiguration.entrypoint, "entrypoint", "", "")
	// TODO: Figure out how to do a short-only flag for -e. See the comment on
	// the -T flag for exec to understand why this isn't currently possible.
	flags.StringSliceVarP(&runConfiguration.label, "label", "l", nil, "")
	flags.StringVarP(&runConfiguration.user, "user", "u", "", "")
	flags.BoolVar(&runConfiguration.noDeps, "no-deps", false, "")
	flags.BoolVar(&runConfiguration.rm, "rm", false, "")
	flags.StringSliceVarP(&runConfiguration.publish, "publish", "p", nil, "")
	flags.BoolVar(&runConfiguration.servicePorts, "service-ports", false, "")
	flags.BoolVar(&runConfiguration.useAliases, "use-aliases", false, "")
	flags.StringSliceVarP(&runConfiguration.volume, "volume", "v", nil, "")
	// TODO: Figure out how to do a short-only flag for -T. See the comment on
	// the -T flag for exec to understand why this isn't currently possible.
	flags.StringVarP(&runConfiguration.workdir, "workdir", "w", "", "")
}
<commit_msg>Added additional notice to compose run warning.<commit_after>package compose

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"

	"github.com/mutagen-io/mutagen/cmd"
)

func runPassthrough(command *cobra.Command, arguments []string) {
	cmd.Warning("The \"run\" command isn't fully supported by Mutagen.")
	fmt.Fprintln(os.Stderr, "Use \"docker-compose run\" directly to suppress this message.")
	fmt.Println()
	passthrough(command, arguments)
}

var runCommand = &cobra.Command{
	Use:                "run",
	Run:                runPassthrough,
	SilenceUsage:       true,
	DisableFlagParsing: true,
}

var runConfiguration struct {
	// help indicates the presence of the -h/--help flag.
	help bool
	// detach indicates the presence of the -d/--detach flag.
	detach bool
	// name stores the value of the --name flag.
	name string
	// entrypoint stores the value of the --entrypoint flag.
	entrypoint string
	// e stores the value(s) of the -e flag(s).
	e []string
	// label stores the value(s) of the -l/--label flag(s).
	label []string
	// user stores the value of the -u/--user flag.
	user string
	// noDeps indicates the presence of the --no-deps flag.
	noDeps bool
	// rm indicates the presence of the --rm flag.
	rm bool
	// publish stores the value(s) of the -p/--publish flag(s).
	publish []string
	// servicePorts indicates the presence of the --service-ports flag.
	servicePorts bool
	// useAliases indicates the presence of the --use-aliases flag.
	useAliases bool
	// volume stores the value(s) of the -v/--volume flag(s).
	volume []string
	// T indicates the presence of the -T flag.
	t bool
	// workdir stores the value of the -w/--workdir flag.
	workdir string
}

func init() {
	// We don't set an explicit help function since we disable flag parsing for
	// this command and simply pass arguments directly through to the underlying
	// command. We still explicitly register a -h/--help flag below for shell
	// completion support.

	// Grab a handle for the command line flags.
	flags := runCommand.Flags()

	// Wire up flags. We don't bother specifying usage information since we'll
	// shell out to Docker Compose if we need to display help information. In
	// the case of this command, we also disable flag parsing and shell out
	// directly, so we only register these flags to support shell completion.
	flags.BoolVarP(&runConfiguration.help, "help", "h", false, "")
	flags.StringVar(&runConfiguration.name, "name", "", "")
	flags.StringVar(&runConfiguration.entrypoint, "entrypoint", "", "")
	// TODO: Figure out how to do a short-only flag for -e. See the comment on
	// the -T flag for exec to understand why this isn't currently possible.
	flags.StringSliceVarP(&runConfiguration.label, "label", "l", nil, "")
	flags.StringVarP(&runConfiguration.user, "user", "u", "", "")
	flags.BoolVar(&runConfiguration.noDeps, "no-deps", false, "")
	flags.BoolVar(&runConfiguration.rm, "rm", false, "")
	flags.StringSliceVarP(&runConfiguration.publish, "publish", "p", nil, "")
	flags.BoolVar(&runConfiguration.servicePorts, "service-ports", false, "")
	flags.BoolVar(&runConfiguration.useAliases, "use-aliases", false, "")
	flags.StringSliceVarP(&runConfiguration.volume, "volume", "v", nil, "")
	// TODO: Figure out how to do a short-only flag for -T. See the comment on
	// the -T flag for exec to understand why this isn't currently possible.
	flags.StringVarP(&runConfiguration.workdir, "workdir", "w", "", "")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type PackageIndex struct {
	URI string
}

type Requirement struct {
	Name string
}

var allPkgRegexp = regexp.MustCompile(`<a href='([A-Za-z0-9\._\-]+)'>([A-Za-z0-9\._\-]+)</a><br/>`)
var pkgFilesRegexp = regexp.MustCompile(`<a href="([/A-Za-z0-9\._\-]+)#md5=[0-9a-z]+"[^>]*>([A-Za-z0-9\._\-]+)</a><br/>`)
var tarRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.(?:tar\.(?:gz|bz2)|tgz)`)
var zipRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.zip`)
var eggRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.egg`)
var requirementRegexp = regexp.MustCompile(`(?P<package>[A-Za-z0-9\._\-]+)(?:\[([A-Za-z0-9\._\-]+)\])?\s*(?:(?P<constraint>==|>=|>|<|<=)\s*(?P<version>[A-Za-z0-9\._\-]+)(?:\s*,\s*[<>=!]+\s*[a-z0-9\.]+)?)?`)
var reqHeaderRegexp = regexp.MustCompile(`\[[A-Za-z0-9\._\-]+\]`)

func (p *PackageIndex) AllPackages() ([]string, error) {
	pkgs := make([]string, 0)

	resp, err := http.Get(fmt.Sprintf("%s/simple", p.URI))
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	matches := allPkgRegexp.FindAllStringSubmatch(string(body), -1)
	for _, match := range matches {
		if len(match) != 3 {
			return nil, fmt.Errorf("Unexpected number of submatches: %d, %v", len(match), match)
		} else if match[1] != match[2] {
			return nil, fmt.Errorf("Names do not match %s != %s", match[1], match[2])
		} else {
			pkgs = append(pkgs, match[1])
		}
	}

	return pkgs, nil
}

func (p *PackageIndex) PackageRequirements(pkg string) ([]*Requirement, error) {
	files, err := p.pkgFiles(pkg)
	if err != nil {
		return nil, err
	} else if len(files) == 0 {
		// os.Stderr.WriteString(fmt.Sprintf("[no-files] no files found for pkg %s\n", pkg))
		return nil, nil
	}

	if path := lastTar(files); path != "" {
		return p.fetchRequiresTar(path)
	} else if path := lastEgg(files); path != "" {
		return p.fetchRequiresZip(path, true)
	} else if path := lastZip(files); path != "" {
		return p.fetchRequiresZip(path, false)
	} else {
		os.Stderr.WriteString(fmt.Sprintf("[tar/zip] no tar or zip found in %+v for pkg %s\n", files, pkg))
		return nil, nil
	}
}

func (p *PackageIndex) pkgFiles(pkg string) ([]string, error) {
	files := make([]string, 0)

	uriPath := fmt.Sprintf("/simple/%s", pkg)
	uri := fmt.Sprintf("%s%s", p.URI, uriPath)
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	matches := pkgFilesRegexp.FindAllStringSubmatch(string(body), -1)
	for _, match := range matches {
		if len(match) != 3 {
			return nil, fmt.Errorf("Unexpected number of submatches: %d, %v", len(match), match)
		} else {
			files = append(files, filepath.Clean(filepath.Join(uriPath, match[1])))
		}
	}

	return files, nil
}

func (p *PackageIndex) fetchRequiresZip(path string, isEgg bool) ([]*Requirement, error) {
	f, err := ioutil.TempFile("", "pypigraph_zip")
	if err != nil {
		return nil, err
	}
	defer os.Remove(f.Name())
	if err := f.Close(); err != nil {
		return nil, err
	}

	uri := fmt.Sprintf("%s%s", p.URI, path)
	wget := exec.Command("wget", uri, "-O", f.Name())
	var eggInfoFilePattern string
	if isEgg {
		eggInfoFilePattern = "EGG-INFO/requires.txt"
	} else {
		eggInfoFilePattern = "**/*.egg-info/requires.txt"
	}
	unzip := exec.Command("unzip", "-cq", f.Name(), eggInfoFilePattern)

	err = wget.Run()
	if err != nil {
		return nil, fmt.Errorf("Error running wget: %s", err)
	}

	unzipOutput, err := unzip.Output()
	if err != nil {
		if strings.Contains(err.Error(), "exit status 11") {
			// os.Stderr.WriteString(fmt.Sprintf("[requires.txt] no requires.txt found in %s\n", uri))
			return nil, nil
		} else {
			return nil, fmt.Errorf("Error running unzip on file %s: %s", f.Name(), err)
		}
	}

	rawReqs := strings.TrimSpace(string(unzipOutput))
	return parseRequirements(rawReqs)
}

func (p *PackageIndex) fetchRequiresTar(path string) ([]*Requirement, error) {
	uri := fmt.Sprintf("%s%s", p.URI, path)
	curl := exec.Command("curl", uri)
	tar := exec.Command("tar", "-xvO", "--include", "**/*.egg-info/requires.txt")

	curlOut, err := curl.StdoutPipe()
	if err != nil {
		return nil, err
	}
	tarIn, err := tar.StdinPipe()
	if err != nil {
		return nil, err
	}
	tarOut, err := tar.StdoutPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(tarIn, curlOut)
		tarIn.Close()
	}()

	curl.Start()
	tar.Start()

	tarOutput, err := ioutil.ReadAll(tarOut)
	if err != nil {
		return nil, err
	}

	curl.Wait()
	tar.Wait()

	rawReqs := strings.TrimSpace(string(tarOutput))
	if rawReqs == "" {
		// os.Stderr.WriteString(fmt.Sprintf("[requires.txt] no requires.txt found in %s\n", uri))
		return nil, nil
	}

	return parseRequirements(rawReqs)
}

func lastTar(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if tarRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func lastEgg(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if eggRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func lastZip(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if zipRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func parseRequirements(rawReqs string) ([]*Requirement, error) {
	rawReqs = strings.TrimSpace(rawReqs)

	reqStrs := strings.Split(rawReqs, "\n")
	reqs := make([]*Requirement, 0)
	for _, reqStr := range reqStrs {
		if reqStr == "" {
			continue
		}

		if req, err := parseRequirement(reqStr); err == nil {
			reqs = append(reqs, req)
		} else if reqHeaderRegexp.MatchString(reqStr) {
			// do nothing
		} else {
			os.Stderr.WriteString(fmt.Sprintf("[req] Could not parse requirement: %s\n", err))
		}
	}
	return reqs, nil
}

func parseRequirement(reqStr string) (*Requirement, error) {
	reqStr = strings.TrimSpace(reqStr)
	match := requirementRegexp.FindStringSubmatch(reqStr)
	if len(match) != 5 {
		return nil, fmt.Errorf("Expected match of length 5, but got %+v from '%s'", match, reqStr)
	} else if match[0] != reqStr {
		return nil, fmt.Errorf("Unable to parse requirement from string: '%s'", reqStr)
	}
	return &Requirement{Name: match[1]}, nil
}

func normalizedPkgName(pkg string) string {
	return strings.ToLower(pkg)
}

func main() {
	p := &PackageIndex{URI: "https://pypi.python.org"}
	pkgs, err := p.AllPackages()
	if err != nil {
		os.Stderr.WriteString(fmt.Sprintf("[FATAL] %s\n", err))
		os.Exit(1)
	}

	for _, pkg := range pkgs {
		reqs, err := p.PackageRequirements(pkg)
		if err != nil {
			os.Stderr.WriteString(fmt.Sprintf("[ERROR] unable to parse pkg %s due to error: %s\n", pkg, err))
		} else {
			for _, req := range reqs {
				fmt.Printf("%s:%s\n", normalizedPkgName(pkg), normalizedPkgName(req.Name))
			}
		}
	}
}
<commit_msg>show progress<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type PackageIndex struct {
	URI string
}

type Requirement struct {
	Name string
}

var allPkgRegexp = regexp.MustCompile(`<a href='([A-Za-z0-9\._\-]+)'>([A-Za-z0-9\._\-]+)</a><br/>`)
var pkgFilesRegexp = regexp.MustCompile(`<a href="([/A-Za-z0-9\._\-]+)#md5=[0-9a-z]+"[^>]*>([A-Za-z0-9\._\-]+)</a><br/>`)
var tarRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.(?:tar\.(?:gz|bz2)|tgz)`)
var zipRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.zip`)
var eggRegexp = regexp.MustCompile(`[/A-Za-z0-9\._\-]+\.egg`)
var requirementRegexp = regexp.MustCompile(`(?P<package>[A-Za-z0-9\._\-]+)(?:\[([A-Za-z0-9\._\-]+)\])?\s*(?:(?P<constraint>==|>=|>|<|<=)\s*(?P<version>[A-Za-z0-9\._\-]+)(?:\s*,\s*[<>=!]+\s*[a-z0-9\.]+)?)?`)
var reqHeaderRegexp = regexp.MustCompile(`\[[A-Za-z0-9\._\-]+\]`)

func (p *PackageIndex) AllPackages() ([]string, error) {
	pkgs := make([]string, 0)

	resp, err := http.Get(fmt.Sprintf("%s/simple", p.URI))
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	matches := allPkgRegexp.FindAllStringSubmatch(string(body), -1)
	for _, match := range matches {
		if len(match) != 3 {
			return nil, fmt.Errorf("Unexpected number of submatches: %d, %v", len(match), match)
		} else if match[1] != match[2] {
			return nil, fmt.Errorf("Names do not match %s != %s", match[1], match[2])
		} else {
			pkgs = append(pkgs, match[1])
		}
	}

	return pkgs, nil
}

func (p *PackageIndex) PackageRequirements(pkg string) ([]*Requirement, error) {
	files, err := p.pkgFiles(pkg)
	if err != nil {
		return nil, err
	} else if len(files) == 0 {
		// os.Stderr.WriteString(fmt.Sprintf("[no-files] no files found for pkg %s\n", pkg))
		return nil, nil
	}

	if path := lastTar(files); path != "" {
		return p.fetchRequiresTar(path)
	} else if path := lastEgg(files); path != "" {
		return p.fetchRequiresZip(path, true)
	} else if path := lastZip(files); path != "" {
		return p.fetchRequiresZip(path, false)
	} else {
		os.Stderr.WriteString(fmt.Sprintf("[tar/zip] no tar or zip found in %+v for pkg %s\n", files, pkg))
		return nil, nil
	}
}

func (p *PackageIndex) pkgFiles(pkg string) ([]string, error) {
	files := make([]string, 0)

	uriPath := fmt.Sprintf("/simple/%s", pkg)
	uri := fmt.Sprintf("%s%s", p.URI, uriPath)
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	matches := pkgFilesRegexp.FindAllStringSubmatch(string(body), -1)
	for _, match := range matches {
		if len(match) != 3 {
			return nil, fmt.Errorf("Unexpected number of submatches: %d, %v", len(match), match)
		} else {
			files = append(files, filepath.Clean(filepath.Join(uriPath, match[1])))
		}
	}

	return files, nil
}

func (p *PackageIndex) fetchRequiresZip(path string, isEgg bool) ([]*Requirement, error) {
	f, err := ioutil.TempFile("", "pypigraph_zip")
	if err != nil {
		return nil, err
	}
	defer os.Remove(f.Name())
	if err := f.Close(); err != nil {
		return nil, err
	}

	uri := fmt.Sprintf("%s%s", p.URI, path)
	wget := exec.Command("wget", uri, "-O", f.Name())
	var eggInfoFilePattern string
	if isEgg {
		eggInfoFilePattern = "EGG-INFO/requires.txt"
	} else {
		eggInfoFilePattern = "**/*.egg-info/requires.txt"
	}
	unzip := exec.Command("unzip", "-cq", f.Name(), eggInfoFilePattern)

	err = wget.Run()
	if err != nil {
		return nil, fmt.Errorf("Error running wget: %s", err)
	}

	unzipOutput, err := unzip.Output()
	if err != nil {
		if strings.Contains(err.Error(), "exit status 11") {
			// os.Stderr.WriteString(fmt.Sprintf("[requires.txt] no requires.txt found in %s\n", uri))
			return nil, nil
		} else {
			return nil, fmt.Errorf("Error running unzip on file %s: %s", f.Name(), err)
		}
	}

	rawReqs := strings.TrimSpace(string(unzipOutput))
	return parseRequirements(rawReqs)
}

func (p *PackageIndex) fetchRequiresTar(path string) ([]*Requirement, error) {
	uri := fmt.Sprintf("%s%s", p.URI, path)
	curl := exec.Command("curl", uri)
	tar := exec.Command("tar", "-xvO", "--include", "**/*.egg-info/requires.txt")

	curlOut, err := curl.StdoutPipe()
	if err != nil {
		return nil, err
	}
	tarIn, err := tar.StdinPipe()
	if err != nil {
		return nil, err
	}
	tarOut, err := tar.StdoutPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(tarIn, curlOut)
		tarIn.Close()
	}()

	curl.Start()
	tar.Start()

	tarOutput, err := ioutil.ReadAll(tarOut)
	if err != nil {
		return nil, err
	}

	curl.Wait()
	tar.Wait()

	rawReqs := strings.TrimSpace(string(tarOutput))
	if rawReqs == "" {
		// os.Stderr.WriteString(fmt.Sprintf("[requires.txt] no requires.txt found in %s\n", uri))
		return nil, nil
	}

	return parseRequirements(rawReqs)
}

func lastTar(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if tarRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func lastEgg(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if eggRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func lastZip(files []string) string {
	for f := len(files) - 1; f >= 0; f-- {
		if zipRegexp.MatchString(files[f]) {
			return files[f]
		}
	}
	return ""
}

func parseRequirements(rawReqs string) ([]*Requirement, error) {
	rawReqs = strings.TrimSpace(rawReqs)

	reqStrs := strings.Split(rawReqs, "\n")
	reqs := make([]*Requirement, 0)
	for _, reqStr := range reqStrs {
		if reqStr == "" {
			continue
		}

		if req, err := parseRequirement(reqStr); err == nil {
			reqs = append(reqs, req)
		} else if reqHeaderRegexp.MatchString(reqStr) {
			// do nothing
		} else {
			os.Stderr.WriteString(fmt.Sprintf("[req] Could not parse requirement: %s\n", err))
		}
	}
	return reqs, nil
}

func parseRequirement(reqStr string) (*Requirement, error) {
	reqStr = strings.TrimSpace(reqStr)
	match := requirementRegexp.FindStringSubmatch(reqStr)
	if len(match) != 5 {
		return nil, fmt.Errorf("Expected match of length 5, but got %+v from '%s'", match, reqStr)
	} else if match[0] != reqStr {
		return nil, fmt.Errorf("Unable to parse requirement from string: '%s'", reqStr)
	}
	return &Requirement{Name: match[1]}, nil
}

func normalizedPkgName(pkg string) string {
	return strings.ToLower(pkg)
}

func main() {
	pkgIndex := &PackageIndex{URI: "https://pypi.python.org"}
	pkgs, err := pkgIndex.AllPackages()
	if err != nil {
		os.Stderr.WriteString(fmt.Sprintf("[FATAL] %s\n", err))
		os.Exit(1)
	}

	for p, pkg := range pkgs {
		if p%50 == 0 { // progress
			log.Printf("[status] %d / %d\n", p, len(pkgs))
		}

		reqs, err := pkgIndex.PackageRequirements(pkg)
		if err != nil {
			os.Stderr.WriteString(fmt.Sprintf("[ERROR] unable to parse pkg %s due to error: %s\n", pkg, err))
		} else {
			for _, req := range reqs {
				fmt.Printf("%s:%s\n", normalizedPkgName(pkg), normalizedPkgName(req.Name))
			}
		}
	}
}
<|endoftext|>