<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
	mlog "github.com/yp-engineering/mesos-runonce/mesosutil"
	"golang.org/x/net/context"
)

var eventCh = make(chan *mesos.TaskStatus)
var _frameworkId string
var exitStatus = 0
var config *Config

type MesosRunonceScheduler struct {
	executor      *mesos.ExecutorInfo
	tasksLaunched int
	tasksFinished int
	totalTasks    int
}

// ----------------------- Mesos interface ------------------------- //

func (sched *MesosRunonceScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.V(1).Infoln("Framework Registered with Master ", masterInfo)
	_frameworkId = frameworkId.GetValue()
	fmt.Println("Registered with master and given framework ID:", _frameworkId)
}
func (sched *MesosRunonceScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.V(1).Infoln("Framework Re-Registered with Master ", masterInfo)
}
func (sched *MesosRunonceScheduler) Disconnected(sched.SchedulerDriver) {
	log.Exitf("disconnected from master, aborting")
}
func (sched *MesosRunonceScheduler) OfferRescinded(_ sched.SchedulerDriver, oid *mesos.OfferID) {
	log.Exitf("offer rescinded: %v", oid)
}
func (sched *MesosRunonceScheduler) FrameworkMessage(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, msg string) {
	log.Errorf("framework message from executor %q slave %q: %q", eid, sid, msg)
}
func (sched *MesosRunonceScheduler) SlaveLost(_ sched.SchedulerDriver, sid *mesos.SlaveID) {
	log.Exitf("slave lost: %v", sid)
}
func (sched *MesosRunonceScheduler) ExecutorLost(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, code int) {
	log.Errorf("executor %q lost on slave %q code %d", eid, sid, code)
}
func (sched *MesosRunonceScheduler) Error(_ sched.SchedulerDriver, err string) {
	log.Exitf("Scheduler received error: %v", err)
}

func (sched *MesosRunonceScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	if sched.tasksLaunched >= sched.totalTasks {
		log.V(1).Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.V(1).Infoln("Received Offer <", offer.Id.GetValue(), "> on host ", *offer.Hostname, "with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		dCpus := config.Task.Docker.Cpus
		dMem := config.Task.Docker.Mem

		var tasks []*mesos.TaskInfo
		for sched.tasksLaunched < sched.totalTasks &&
			dCpus <= remainingCpus &&
			dMem <= remainingMems {

			sched.tasksLaunched++

			tID := strconv.Itoa(sched.tasksLaunched)
			if config.Task.Id != "" {
				tID = config.Task.Id
			}

			tName := "mesos-runonce-" + tID
			if config.Task.Name != "" {
				tName = config.Task.Name
			}

			taskId := &mesos.TaskID{
				Value: proto.String(tID),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task := &mesos.TaskInfo{
				Name:    proto.String(tName),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", dCpus),
					util.NewScalarResource("mem", dMem),
				},
				Command: &mesos.CommandInfo{
					Shell: proto.Bool(false),
				},
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image:          proto.String(config.Task.Docker.Image),
						ForcePullImage: proto.Bool(config.Task.Docker.ForcePullImage),
					},
				},
			}
			log.V(1).Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			if len(config.Task.Docker.Env) != 0 || config.Task.Docker.EnvString != "" {
				task.Command.Environment = config.EnvVars()
			}

			// Allow arbitrary commands, else just use whatever the image defines in CMD
			if config.Task.Docker.Cmd != "" {
				task.Command.Value = proto.String(config.Task.Docker.Cmd)
				task.Command.Shell = proto.Bool(true)
			}

			tasks = append(tasks, task)
			remainingCpus -= dCpus
			remainingMems -= dMem
		}
		log.V(1).Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}

func (sched *MesosRunonceScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.V(1).Infoln("Status update: task", status.TaskId.GetValue(), " is in state ", status.State.Enum().String())
	eventCh <- status

	if status.GetState() == mesos.TaskState_TASK_FINISHED {
		sched.tasksFinished++
	}

	if sched.tasksFinished >= sched.totalTasks {
		log.V(1).Infoln("Total tasks completed, stopping framework.")
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED ||
		status.GetState() == mesos.TaskState_TASK_FAILED ||
		status.GetState() == mesos.TaskState_TASK_ERROR {
		exitStatus = 1
		log.Warningf("mesos TaskStatus: %v", status)
		driver.Abort()
		log.Errorln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message.", status.GetMessage(),
		)
	}
}

// ----------------------- Helper methods ------------------------- //

func printLogs() {
	timer := time.Tick(500 * time.Millisecond)
	var (
		readableStatus *mesos.TaskStatus
		finished       bool
		oout, oerr     int
	)
	for {
		select {
		case status := <-eventCh:
			switch status.GetState() {
			case mesos.TaskState_TASK_RUNNING,
				mesos.TaskState_TASK_STARTING:
				readableStatus = status
			case mesos.TaskState_TASK_FAILED,
				mesos.TaskState_TASK_KILLED:
				readableStatus = status
				finished = true
			case mesos.TaskState_TASK_FINISHED:
				finished = true
			}
		case <-timer:
			if readableStatus != nil {
				if finished {
					time.Sleep(3 * time.Second)
				}
				x := printLog(readableStatus, oout, os.Stdout)
				y := printLog(readableStatus, oerr, os.Stderr)
				if finished && x == 0 && y == 0 {
					return
				}
				oout += x
				oerr += y
			}
		}
	}
}

func printLog(status *mesos.TaskStatus, offset int, w io.Writer) int {
	file := "stdout"
	if w == os.Stderr {
		file = "stderr"
	}
	data, err := mlog.FetchLogs(status, offset, file, _frameworkId)
	if err != nil {
		log.V(1).Infof("fetch logs err: %s\n", err)
		return 0
	}
	if len(data) > 0 {
		fmt.Fprint(w, string(data))
	}
	return len(data)
}

func prepareExecutorInfo() *mesos.ExecutorInfo {
	// Create mesos scheduler driver.
	containerType := mesos.ContainerInfo_DOCKER
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("default"),
		Name:       proto.String("mesos-runonce-executor"),
		Source:     proto.String("mesos-runonce-executor"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Exit(err)
	}
	if len(addr) < 1 {
		log.Exitf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

func newMesosRunonceScheduler(exec *mesos.ExecutorInfo) *MesosRunonceScheduler {
	return &MesosRunonceScheduler{
		executor:      exec,
		tasksLaunched: 0,
		tasksFinished: 0,
		totalTasks:    config.Runonce.TaskCount,
	}
}

func fwinfo() *mesos.FrameworkInfo {
	return &mesos.FrameworkInfo{
		User: proto.String(config.Runonce.MesosRunasUser),
		Name: proto.String("mesos-runonce"),
	}
}

func cred(fwinfo *mesos.FrameworkInfo) *mesos.Credential {
	cred := (*mesos.Credential)(nil)
	mesosAuthPrincipal := config.Runonce.AuthPrincipal
	if mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(mesosAuthPrincipal)
		cred = &mesos.Credential{
			Principal: proto.String(mesosAuthPrincipal),
		}
		mesosAuthSecretFile := config.Runonce.AuthSecretFile
		if mesosAuthSecretFile != "" {
			_, err := os.Stat(mesosAuthSecretFile)
			if err != nil {
				log.Exit("missing secret file: ", err.Error())
			}
			secret, err := ioutil.ReadFile(mesosAuthSecretFile)
			if err != nil {
				log.Exit("failed to read secret file: ", err.Error())
			}
			cred.Secret = proto.String(strings.TrimSuffix(string(secret), "\n"))
		}
	}
	return cred
}

func driverConfig(fwinfo *mesos.FrameworkInfo) sched.DriverConfig {
	// build command executor
	exec := prepareExecutorInfo()

	publishedAddress := parseIP(config.Runonce.Address)
	dConfig := sched.DriverConfig{
		Scheduler:        newMesosRunonceScheduler(exec),
		Framework:        fwinfo,
		Master:           config.Runonce.Master,
		Credential:       cred(fwinfo),
		PublishedAddress: publishedAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, config.Runonce.AuthProvider)
			ctx = sasl.WithBindingAddress(ctx, publishedAddress)
			return ctx
		},
	}

	// Allow listening port to be configurable so we can run this inside of
	// mesos if desired.
	// NOTE only affects main PID, meaning the authentication step uses
	// another PID which picks a random (32K range) port :(. Opened
	// https://github.com/mesos/mesos-go/issues/229 to discuss.
	if config.Runonce.BindingPort != 0 {
		dConfig.BindingPort = uint16(config.Runonce.BindingPort)
	}
	return dConfig
}

// ----------------------- func main() ------------------------- //

func main() {
	log.V(1).Infoln("Initializing the Example Scheduler...")
	config = loadConfig()

	// the framework
	fwinfo := fwinfo()

	driver, err := sched.NewMesosSchedulerDriver(driverConfig(fwinfo))
	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	// Don't block on this call because we need printLogs to be the process blocker to ensure we retrieve all the logs.
	go func() {
		if stat, err := driver.Run(); err != nil {
			log.V(1).Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		}
	}()

	printLogs()
	log.V(1).Infof("framework terminating")
	os.Exit(exitStatus)
}
<commit_msg>Stop trying to fetch logs if we get a task lost.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
	mlog "github.com/yp-engineering/mesos-runonce/mesosutil"
	"golang.org/x/net/context"
)

var eventCh = make(chan *mesos.TaskStatus)
var _frameworkId string
var exitStatus = 0
var config *Config

type MesosRunonceScheduler struct {
	executor      *mesos.ExecutorInfo
	tasksLaunched int
	tasksFinished int
	totalTasks    int
}

// ----------------------- Mesos interface ------------------------- //

func (sched *MesosRunonceScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.V(1).Infoln("Framework Registered with Master ", masterInfo)
	_frameworkId = frameworkId.GetValue()
	fmt.Println("Registered with master and given framework ID:", _frameworkId)
}
func (sched *MesosRunonceScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.V(1).Infoln("Framework Re-Registered with Master ", masterInfo)
}
func (sched *MesosRunonceScheduler) Disconnected(sched.SchedulerDriver) {
	log.Exitf("disconnected from master, aborting")
}
func (sched *MesosRunonceScheduler) OfferRescinded(_ sched.SchedulerDriver, oid *mesos.OfferID) {
	log.Exitf("offer rescinded: %v", oid)
}
func (sched *MesosRunonceScheduler) FrameworkMessage(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, msg string) {
	log.Errorf("framework message from executor %q slave %q: %q", eid, sid, msg)
}
func (sched *MesosRunonceScheduler) SlaveLost(_ sched.SchedulerDriver, sid *mesos.SlaveID) {
	log.Exitf("slave lost: %v", sid)
}
func (sched *MesosRunonceScheduler) ExecutorLost(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, code int) {
	log.Errorf("executor %q lost on slave %q code %d", eid, sid, code)
}
func (sched *MesosRunonceScheduler) Error(_ sched.SchedulerDriver, err string) {
	log.Exitf("Scheduler received error: %v", err)
}

func (sched *MesosRunonceScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	if sched.tasksLaunched >= sched.totalTasks {
		log.V(1).Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.V(1).Infoln("Received Offer <", offer.Id.GetValue(), "> on host ", *offer.Hostname, "with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		dCpus := config.Task.Docker.Cpus
		dMem := config.Task.Docker.Mem

		var tasks []*mesos.TaskInfo
		for sched.tasksLaunched < sched.totalTasks &&
			dCpus <= remainingCpus &&
			dMem <= remainingMems {

			sched.tasksLaunched++

			tID := strconv.Itoa(sched.tasksLaunched)
			if config.Task.Id != "" {
				tID = config.Task.Id
			}

			tName := "mesos-runonce-" + tID
			if config.Task.Name != "" {
				tName = config.Task.Name
			}

			taskId := &mesos.TaskID{
				Value: proto.String(tID),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task := &mesos.TaskInfo{
				Name:    proto.String(tName),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", dCpus),
					util.NewScalarResource("mem", dMem),
				},
				Command: &mesos.CommandInfo{
					Shell: proto.Bool(false),
				},
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image:          proto.String(config.Task.Docker.Image),
						ForcePullImage: proto.Bool(config.Task.Docker.ForcePullImage),
					},
				},
			}
			log.V(1).Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			if len(config.Task.Docker.Env) != 0 || config.Task.Docker.EnvString != "" {
				task.Command.Environment = config.EnvVars()
			}

			// Allow arbitrary commands, else just use whatever the image defines in CMD
			if config.Task.Docker.Cmd != "" {
				task.Command.Value = proto.String(config.Task.Docker.Cmd)
				task.Command.Shell = proto.Bool(true)
			}

			tasks = append(tasks, task)
			remainingCpus -= dCpus
			remainingMems -= dMem
		}
		log.V(1).Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}

func (sched *MesosRunonceScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.V(1).Infoln("Status update: task", status.TaskId.GetValue(), " is in state ", status.State.Enum().String())
	eventCh <- status

	if status.GetState() == mesos.TaskState_TASK_FINISHED {
		sched.tasksFinished++
	}

	if sched.tasksFinished >= sched.totalTasks {
		log.V(1).Infoln("Total tasks completed, stopping framework.")
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED ||
		status.GetState() == mesos.TaskState_TASK_FAILED ||
		status.GetState() == mesos.TaskState_TASK_ERROR {
		exitStatus = 1
		log.Warningf("mesos TaskStatus: %v", status)
		driver.Abort()
		log.Errorln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message.", status.GetMessage(),
		)
	}
}

// ----------------------- Helper methods ------------------------- //

func printLogs() {
	timer := time.Tick(500 * time.Millisecond)
	var (
		readableStatus *mesos.TaskStatus
		finished       bool
		oout, oerr     int
	)
	for {
		select {
		case status := <-eventCh:
			switch status.GetState() {
			case mesos.TaskState_TASK_RUNNING,
				mesos.TaskState_TASK_STARTING:
				readableStatus = status
			case mesos.TaskState_TASK_FAILED,
				mesos.TaskState_TASK_LOST,
				mesos.TaskState_TASK_KILLED:
				readableStatus = status
				finished = true
			case mesos.TaskState_TASK_FINISHED:
				finished = true
			}
		case <-timer:
			if readableStatus != nil {
				if finished {
					time.Sleep(3 * time.Second)
				}
				x := printLog(readableStatus, oout, os.Stdout)
				y := printLog(readableStatus, oerr, os.Stderr)
				if finished && x == 0 && y == 0 {
					return
				}
				oout += x
				oerr += y
			}
		}
	}
}

func printLog(status *mesos.TaskStatus, offset int, w io.Writer) int {
	file := "stdout"
	if w == os.Stderr {
		file = "stderr"
	}
	data, err := mlog.FetchLogs(status, offset, file, _frameworkId)
	if err != nil {
		log.V(1).Infof("fetch logs err: %s\n", err)
		return 0
	}
	if len(data) > 0 {
		fmt.Fprint(w, string(data))
	}
	return len(data)
}

func prepareExecutorInfo() *mesos.ExecutorInfo {
	// Create mesos scheduler driver.
	containerType := mesos.ContainerInfo_DOCKER
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("default"),
		Name:       proto.String("mesos-runonce-executor"),
		Source:     proto.String("mesos-runonce-executor"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Exit(err)
	}
	if len(addr) < 1 {
		log.Exitf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

func newMesosRunonceScheduler(exec *mesos.ExecutorInfo) *MesosRunonceScheduler {
	return &MesosRunonceScheduler{
		executor:      exec,
		tasksLaunched: 0,
		tasksFinished: 0,
		totalTasks:    config.Runonce.TaskCount,
	}
}

func fwinfo() *mesos.FrameworkInfo {
	return &mesos.FrameworkInfo{
		User: proto.String(config.Runonce.MesosRunasUser),
		Name: proto.String("mesos-runonce"),
	}
}

func cred(fwinfo *mesos.FrameworkInfo) *mesos.Credential {
	cred := (*mesos.Credential)(nil)
	mesosAuthPrincipal := config.Runonce.AuthPrincipal
	if mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(mesosAuthPrincipal)
		cred = &mesos.Credential{
			Principal: proto.String(mesosAuthPrincipal),
		}
		mesosAuthSecretFile := config.Runonce.AuthSecretFile
		if mesosAuthSecretFile != "" {
			_, err := os.Stat(mesosAuthSecretFile)
			if err != nil {
				log.Exit("missing secret file: ", err.Error())
			}
			secret, err := ioutil.ReadFile(mesosAuthSecretFile)
			if err != nil {
				log.Exit("failed to read secret file: ", err.Error())
			}
			cred.Secret = proto.String(strings.TrimSuffix(string(secret), "\n"))
		}
	}
	return cred
}

func driverConfig(fwinfo *mesos.FrameworkInfo) sched.DriverConfig {
	// build command executor
	exec := prepareExecutorInfo()

	publishedAddress := parseIP(config.Runonce.Address)
	dConfig := sched.DriverConfig{
		Scheduler:        newMesosRunonceScheduler(exec),
		Framework:        fwinfo,
		Master:           config.Runonce.Master,
		Credential:       cred(fwinfo),
		PublishedAddress: publishedAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, config.Runonce.AuthProvider)
			ctx = sasl.WithBindingAddress(ctx, publishedAddress)
			return ctx
		},
	}

	// Allow listening port to be configurable so we can run this inside of
	// mesos if desired.
	// NOTE only affects main PID, meaning the authentication step uses
	// another PID which picks a random (32K range) port :(. Opened
	// https://github.com/mesos/mesos-go/issues/229 to discuss.
	if config.Runonce.BindingPort != 0 {
		dConfig.BindingPort = uint16(config.Runonce.BindingPort)
	}
	return dConfig
}

// ----------------------- func main() ------------------------- //

func main() {
	log.V(1).Infoln("Initializing the Example Scheduler...")
	config = loadConfig()

	// the framework
	fwinfo := fwinfo()

	driver, err := sched.NewMesosSchedulerDriver(driverConfig(fwinfo))
	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	// Don't block on this call because we need printLogs to be the process blocker to ensure we retrieve all the logs.
	go func() {
		if stat, err := driver.Run(); err != nil {
			log.V(1).Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
		}
	}()

	printLogs()
	log.V(1).Infof("framework terminating")
	os.Exit(exitStatus)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
)

type rss struct {
	Channel channel `xml:"channel"`
}

type channel struct {
	Title       string `xml:"title"`
	Link        string `xml:"link"`
	Description string `xml:"description"`
	Items       []item `xml:"item"`
}

type item struct {
	Enclosure enclosure `xml:"enclosure"`
	Link      string    `xml:"link"`
	Guid      string    `xml:"guid"`
}

type enclosure struct {
	Url  string `xml:"url,attr"`
	Type string `xml:"type,attr"`
}

type archive struct {
	Guid []string `json:"guid"`
}

func main() {

	accountPtr := flag.String("user", "", "soup.io username")
	flag.Parse()

	if *accountPtr == "" {
		flag.Usage()
		os.Exit(0)
	}

	url := fmt.Sprintf("http://%s.soup.io/rss", *accountPtr)
	feedResponse, err := http.Get(url)
	if err != nil {
		fmt.Printf("Error fetching %s: %s", url, err)
		os.Exit(1)
	}
	defer feedResponse.Body.Close()
	feedBody, err := ioutil.ReadAll(feedResponse.Body)
	if err != nil {
		fmt.Printf("Error reading feed from %s: %s", url, err)
		os.Exit(1)
	}

	var feed rss
	xml.Unmarshal(feedBody, &feed)

	var wg sync.WaitGroup

	a, err := readArchive()
	if err != nil {
		// no archive yet. it will be created
		a = archive{}
	}

	if _, err := os.Stat("archive"); os.IsNotExist(err) {
		err = os.Mkdir("archive", 0755)
		if err != nil {
			fmt.Printf("Error creating archive: %s\n", err)
			os.Exit(1)
		}
	}

	c := make(chan string)

	for _, i := range feed.Channel.Items {
		wg.Add(1)
		go func(i item, c chan string) {
			defer wg.Done()
			if inArchive(i.Guid, a) {
				fmt.Printf("Skipping %s. Already in archive.\n", i.Enclosure.Url)
				return
			}
			fmt.Printf("Saving %s...\n", i.Enclosure.Url)

			filepath := "archive/" + path.Base(i.Enclosure.Url)
			file, err := os.Create(filepath)
			if err != nil {
				fmt.Printf("Error opening file %s: %s\n", filepath, err)
				return
			}

			response, err := http.Get(i.Enclosure.Url)
			if err != nil {
				fmt.Printf("Error fetching %s: %s\n", i.Enclosure.Url, err)
				return
			}
			_, err = io.Copy(file, response.Body)
			if err != nil {
				fmt.Printf("Error writing file %s: %s\n", filepath, err)
				response.Body.Close()
				return
			}
			response.Body.Close()
			file.Close()

			c <- i.Guid
		}(i, c)
	}

	waitForArchive := make(chan bool)
	go func(c chan string) {
		for guid := range c {
			addToArchive(guid)
		}
		waitForArchive <- true
	}(c)

	wg.Wait()
	close(c)
	<-waitForArchive
}

func addToArchive(guid string) {
	a, err := readArchive()
	if err != nil {
		a = archive{}
	}
	a.Guid = append(a.Guid, guid)
	data, err := json.Marshal(a)
	if err != nil {
		fmt.Println("Error marshalling archive data: ", err)
		return
	}
	ioutil.WriteFile("archive/guids.json", data, 0600)
}

func inArchive(guid string, a archive) bool {
	for _, s := range a.Guid {
		if guid == s {
			return true
		}
	}

	return false
}

func readArchive() (archive, error) {
	var a archive

	archiveData, err := ioutil.ReadFile("archive/guids.json")
	if err != nil {
		return a, err
	}

	err = json.Unmarshal(archiveData, &a)
	if err != nil {
		return a, err
	}

	return a, nil
}
<commit_msg>add check for http status and only save on status 200<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
)

type rss struct {
	Channel channel `xml:"channel"`
}

type channel struct {
	Title       string `xml:"title"`
	Link        string `xml:"link"`
	Description string `xml:"description"`
	Items       []item `xml:"item"`
}

type item struct {
	Enclosure enclosure `xml:"enclosure"`
	Link      string    `xml:"link"`
	Guid      string    `xml:"guid"`
}

type enclosure struct {
	Url  string `xml:"url,attr"`
	Type string `xml:"type,attr"`
}

type archive struct {
	Guid []string `json:"guid"`
}

func main() {

	accountPtr := flag.String("user", "", "soup.io username")
	flag.Parse()

	if *accountPtr == "" {
		flag.Usage()
		os.Exit(0)
	}

	url := fmt.Sprintf("http://%s.soup.io/rss", *accountPtr)
	feedResponse, err := http.Get(url)
	if err != nil {
		fmt.Printf("Error fetching %s: %s", url, err)
		os.Exit(1)
	}
	defer feedResponse.Body.Close()
	feedBody, err := ioutil.ReadAll(feedResponse.Body)
	if err != nil {
		fmt.Printf("Error reading feed from %s: %s", url, err)
		os.Exit(1)
	}

	var feed rss
	xml.Unmarshal(feedBody, &feed)

	var wg sync.WaitGroup

	a, err := readArchive()
	if err != nil {
		// no archive yet. it will be created
		a = archive{}
	}

	if _, err := os.Stat("archive"); os.IsNotExist(err) {
		err = os.Mkdir("archive", 0755)
		if err != nil {
			fmt.Printf("Error creating archive: %s\n", err)
			os.Exit(1)
		}
	}

	c := make(chan string)

	for _, i := range feed.Channel.Items {
		wg.Add(1)
		go func(i item, c chan string) {
			defer wg.Done()
			if inArchive(i.Guid, a) {
				fmt.Printf("Skipping %s. Already in archive.\n", i.Enclosure.Url)
				return
			}
			fmt.Printf("Saving %s...\n", i.Enclosure.Url)

			filepath := "archive/" + path.Base(i.Enclosure.Url)
			file, err := os.Create(filepath)
			if err != nil {
				fmt.Printf("Error opening file %s: %s\n", filepath, err)
				return
			}

			response, err := http.Get(i.Enclosure.Url)
			if err != nil {
				fmt.Printf("Error fetching %s: %s\n", i.Enclosure.Url, err)
				return
			}
			if response.StatusCode != http.StatusOK {
				fmt.Printf("Error fetching %s: Status %d\n", i.Enclosure.Url, response.StatusCode)
				return
			}
			_, err = io.Copy(file, response.Body)
			if err != nil {
				fmt.Printf("Error writing file %s: %s\n", filepath, err)
				response.Body.Close()
				return
			}
			response.Body.Close()
			file.Close()

			c <- i.Guid
		}(i, c)
	}

	waitForArchive := make(chan bool)
	go func(c chan string) {
		for guid := range c {
			addToArchive(guid)
		}
		waitForArchive <- true
	}(c)

	wg.Wait()
	close(c)
	<-waitForArchive
}

func addToArchive(guid string) {
	a, err := readArchive()
	if err != nil {
		a = archive{}
	}
	a.Guid = append(a.Guid, guid)
	data, err := json.Marshal(a)
	if err != nil {
		fmt.Println("Error marshalling archive data: ", err)
		return
	}
	ioutil.WriteFile("archive/guids.json", data, 0600)
}

func inArchive(guid string, a archive) bool {
	for _, s := range a.Guid {
		if guid == s {
			return true
		}
	}

	return false
}

func readArchive() (archive, error) {
	var a archive

	archiveData, err := ioutil.ReadFile("archive/guids.json")
	if err != nil {
		return a, err
	}

	err = json.Unmarshal(archiveData, &a)
	if err != nil {
		return a, err
	}

	return a, nil
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "github.com/ugol/infinispan-go/infinispan"

func main() {

	fmt.Println("Infinispan GO client")
	infinispan.TestConnect()
}
<commit_msg>moved testconnection to a proper test<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var (
	logging = flag.Bool("log", false, "Enable logging")
	region  = flag.String("region", "us-east-1", "Set S3 region")
	bucket  = flag.String("bucket", "", "Name of bucket in S3")
	key     = flag.String("key", "", "Key for object in bucket")
	path    = flag.String("path", "", "Path for download")
)

func main() {
	flag.Parse()
	if *bucket == "" || *key == "" {
		fmt.Println("Please, specify valid parameters!")
		os.Exit(1)
	}

	client := s3.New(session.New(createConfig()))
	request, _ := client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(*bucket),
		Key:    aws.String(*key),
	})

	url, err := request.Presign(10 * time.Minute)
	if err != nil {
		log.Fatal("Failed to create pre-signed URL because of: ", err)
	}

	log.Println("Pre-signed URL:", url)
	downloadFile(url)
	log.Println("Fail is downloaded!")
}

func downloadFile(url string) {
	resp, err := http.DefaultClient.Get(url)
	if err != nil {
		log.Fatal("Failed to download a file by pre-signed URL because of: ", err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("Failed to get a body from HTTP request because of: ", err)
	}
	err = ioutil.WriteFile(getFileName(*key), body, 0777)
	if err != nil {
		log.Fatal("Failed to create a file from a body of HTTP request because of: ", err)
	}
}

func getFileName(filepath string) string {
	index := strings.LastIndex(filepath, "/")
	if index == -1 {
		return filepath
	}
	return filepath[index+1:]
}

func createConfig() *aws.Config {
	config := aws.NewConfig()
	config.WithCredentials(credentials.NewEnvCredentials())
	config.WithRegion(*region)
	if *logging {
		config.WithLogLevel(aws.LogDebugWithHTTPBody)
	}
	return config
}
<commit_msg>Fix typo<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var (
	logging = flag.Bool("log", false, "Enable logging")
	region  = flag.String("region", "us-east-1", "Set S3 region")
	bucket  = flag.String("bucket", "", "Name of bucket in S3")
	key     = flag.String("key", "", "Key for object in bucket")
	path    = flag.String("path", "", "Path for download")
)

func main() {
	flag.Parse()
	if *bucket == "" || *key == "" {
		fmt.Println("Please, specify valid parameters!")
		os.Exit(1)
	}

	client := s3.New(session.New(createConfig()))
	request, _ := client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(*bucket),
		Key:    aws.String(*key),
	})

	url, err := request.Presign(10 * time.Minute)
	if err != nil {
		log.Fatal("Failed to create pre-signed URL because of: ", err)
	}

	log.Println("Pre-signed URL:", url)
	downloadFile(url)
	log.Println("File is downloaded!")
}

func downloadFile(url string) {
	resp, err := http.DefaultClient.Get(url)
	if err != nil {
		log.Fatal("Failed to download a file by pre-signed URL because of: ", err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("Failed to get a body from HTTP request because of: ", err)
	}
	err = ioutil.WriteFile(getFileName(*key), body, 0777)
	if err != nil {
		log.Fatal("Failed to create a file from a body of HTTP request because of: ", err)
	}
}

func getFileName(filepath string) string {
	index := strings.LastIndex(filepath, "/")
	if index == -1 {
		return filepath
	}
	return filepath[index+1:]
}

func createConfig() *aws.Config {
	config := aws.NewConfig()
	config.WithCredentials(credentials.NewEnvCredentials())
	config.WithRegion(*region)
	if *logging {
		config.WithLogLevel(aws.LogDebugWithHTTPBody)
	}
	return config
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/miketheprogrammer/go-thrust/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/miketheprogrammer/go-thrust/lib/spawn"
)

func install(c *cli.Context) {
	spawn.SetBaseDirectory("") // Default to usr.homedir.
	tp := spawn.NewThrustProvisioner()
	if err := tp.Provision(); err != nil {
		panic(err)
	}
	fmt.Println("Thrust installed")
}

func main() {
	app := cli.NewApp()
	app.Name = "go-thrust"
	app.Usage = "Tools to developp Thrust applications in Go"

	app.Commands = []cli.Command{
		{
			Name:      "install",
			ShortName: "i",
			Usage:     "Install Thrust",
			Action: func(c *cli.Context) {
				install(c)
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>fix deps in main.go<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/miketheprogrammer/go-thrust/lib/spawn"
)

func install(c *cli.Context) {
	spawn.SetBaseDirectory("") // Default to usr.homedir.
	tp := spawn.NewThrustProvisioner()
	if err := tp.Provision(); err != nil {
		panic(err)
	}
	fmt.Println("Thrust installed")
}

func main() {
	app := cli.NewApp()
	app.Name = "go-thrust"
	app.Usage = "Tools to developp Thrust applications in Go"

	app.Commands = []cli.Command{
		{
			Name:      "install",
			ShortName: "i",
			Usage:     "Install Thrust",
			Action: func(c *cli.Context) {
				install(c)
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

type PullRequestList struct {
	PageLen int               `json:"pagelen"`
	Page    int               `json:"page"`
	Size    int               `json:"size"`
	Items   []PullRequestInfo `json:"values"`
}

type PullRequestInfo struct {
	ID          int       `json:"id"`
	Title       string    `json:"title"`
	CreatedOn   time.Time `json:"created_on"`
	UpdatedOn   time.Time `json:"updated_on"`
	Author      User      `json:"author"`
	Destination Commit    `json:"destination"`
	Source      Commit    `json:"source"`
}

type User struct {
	Username    string `json:"username"`
	DisplayName string `json:"display_name"`
}

type Commit struct {
	Branch Branch `json:"branch"`
}

type Branch struct {
	Name string `json:"name"`
}

type Remote struct {
	Org  string
	Repo string
}

func main() {
	username := os.Getenv("bbuser")
	password := os.Getenv("bbpassword")

	if username == "" {
		log.Fatalln("bbuser is not set")
		return
	}
	if password == "" {
		log.Fatalln("bbpassword is not set")
		return
	}

	remote, errRemote := getRemote()
	dumpError(errRemote)

	fmt.Println(remote.Org, remote.Repo)
	prList, err := getPullRequestList(remote.Org, remote.Repo, username, password)
	dumpError(err)

	for _, pr := range prList.Items {
		fmt.Printf("%d %s %s->%s %s\n", pr.ID, pr.Author.DisplayName, pr.Source.Branch.Name, pr.Destination.Branch.Name, pr.Title)
	}
}

func getPullRequestList(org string, repo string, username string, password string) (*PullRequestList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", fmt.Sprintf("https://bitbucket.org/api/2.0/repositories/%s/%s/pullrequests", org, repo), nil)
	req.SetBasicAuth(username, password)
	resp, err := client.Do(req)
	dumpError(err)
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		dumpResponse(resp)
		return nil, errors.New("Failed response")
	}
	return parsePullRequestListResponse(resp), nil
}

func parsePullRequestListResponse(resp *http.Response) *PullRequestList {
	var prList PullRequestList
	errDecode := json.NewDecoder(resp.Body).Decode(&prList)
	dumpError(errDecode)
	return &prList
}

func dumpResponse(resp *http.Response) {
	_, err := io.Copy(os.Stdout, resp.Body)
	dumpError(err)
}

func dumpError(err error) {
	if err != nil {
		panic(err.Error())
	}
}

func getRemote() (*Remote, error) {
	cmdName := "git"
	cmdArgs := []string{"remote", "get-url", "origin"}
	cmdOut, err := exec.Command(cmdName, cmdArgs...).Output()
	dumpError(err)

	remote := string(cmdOut)

	if !strings.Contains(remote, "bitbucket.org") {
		return nil, errors.New("Only BitBucket remote is supported")
	}

	remote = strings.Replace(remote, "https://bitbucket.org/", "", -1)
	remote = strings.Replace(remote, ".git", "", -1)
	remote = strings.Replace(remote, "\n", "", -1)

	parts := strings.Split(remote, "/")

	r := Remote{
		Org:  parts[0],
		Repo: parts[1],
	}

	return &r, nil
}
<commit_msg>Minor refactoring<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

type PullRequestList struct {
	PageLen int               `json:"pagelen"`
	Page    int               `json:"page"`
	Size    int               `json:"size"`
	Items   []PullRequestInfo `json:"values"`
}

type PullRequestInfo struct {
	ID          int       `json:"id"`
	Title       string    `json:"title"`
	CreatedOn   time.Time `json:"created_on"`
	UpdatedOn   time.Time `json:"updated_on"`
	Author      User      `json:"author"`
	Destination Commit    `json:"destination"`
	Source      Commit    `json:"source"`
}

type User struct {
	Username    string `json:"username"`
	DisplayName string `json:"display_name"`
}

type Commit struct {
	Branch Branch `json:"branch"`
}

type Branch struct {
	Name string `json:"name"`
}

type Remote struct {
	Org  string
	Repo string
}

func main() {
	username, password, errCred := getCredentials()
	dumpError(errCred)

	remote, errRemote := getRemote()
	dumpError(errRemote)

	prList, err := getPullRequestList(remote.Org, remote.Repo, username, password)
	dumpError(err)

	for _, pr := range prList.Items {
		fmt.Printf("%d %s %s->%s %s\n", pr.ID, pr.Author.DisplayName, pr.Source.Branch.Name, pr.Destination.Branch.Name, pr.Title)
	}
}

func getCredentials() (string, string, error) {
	username := os.Getenv("bbuser")
	password := os.Getenv("bbpassword")

	if username == "" {
		return "", "", errors.New("bbuser is not set")
	}
	if password == "" {
		return "", "", errors.New("bbpassword is not set")
	}

	return username, password, nil
}

func getPullRequestList(org string, repo string, username string, password string) (*PullRequestList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", fmt.Sprintf("https://bitbucket.org/api/2.0/repositories/%s/%s/pullrequests", org, repo), nil)
	req.SetBasicAuth(username, password)
	resp, err := client.Do(req)
	dumpError(err)
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		dumpResponse(resp)
		return nil, errors.New("Failed response")
	}
	return parsePullRequestListResponse(resp), nil
}

func parsePullRequestListResponse(resp *http.Response) *PullRequestList {
	var prList PullRequestList
	errDecode := json.NewDecoder(resp.Body).Decode(&prList)
	dumpError(errDecode)
	return &prList
}

func dumpResponse(resp *http.Response) {
	_, err := io.Copy(os.Stdout, resp.Body)
	dumpError(err)
}

func dumpError(err error) {
	if err != nil {
		panic(err.Error())
	}
}

func getRemote() (*Remote, error) {
	cmdName := "git"
	cmdArgs := []string{"remote", "get-url", "origin"}
	cmdOut, err := exec.Command(cmdName, cmdArgs...).Output()
	dumpError(err)

	remote := string(cmdOut)

	if !strings.Contains(remote, "bitbucket.org") {
		return nil, errors.New("Only BitBucket remote is supported")
	}

	remote = strings.Replace(remote, "https://bitbucket.org/", "", -1)
	remote = strings.Replace(remote, ".git", "", -1)
	remote = strings.Replace(remote, "\n", "", -1)

	parts := strings.Split(remote, "/")

	r := Remote{
		Org:  parts[0],
		Repo: parts[1],
	}

	return &r, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"

	"github.com/adamdecaf/cert-manage/cmd"
)

const (
	version = "0.0.1-dev"
)

var (
	fs = flag.NewFlagSet("flags", flag.ExitOnError)

	// Commands
	backup    = fs.Bool("backup", false, "Make a backup")
	list      = fs.Bool("list", false, "List certificates (by default on the system, see -app)")
	restore   = fs.Bool("restore", false, "Restore from a given backup file, if it exists (and supported)")
	whitelist = fs.Bool("whitelist", false, "Filter certificates according to the provided whitelist")
	ver       = fs.Bool("version", false, "Show the version of cert-manage")

	// Qualifiers
	file = fs.String("file", "", "File to use for operation (restore, whitelist)")

	// Filters
	app = fs.String("app", "", "Specify an application (see -list)")

	// Output
	// -count: Just output the count of trusted certs
	format = fs.String("format", "table", "Specify the output format (options: raw, table)")
)

func init() {
	fs.Usage = func() {
		fmt.Printf(`Usage of cert-manage (version %s)
COMMANDS
  -backup   Take a backup of the specified certificatestore
  -list     List the currently installed and trusted certificates
  -restore  Revert the certificate trust back to, optionally takes -file <path>
  -version  Show the version of cert-manage

  Commands which require a file (via -file)
  -whitelist -file <path> Remove trust from certificates which do not match the whitelist in <path>

FILTERS
  Filters can be applied to the following commands: -backup, -list, -restore, -whitelist
  -app <name> The name of an application which to perform the given command on. (Examples: chrome, java)

OUTPUT
  -format Change the output format for a given command (default: table, options: table, raw)
`, version)
	}
}

func main() {
	fs.Parse(os.Args[1:])

	// Show the version
	if ver != nil && *ver {
		fmt.Printf("%s\n", version)
	}

	// Perform a restore
	// Note: This always needs to happen before -whitelist and before -backup
	if restore != nil && *restore {
		err := appChoice(app,
			func(a string) error {
				return cmd.RestoreForApp(a, *file)
			},
			func() error {
				return cmd.RestoreForPlatform(*file)
			})
		exit("Restore completed successfully", err)
	}

	// Take a backup
	// Note: This always needs to be done before -whitelist
	if backup != nil && *backup {
		err := appChoice(app,
			func(a string) error {
				return cmd.BackupForApp(a)
			},
			func() error {
				return cmd.BackupForPlatform()
			})
		exit("Backup completed successfully", err)
	}

	// Whitelist
	if whitelist != nil && *whitelist {
		if *file == "" {
			exit("", errors.New("no -file specified"))
		}
		err := appChoice(app,
			func(a string) error {
				return cmd.WhitelistForApp(a, *file, *format)
			},
			func() error {
				return cmd.WhitelistForPlatform(*file, *format)
			})
		exit("Whitelist completed successfully", err)
	}

	// List
	if list != nil && *list {
		err := appChoice(app,
			func(a string) error {
				return cmd.ListCertsForApp(*app, *format)
			},
			func() error {
				return cmd.ListCertsForPlatform(*format)
			})
		exit("", err)
	}
}

type fn func() error
type appfn func(string) error

// appChoice decides between `appfn` and `fn` given the presence of `app`
// being non-nil and a non-empty string.
func appChoice(app *string, appfn appfn, fn fn) error {
	if app != nil && *app != "" {
		return appfn(*app)
	}
	return fn()
}

func exit(msg string, err error) {
	if err != nil {
		fmt.Println(err)
		fs.PrintDefaults()
		os.Exit(1)
	}
	if msg != "" {
		fmt.Println(msg)
	}
}
<commit_msg>print our Usage() on errors<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"

	"github.com/adamdecaf/cert-manage/cmd"
)

const (
	version = "0.0.1-dev"
)

var (
	fs = flag.NewFlagSet("flags", flag.ExitOnError)

	// Commands
	backup    = fs.Bool("backup", false, "Make a backup")
	list      = fs.Bool("list", false, "List certificates (by default on the system, see -app)")
	restore   = fs.Bool("restore", false, "Restore from a given backup file, if it exists (and supported)")
	whitelist = fs.Bool("whitelist", false, "Filter certificates according to the provided whitelist")
	ver       = fs.Bool("version", false, "Show the version of cert-manage")

	// Qualifiers
	file = fs.String("file", "", "File to use for operation (restore, whitelist)")

	// Filters
	app = fs.String("app", "", "Specify an application (see -list)")

	// Output
	// -count: Just output the count of trusted certs
	format = fs.String("format", "table", "Specify the output format (options: raw, table)")
)

func init() {
	fs.Usage = func() {
		fmt.Printf(`Usage of cert-manage (version %s)
COMMANDS
  -backup   Take a backup of the specified certificatestore
  -list     List the currently installed and trusted certificates
  -restore  Revert the certificate trust back to, optionally takes -file <path>
  -version  Show the version of cert-manage

  Commands which require a file (via -file)
  -whitelist -file <path> Remove trust from certificates which do not match the whitelist in <path>

FILTERS
  Filters can be applied to the following commands: -backup, -list, -restore, -whitelist
  -app <name> The name of an application which to perform the given command on. (Examples: chrome, java)

OUTPUT
  -format Change the output format for a given command (default: table, options: table, raw)
`, version)
	}
}

func main() {
	fs.Parse(os.Args[1:])

	// Show the version
	if ver != nil && *ver {
		fmt.Printf("%s\n", version)
	}

	// Perform a restore
	// Note: This always needs to happen before -whitelist and before -backup
	if restore != nil && *restore {
		err := appChoice(app,
			func(a string) error {
				return cmd.RestoreForApp(a, *file)
			},
			func() error {
				return cmd.RestoreForPlatform(*file)
			})
		exit("Restore completed successfully", err)
	}

	// Take a backup
	// Note: This always needs to be done before -whitelist
	if backup != nil && *backup {
		err := appChoice(app,
			func(a string) error {
				return cmd.BackupForApp(a)
			},
			func() error {
				return cmd.BackupForPlatform()
			})
		exit("Backup completed successfully", err)
	}

	// Whitelist
	if whitelist != nil && *whitelist {
		if *file == "" {
			exit("", errors.New("no -file specified"))
		}
		err := appChoice(app,
			func(a string) error {
				return cmd.WhitelistForApp(a, *file, *format)
			},
			func() error {
				return cmd.WhitelistForPlatform(*file, *format)
			})
		exit("Whitelist completed successfully", err)
	}

	// List
	if list != nil && *list {
		err := appChoice(app,
			func(a string) error {
				return cmd.ListCertsForApp(*app, *format)
			},
			func() error {
				return cmd.ListCertsForPlatform(*format)
			})
		exit("", err)
	}
}

type fn func() error
type appfn func(string) error

// appChoice decides between `appfn` and `fn` given the presence of `app`
// being non-nil and a non-empty string.
func appChoice(app *string, appfn appfn, fn fn) error {
	if app != nil && *app != "" {
		return appfn(*app)
	}
	return fn()
}

func exit(msg string, err error) {
	if err != nil {
		fmt.Println(err)
		fs.Usage()
		os.Exit(1)
	}
	if msg != "" {
		fmt.Println(msg)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"os"
)

func main() {
	var opts struct {
		Name string `short:"n" long:"name" description:"Name to greet" default:"world"`
	}

	parser := flags.NewParser(&opts, flags.Default)
	parser.Usage = "[options]"
	if _, err := parser.Parse(); err != nil {
		fmt.Fprintln(os.Stderr, "Error parsing command line")
		os.Exit(1)
	}

	fmt.Printf("Hello, %s!\n", opts.Name)
}
<commit_msg>Add version option<commit_after>package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"os"
)

const VERSION = "0.2.0pre"

func main() {
	var opts struct {
		Name    string `short:"n" long:"name" description:"Name to greet" default:"world"`
		Version bool   `long:"version" description:"Display version and exit"`
	}

	parser := flags.NewParser(&opts, flags.Default)
	parser.Usage = "[options]"
	if _, err := parser.Parse(); err != nil {
		fmt.Fprintln(os.Stderr, "Error parsing command line")
		os.Exit(1)
	}

	if opts.Version {
		fmt.Println("go-hello " + VERSION)
		os.Exit(0)
	}

	fmt.Printf("Hello, %s!\n", opts.Name)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"archive/tar"
	"io"
	"net/http/httptest"
	"net/url"
	"testing"

	"github.com/tektoncd/pipeline/test/parse"

	"github.com/ghodss/yaml"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/registry"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	tkremote "github.com/tektoncd/pipeline/pkg/remote/oci"
)

func TestCreateImage(t *testing.T) {
	// Set up a fake registry to push an image to.
	s := httptest.NewServer(registry.New())
	defer s.Close()
	u, _ := url.Parse(s.URL)

	task := parse.MustParseTask(t, `
metadata:
  name: test-create-image
`)

	ref, err := CreateImage(u.Host+"/task/test-create-image", task)
	if err != nil {
		t.Errorf("uploading image failed unexpectedly with an error: %w", err)
	}

	// Pull the image and ensure the layers are composed correctly.
	imgRef, err := name.ParseReference(ref)
	if err != nil {
		t.Errorf("digest %s is not a valid reference: %w", ref, err)
	}

	img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(authn.DefaultKeychain))
	if err != nil {
		t.Errorf("could not fetch created image: %w", err)
	}

	m, err := img.Manifest()
	if err != nil {
		t.Errorf("failed to fetch img manifest: %w", err)
	}

	layers, err := img.Layers()
	if err != nil || len(layers) != 1 {
		t.Errorf("img layers were incorrect. Num Layers: %d. Err: %w", len(layers), err)
	}

	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.TitleAnnotation], "test-create-image"); diff != "" {
		t.Error(diff)
	}
	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.KindAnnotation], "task"); diff != "" {
		t.Error(diff)
	}
	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.APIVersionAnnotation], "v1beta1"); diff != "" {
		t.Error(diff)
	}

	// Read the layer's contents and ensure it matches the task we uploaded.
	rc, err := layers[0].Uncompressed()
	if err != nil {
		t.Errorf("layer contents were corrupted: %w", err)
	}
	defer rc.Close()

	reader := tar.NewReader(rc)
	header, err := reader.Next()
	if err != nil {
		t.Errorf("failed to load tar bundle: %w", err)
	}

	actual := make([]byte, header.Size)
	if _, err := reader.Read(actual); err != nil && err != io.EOF {
		t.Errorf("failed to read contents of tar bundle: %w", err)
	}

	raw, err := yaml.Marshal(task)
	if err != nil {
		t.Fatalf("Could not marshal task to bytes: %#v", err)
	}
	if diff := cmp.Diff(string(raw), string(actual)); diff != "" {
		t.Error(diff)
	}
}
<commit_msg>Fix incorrect format specifier in test files<commit_after>/*
Copyright 2020 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"archive/tar"
	"io"
	"net/http/httptest"
	"net/url"
	"testing"

	"github.com/tektoncd/pipeline/test/parse"

	"github.com/ghodss/yaml"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/registry"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	tkremote "github.com/tektoncd/pipeline/pkg/remote/oci"
)

func TestCreateImage(t *testing.T) {
	// Set up a fake registry to push an image to.
	s := httptest.NewServer(registry.New())
	defer s.Close()
	u, _ := url.Parse(s.URL)

	task := parse.MustParseTask(t, `
metadata:
  name: test-create-image
`)

	ref, err := CreateImage(u.Host+"/task/test-create-image", task)
	if err != nil {
		t.Errorf("uploading image failed unexpectedly with an error: %v", err)
	}

	// Pull the image and ensure the layers are composed correctly.
	imgRef, err := name.ParseReference(ref)
	if err != nil {
		t.Errorf("digest %s is not a valid reference: %v", ref, err)
	}

	img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(authn.DefaultKeychain))
	if err != nil {
		t.Errorf("could not fetch created image: %v", err)
	}

	m, err := img.Manifest()
	if err != nil {
		t.Errorf("failed to fetch img manifest: %v", err)
	}

	layers, err := img.Layers()
	if err != nil || len(layers) != 1 {
		t.Errorf("img layers were incorrect. Num Layers: %d. Err: %v", len(layers), err)
	}

	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.TitleAnnotation], "test-create-image"); diff != "" {
		t.Error(diff)
	}
	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.KindAnnotation], "task"); diff != "" {
		t.Error(diff)
	}
	if diff := cmp.Diff(m.Layers[0].Annotations[tkremote.APIVersionAnnotation], "v1beta1"); diff != "" {
		t.Error(diff)
	}

	// Read the layer's contents and ensure it matches the task we uploaded.
	rc, err := layers[0].Uncompressed()
	if err != nil {
		t.Errorf("layer contents were corrupted: %v", err)
	}
	defer rc.Close()

	reader := tar.NewReader(rc)
	header, err := reader.Next()
	if err != nil {
		t.Errorf("failed to load tar bundle: %v", err)
	}

	actual := make([]byte, header.Size)
	if _, err := reader.Read(actual); err != nil && err != io.EOF {
		t.Errorf("failed to read contents of tar bundle: %v", err)
	}

	raw, err := yaml.Marshal(task)
	if err != nil {
		t.Fatalf("Could not marshal task to bytes: %#v", err)
	}
	if diff := cmp.Diff(string(raw), string(actual)); diff != "" {
		t.Error(diff)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

var httpFlag = flag.String("http", ":8080", "Serve HTTP at given address")
var httpsFlag = flag.String("https", "", "Serve HTTPS at given address")
var certFlag = flag.String("cert", "", "Use the provided TLS certificate")
var keyFlag = flag.String("key", "", "Use the provided TLS key")

func main() {
	if err := run(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func run() error {
	flag.Parse()
	if len(flag.Args()) > 0 {
		return fmt.Errorf("too many arguments: %s", flag.Args()[0])
	}

	http.HandleFunc("/", handler)

	if *httpFlag == "" && *httpsFlag == "" {
		return fmt.Errorf("must provide -http and/or -https")
	}
	if (*httpsFlag != "" || *certFlag != "" || *keyFlag != "") && (*httpsFlag == "" || *certFlag == "" || *keyFlag == "") {
		return fmt.Errorf("-https -cert and -key must be used together")
	}

	ch := make(chan error, 2)

	if *httpFlag != "" {
		go func() {
			ch <- http.ListenAndServe(*httpFlag, nil)
		}()
	}
	if *httpsFlag != "" {
		go func() {
			ch <- http.ListenAndServeTLS(*httpsFlag, *certFlag, *keyFlag, nil)
		}()
	}
	return <-ch
}

var tmpl = template.Must(template.New("").Parse(`
<html>
<head>
<meta name="go-import" content="{{.PkgRoot}} git {{.GitRoot}}">
</head>
<body>
<script>
window.location = "http://godoc.org/{{.PkgPath}}" + window.location.hash;
</script>
</body>
</html>
`))

type Repo struct {
	GitRoot string
	HubRoot string
	PkgRoot string
	PkgPath string
	Version string
}

var re = regexp.MustCompile(`^/([a-z0-9][-a-z0-9]+/)?(v0|v[1-9][0-9]*)/([a-z][-a-z0-9]*)(?:\.git)?((?:/[a-z][-a-z0-9]*)*)$`)

func handler(resp http.ResponseWriter, req *http.Request) {
	log.Printf("%s requested %s", req.RemoteAddr, req.URL)

	if req.URL.Path == "/" {
		resp.Header().Set("Location", "http://godoc.org/gopkg.in/v1/docs")
		resp.WriteHeader(http.StatusTemporaryRedirect)
		return
	}

	m := re.FindStringSubmatch(req.URL.Path)
	if m == nil {
		resp.WriteHeader(404)
		return
	}

	var repo *Repo
	if m[1] == "" {
		repo = &Repo{
			GitRoot: "https://github.com/go-" + m[3] + "/" + m[3],
			PkgRoot: "gopkg.in/" + m[2] + "/" + m[3],
			PkgPath: "gopkg.in/" + m[2] + "/" + m[3] + m[4],
			Version: m[2],
		}
	} else {
		repo = &Repo{
			GitRoot: "https://github.com/" + m[1] + m[3],
			PkgRoot: "gopkg.in/" + m[1] + m[2] + "/" + m[3],
			PkgPath: "gopkg.in/" + m[1] + m[2] + "/" + m[3] + m[4],
			Version: m[2],
		}
	}

	repo.HubRoot = repo.GitRoot

	refs, err := hackedRefs(repo)
	switch err {
	case nil:
		repo.GitRoot = "https://" + repo.PkgRoot
	case ErrNoRepo:
		repo.GitRoot += "-" + repo.Version
		repo.HubRoot += "-" + repo.Version
	case ErrNoVersion:
		log.Print(err)
		resp.WriteHeader(http.StatusNotFound)
		return
	default:
		log.Print(err)
		resp.WriteHeader(http.StatusNotFound)
		return
	}

	if m[4] == "/git-upload-pack" {
		resp.Header().Set("Location", repo.HubRoot+"/git-upload-pack")
		resp.WriteHeader(http.StatusMovedPermanently)
		return
	}

	if m[4] == "/info/refs" {
		resp.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")
		resp.Write(refs)
		return
	}

	resp.Header().Set("Content-Type", "text/html")
	tmpl.Execute(resp, repo)
}

var ErrNoRepo = errors.New("repository not found in github")
var ErrNoVersion = errors.New("version reference not found in github")

func hackedRefs(repo *Repo) (data []byte, err error) {
	resp, err := http.Get(repo.HubRoot + ".git/info/refs?service=git-upload-pack")
	if err != nil {
		return nil, fmt.Errorf("cannot talk to github: %v", err)
	}
	switch resp.StatusCode {
	case 200:
		defer resp.Body.Close()
	case 401, 404:
		return nil, ErrNoRepo
	default:
		return nil, fmt.Errorf("error from github: %v", resp.Status)
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading from github: %v", err)
	}

	vhead := "refs/heads/" + repo.Version
	vtag := "refs/tags/" + repo.Version

	var mrefi, mrefj, vrefi, vrefj int

	sdata := string(data)
	for i, j := 0, 0; i < len(data); i = j {
		size, err := strconv.ParseInt(sdata[i:i+4], 16, 32)
		if err != nil {
			return nil, fmt.Errorf("cannot parse refs line size: %s", string(data[i:i+4]))
		}
		if size == 0 {
			size = 4
		}
		j = i + int(size)
		if j > len(sdata) {
			return nil, fmt.Errorf("incomplete refs data received from github")
		}
		if sdata[0] == '#' {
			continue
		}

		hashi := i + 4
		hashj := strings.IndexByte(sdata[hashi:j], ' ')
		if hashj < 0 || hashj != 40 {
			continue
		}
		hashj += hashi

		namei := hashj + 1
		namej := strings.IndexAny(sdata[namei:j], "\n\x00")
		if namej < 0 {
			namej = j
		} else {
			namej += namei
		}

		name := sdata[namei:namej]

		if name == "refs/heads/master" {
			mrefi = hashi
			mrefj = hashj
		}

		if name == vtag || name == vhead {
			vrefi = hashi
			vrefj = hashj
		}

		if mrefi > 0 && vrefi > 0 {
			break
		}
	}

	if mrefi == 0 || vrefi == 0 {
		return nil, ErrNoVersion
	}

	copy(data[mrefi:mrefj], data[vrefi:vrefj])
	return data, nil
}
<commit_msg>Improved version matching.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

var httpFlag = flag.String("http", ":8080", "Serve HTTP at given address")
var httpsFlag = flag.String("https", "", "Serve HTTPS at given address")
var certFlag = flag.String("cert", "", "Use the provided TLS certificate")
var keyFlag = flag.String("key", "", "Use the provided TLS key")

func main() {
	if err := run(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func run() error {
	flag.Parse()
	if len(flag.Args()) > 0 {
		return fmt.Errorf("too many arguments: %s", flag.Args()[0])
	}

	http.HandleFunc("/", handler)

	if *httpFlag == "" && *httpsFlag == "" {
		return fmt.Errorf("must provide -http and/or -https")
	}
	if (*httpsFlag != "" || *certFlag != "" || *keyFlag != "") && (*httpsFlag == "" || *certFlag == "" || *keyFlag == "") {
		return fmt.Errorf("-https -cert and -key must be used together")
	}

	ch := make(chan error, 2)

	if *httpFlag != "" {
		go func() {
			ch <- http.ListenAndServe(*httpFlag, nil)
		}()
	}
	if *httpsFlag != "" {
		go func() {
			ch <- http.ListenAndServeTLS(*httpsFlag, *certFlag, *keyFlag, nil)
		}()
	}
	return <-ch
}

var tmpl = template.Must(template.New("").Parse(`
<html>
<head>
<meta name="go-import" content="{{.PkgRoot}} git {{.GitRoot}}">
</head>
<body>
<script>
window.location = "http://godoc.org/{{.PkgPath}}" + window.location.hash;
</script>
</body>
</html>
`))

type Repo struct {
	GitRoot string
	HubRoot string
	PkgRoot string
	PkgPath string
	Version Version
}

var re = regexp.MustCompile(`^/([a-z0-9][-a-z0-9]+/)?((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2})/([a-zA-Z][-a-zA-Z0-9]*)(?:\.git)?((?:/[a-zA-Z][-a-zA-Z0-9]*)*)$`)

func handler(resp http.ResponseWriter, req *http.Request) {
	if req.URL.Path == "/health-check" {
		resp.Write([]byte("ok"))
		return
	}

	log.Printf("%s requested %s", req.RemoteAddr, req.URL)

	if req.URL.Path == "/" {
		resp.Header().Set("Location", "http://godoc.org/gopkg.in/v1/docs")
		resp.WriteHeader(http.StatusTemporaryRedirect)
		return
	}

	m := re.FindStringSubmatch(req.URL.Path)
	if m == nil {
		sendNotFound(resp, "Unsupported URL pattern; see the documentation at gopkg.in for details.")
		return
	}

	if strings.Contains(m[2], ".") {
		sendNotFound(resp, "Import paths take the major version only (/%s/ instead of /%s/); see docs at gopkg.in for the reasoning.",
			m[2][:strings.Index(m[2], ".")], m[2])
		return
	}

	var repo *Repo
	if m[1] == "" {
		repo = &Repo{
			GitRoot: "https://github.com/go-" + m[3] + "/" + m[3],
			PkgRoot: "gopkg.in/" + m[2] + "/" + m[3],
			PkgPath: "gopkg.in/" + m[2] + "/" + m[3] + m[4],
		}
	} else {
		repo = &Repo{
			GitRoot: "https://github.com/" + m[1] + m[3],
			PkgRoot: "gopkg.in/" + m[1] + m[2] + "/" + m[3],
			PkgPath: "gopkg.in/" + m[1] + m[2] + "/" + m[3] + m[4],
		}
	}

	var ok bool
	repo.Version, ok = parseVersion(m[2])
	if !ok {
		sendNotFound(resp, "Version %q improperly considered invalid; please warn the service maintainers.", m[2])
	}

	repo.HubRoot = repo.GitRoot

	// Run this concurrently to avoid waiting later.
	nameVersioned := nameHasVersion(repo)

	refs, err := hackedRefs(repo)
	switch err {
	case nil:
		repo.GitRoot = "https://" + repo.PkgRoot
	case ErrNoRepo:
		if <-nameVersioned {
			v := repo.Version.String()
			repo.GitRoot += "-" + v
			repo.HubRoot += "-" + v
			break
		}
		sendNotFound(resp, "GitHub repository not found at %s or %s-%s", repo.HubRoot, repo.HubRoot, repo.Version)
		return
	case ErrNoVersion:
		v := repo.Version.String()
		if repo.Version.Minor == -1 {
			sendNotFound(resp, `GitHub repository at %s has no branch or tag "%s", "%s.N" or "%s.N.M"`, repo.HubRoot, v, v, v)
		} else if repo.Version.Patch == -1 {
			sendNotFound(resp, `GitHub repository at %s has no branch or tag "%s" or "%s.N"`, repo.HubRoot, v, v)
		} else {
			sendNotFound(resp, `GitHub repository at %s has no branch or tag "%s"`, repo.HubRoot, v)
		}
		return
	default:
		resp.WriteHeader(http.StatusBadGateway)
		resp.Write([]byte(fmt.Sprintf("Cannot obtain refs from GitHub: %v", err)))
		return
	}

	if m[4] == "/git-upload-pack" {
		resp.Header().Set("Location", repo.HubRoot+"/git-upload-pack")
		resp.WriteHeader(http.StatusMovedPermanently)
		return
	}

	if m[4] == "/info/refs" {
		resp.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")
		resp.Write(refs)
		return
	}

	resp.Header().Set("Content-Type", "text/html")
	tmpl.Execute(resp, repo)
}

func sendNotFound(resp http.ResponseWriter, msg string, args ...interface{}) {
	if len(args) > 0 {
		msg = fmt.Sprintf(msg, args...)
	}
	resp.WriteHeader(http.StatusNotFound)
	resp.Write([]byte(msg))
}

// TODO Timeouts for these http interactions. Use the new support coming in 1.3.

const refsSuffix = ".git/info/refs?service=git-upload-pack"

func nameHasVersion(repo *Repo) chan bool {
	ch := make(chan bool, 1)
	go func() {
		resp, err := http.Head(repo.HubRoot + "-" + repo.Version.String() + refsSuffix)
		if err != nil {
			ch <- false
		}
		if resp.Body != nil {
			resp.Body.Close()
		}
		ch <- resp.StatusCode == 200
	}()
	return ch
}

var ErrNoRepo = errors.New("repository not found in github")
var ErrNoVersion = errors.New("version reference not found in github")

func hackedRefs(repo *Repo) (data []byte, err error) {
	resp, err := http.Get(repo.HubRoot + refsSuffix)
	if err != nil {
		return nil, fmt.Errorf("cannot talk to GitHub: %v", err)
	}
	switch resp.StatusCode {
	case 200:
		defer resp.Body.Close()
	case 401, 404:
		return nil, ErrNoRepo
	default:
		return nil, fmt.Errorf("error from GitHub: %v", resp.Status)
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading from GitHub: %v", err)
	}

	var mrefi, mrefj int
	var vrefi, vrefj int
	var vrefv = InvalidVersion
	var unversioned = true

	sdata := string(data)
	for i, j := 0, 0; i < len(data); i = j {
		size, err := strconv.ParseInt(sdata[i:i+4], 16, 32)
		if err != nil {
			return nil, fmt.Errorf("cannot parse refs line size: %s", string(data[i:i+4]))
		}
		if size == 0 {
			size = 4
		}
		j = i + int(size)
		if j > len(sdata) {
			return nil, fmt.Errorf("incomplete refs data received from GitHub")
		}
		if sdata[0] == '#' {
			continue
		}

		hashi := i + 4
		hashj := strings.IndexByte(sdata[hashi:j], ' ')
		if hashj < 0 || hashj != 40 {
			continue
		}
		hashj += hashi

		namei := hashj + 1
		namej := strings.IndexAny(sdata[namei:j], "\n\x00")
		if namej < 0 {
			namej = j
		} else {
			namej += namei
		}

		name := sdata[namei:namej]

		if name == "refs/heads/master" {
			mrefi = hashi
			mrefj = hashj
		}

		if strings.HasPrefix(name, "refs/heads/v") || strings.HasPrefix(name, "refs/tags/v") {
			v, ok := parseVersion(name[strings.IndexByte(name, 'v'):])
			if ok && repo.Version.Contains(v) && (!vrefv.IsValid() || vrefv.Less(v)) {
				vrefv = v
				vrefi = hashi
				vrefj = hashj
			}
			if ok {
				unversioned = false
			}
		}
	}

	// If there were absolutely no versions, and v0 was requested, accept the master as-is.
	if unversioned && repo.Version == (Version{0, -1, -1}) {
		return data, nil
	}

	if mrefi == 0 || vrefi == 0 {
		return nil, ErrNoVersion
	}

	copy(data[mrefi:mrefj], data[vrefi:vrefj])
	return data, nil
}
<|endoftext|>


<commit_before>//
// Copyright 2022 The GUAC Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package assembler

type assembler struct{} //nolint: unused

// NOTE: `GuacNode` and `GuacEdge` interfaces are very experimental and might
// change in the future as we discover issues with reading/writing from the
// graph database.
//
// For now, the design of the interface follows these guideliness:
//
//   1. We want to serialize `GuacNode`s and `GuacEdge`s to graph database
//      (e.g. Neo4j) without creating duplicate nodes. To do this, we need
//      ability to uniquely identify a node. Since a node could be created from
//      different document types, it can be uniquely identified by different
//      subsets of attributes/properties. For example, we could have a node
//      that is identified by an `"id"` field from one document and by the pair
//      `"name"`, `"digest"` from another one.
//   2. Nodes can also have attributes that are not unique and are generated
//      from various documents.
//   3. In order to write the serialization/deserialization code, we need to
//      get the name of the attributes separate from the pairing between the
//      attribute and the value.
//
// In broad lines, the serialization process for a node would look like:
//
//   1. For each identifiable set in `IdentifiablePropertyNames()` check if the
//      node has values for all of the specified properties. If one is missing,
//      try the next set. If no set is left, panic.
//   2. If a set of identifiable properties is found and we have values for all
//      of these, write a query that would match on nodes which have these
//      property:value attributes. The graph database engine will allow us to
//      run separate code if a node already exists or one is newly created. In
//      our case, in both instances we will just need to set the other
//      attributes that have a value. To do this, the `Properties()` returned
//      map will be passed directly to the prepared statement (which uses
//      `Type()` to select the graph database node type and `PropertyNames()`
//      to build the rest of the query).
//
// The serialization process for an edge would be similar, with the caveat that
// an edge is always created between two existing nodes.
//
// Deserialization is left for later, with the only caveat that we might
// envision a case where we'd like to match on edges without first matching on
// their endpoints (e.g., "retrieve all attestations from this time period and
// for each of them return the artifact nodes"). Hence, we need ways to
// uniquely identify edges without having endpoint nodes.
//
// TODO(mihaimaruseac): Look into using tags of fields to automate
// serialization/deserialization, similar to how json is done.

// GuacNode represents a node in the GUAC graph
type GuacNode interface {
	// Type returns the type of node
	Type() string

	// Properties returns the list of properties of the node
	Properties() map[string]interface{}

	// PropertyNames returns the names of the properties of the node.
	//
	// If a string `s` is in the list returned by `PropertyNames` then it
	// should also be a key in the map returned by `Properties`.
	PropertyNames() []string

	// IdentifiablePropertyNames returns a list of tuples of property names
	// that can uniquely specify a GuacNode.
	//
	// Any string found in a tuple returned by `IdentifiablePropertyNames`
	// must also be returned by `PropertyNames`.
	IdentifiablePropertyNames() [][]string
}

// GuacEdge represents an edge in the GUAC graph
type GuacEdge interface {
	// Type returns the type of edge
	Type() string

	// Nodes returns the (v,u) nodes of the edge
	//
	// For directional edges: v-[edge]->u.
	// For non-directional edges there is no guaranteed order.
	Nodes() (v, u GuacNode)

	// Properties returns the list of properties of the edge
	Properties() map[string]interface{}

	// PropertyNames returns the names of the properties of the edge.
	//
	// If a string `s` is in the list returned by `PropertyNames` then it
	// should also be a key in the map returned by `Properties`.
	PropertyNames() []string

	// IdentifiablePropertyNames returns a list of tuples of property names
	// that can uniquely specify a GuacEdge, as an alternative to the two
	// node endpoints.
	//
	// Any string found in a tuple returned by `IdentifiablePropertyNames`
	// must also be returned by `PropertyNames`.
	//
	// TODO(mihaimaruseac): We might not need this?
	IdentifiablePropertyNames() [][]string
}

// Subgraph represents a subgraph read from the database or written to it.
type Subgraph struct {
	V []GuacNode
	E []GuacEdge
}

// TODO(mihaimaruseac): Write queries to write/read subgraphs from DB?

// AssemblerInput represents the inputs to add to the graph
type AssemblerInput = Subgraph
<commit_msg>Fix typo, add experimental notice<commit_after>//
// Copyright 2022 The GUAC Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package assembler

type assembler struct{} //nolint: unused

// NOTE: `GuacNode` and `GuacEdge` interfaces are very experimental and might
// change in the future as we discover issues with reading/writing from the
// graph database.
//
// For now, the design of the interface follows these guidelines:
//
//   1. We want to serialize `GuacNode`s and `GuacEdge`s to graph database
//      (e.g. Neo4j) without creating duplicate nodes. To do this, we need
//      ability to uniquely identify a node. Since a node could be created from
//      different document types, it can be uniquely identified by different
//      subsets of attributes/properties. For example, we could have a node
//      that is identified by an `"id"` field from one document and by the pair
//      `"name"`, `"digest"` from another one.
//   2. Nodes can also have attributes that are not unique and are generated
//      from various documents.
//   3. In order to write the serialization/deserialization code, we need to
//      get the name of the attributes separate from the pairing between the
//      attribute and the value.
//
// In broad lines, the serialization process for a node would look like:
//
//   1. For each identifiable set in `IdentifiablePropertyNames()` check if the
//      node has values for all of the specified properties. If one is missing,
//      try the next set. If no set is left, panic.
//   2. If a set of identifiable properties is found and we have values for all
//      of these, write a query that would match on nodes which have these
//      property:value attributes. The graph database engine will allow us to
//      run separate code if a node already exists or one is newly created. In
//      our case, in both instances we will just need to set the other
//      attributes that have a value. To do this, the `Properties()` returned
//      map will be passed directly to the prepared statement (which uses
//      `Type()` to select the graph database node type and `PropertyNames()`
//      to build the rest of the query).
//
// The serialization process for an edge would be similar, with the caveat that
// an edge is always created between two existing nodes.
//
// Deserialization is left for later, with the only caveat that we might
// envision a case where we'd like to match on edges without first matching on
// their endpoints (e.g., "retrieve all attestations from this time period and
// for each of them return the artifact nodes"). Hence, we need ways to
// uniquely identify edges without having endpoint nodes.
//
// TODO(mihaimaruseac): Look into using tags of fields to automate
// serialization/deserialization, similar to how json is done.

// GuacNode represents a node in the GUAC graph
// Note: this is experimental and might change. Please refer to source code for
// more details about usage.
type GuacNode interface {
	// Type returns the type of node
	Type() string

	// Properties returns the list of properties of the node
	Properties() map[string]interface{}

	// PropertyNames returns the names of the properties of the node.
	//
	// If a string `s` is in the list returned by `PropertyNames` then it
	// should also be a key in the map returned by `Properties`.
	PropertyNames() []string

	// IdentifiablePropertyNames returns a list of tuples of property names
	// that can uniquely specify a GuacNode.
	//
	// Any string found in a tuple returned by `IdentifiablePropertyNames`
	// must also be returned by `PropertyNames`.
	IdentifiablePropertyNames() [][]string
}

// GuacEdge represents an edge in the GUAC graph
// Note: this is experimental and might change. Please refer to source code for
// more details about usage.
type GuacEdge interface {
	// Type returns the type of edge
	Type() string

	// Nodes returns the (v,u) nodes of the edge
	//
	// For directional edges: v-[edge]->u.
	// For non-directional edges there is no guaranteed order.
	Nodes() (v, u GuacNode)

	// Properties returns the list of properties of the edge
	Properties() map[string]interface{}

	// PropertyNames returns the names of the properties of the edge.
	//
	// If a string `s` is in the list returned by `PropertyNames` then it
	// should also be a key in the map returned by `Properties`.
	PropertyNames() []string

	// IdentifiablePropertyNames returns a list of tuples of property names
	// that can uniquely specify a GuacEdge, as an alternative to the two
	// node endpoints.
	//
	// Any string found in a tuple returned by `IdentifiablePropertyNames`
	// must also be returned by `PropertyNames`.
	//
	// TODO(mihaimaruseac): We might not need this?
	IdentifiablePropertyNames() [][]string
}

// Subgraph represents a subgraph read from the database or written to it.
// Note: this is experimental and might change. Please refer to source code for
// more details about usage.
type Subgraph struct {
	V []GuacNode
	E []GuacEdge
}

// TODO(mihaimaruseac): Write queries to write/read subgraphs from DB?

// AssemblerInput represents the inputs to add to the graph
type AssemblerInput = Subgraph
<|endoftext|>