<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package route

import (
	"context"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"

	"knative.dev/pkg/kmeta"
	"knative.dev/pkg/ptr"
	"knative.dev/pkg/system"
	netv1alpha1 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	v1 "knative.dev/serving/pkg/apis/serving/v1"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	fakecertinformer "knative.dev/serving/pkg/client/injection/informers/networking/v1alpha1/certificate/fake"
	fakeciinformer "knative.dev/serving/pkg/client/injection/informers/networking/v1alpha1/ingress/fake"
	"knative.dev/serving/pkg/gc"
	"knative.dev/serving/pkg/reconciler/route/config"
	"knative.dev/serving/pkg/reconciler/route/resources"
	"knative.dev/serving/pkg/reconciler/route/traffic"

	. "knative.dev/pkg/logging/testing"
	. "knative.dev/serving/pkg/testing/v1alpha1"
)

func TestReconcileIngressInsert(t *testing.T) {
	_, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	ci := newTestIngress(t, r)

	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
}

func TestReconcileIngressUpdate(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")

	ci := newTestIngress(t, r)
	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated := getRouteIngressFromClient(ctx, t, r)
	fakeciinformer.Get(ctx).Informer().GetIndexer().Add(updated)

	ci2 := newTestIngress(t, r, func(tc *traffic.Config) {
		tc.Targets[traffic.DefaultTarget][0].TrafficTarget.Percent = ptr.Int64(50)
		tc.Targets[traffic.DefaultTarget] = append(tc.Targets[traffic.DefaultTarget], traffic.RevisionTarget{
			TrafficTarget: v1.TrafficTarget{
				Percent:      ptr.Int64(50),
				RevisionName: "revision2",
			},
		})
	})
	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci2); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated = getRouteIngressFromClient(ctx, t, r)
	if diff := cmp.Diff(ci2, updated); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %v", diff)
	}
	if diff := cmp.Diff(ci, updated); diff == "" {
		t.Error("Expected difference, but found none")
	}
}

func TestReconcileTargetRevisions(t *testing.T) {
	_, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route", WithRouteLabel(map[string]string{"route": "test-route"}))
	cases := []struct {
		name      string
		tc        traffic.Config
		expectErr error
	}{{
		name: "Valid target revision",
		tc: traffic.Config{Targets: map[string]traffic.RevisionTargets{
			traffic.DefaultTarget: {{
				TrafficTarget: v1.TrafficTarget{
					RevisionName: "revision",
					Percent:      ptr.Int64(100),
				},
				Active: true,
			}}}},
	}, {
		name: "invalid target revision",
		tc: traffic.Config{Targets: map[string]traffic.RevisionTargets{
			traffic.DefaultTarget: {{
				TrafficTarget: v1.TrafficTarget{
					RevisionName: "inal-revision",
					Percent:      ptr.Int64(100),
				},
				Active: true,
			}}}},
	}}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			ctx := TestContextWithLogger(t)
			ctx = config.ToContext(ctx, &config.Config{
				GC: &gc.Config{
					StaleRevisionLastpinnedDebounce: time.Duration(1 * time.Minute),
				},
			})
			err := reconciler.reconcileTargetRevisions(ctx, &tc.tc, r)
			if err != tc.expectErr {
				t.Fatalf("Got error = %v, want: %v", err, tc.expectErr)
			}
		})

		// TODO(greghaynes): Assert annotations correctly added
	}
}

func newTestIngress(t *testing.T, r *v1alpha1.Route, trafficOpts ...func(tc *traffic.Config)) *netv1alpha1.Ingress {
	tc := &traffic.Config{Targets: map[string]traffic.RevisionTargets{
		traffic.DefaultTarget: {{
			TrafficTarget: v1.TrafficTarget{
				RevisionName: "revision",
				Percent:      ptr.Int64(100),
			},
			Active: true,
		}}}}

	for _, opt := range trafficOpts {
		opt(tc)
	}

	tls := []netv1alpha1.IngressTLS{{
		Hosts:           []string{"test-route.test-ns.example.com"},
		SecretName:      "test-secret",
		SecretNamespace: "test-ns",
	}}
	ingress, err := resources.MakeIngress(getContext(), r, tc, tls, sets.NewString(), "foo-ingress")
	if err != nil {
		t.Errorf("Unexpected error %v", err)
	}
	return ingress
}

func TestReconcileCertificatesInsert(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	certificate := newCerts([]string{"*.default.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, certificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	created := getCertificateFromClient(t, ctx, certificate)
	if diff := cmp.Diff(certificate, created); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %s", diff)
	}
}

func TestReconcileCertificateUpdate(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	certificate := newCerts([]string{"old.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, certificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	storedCert := getCertificateFromClient(t, ctx, certificate)
	fakecertinformer.Get(ctx).Informer().GetIndexer().Add(storedCert)

	newCertificate := newCerts([]string{"new.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, newCertificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated := getCertificateFromClient(t, ctx, newCertificate)
	if diff := cmp.Diff(newCertificate, updated); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %s", diff)
	}
	if diff := cmp.Diff(certificate, updated); diff == "" {
		t.Error("Expected difference, but found none")
	}
}

func newCerts(dnsNames []string, r *v1alpha1.Route) *netv1alpha1.Certificate {
	return &netv1alpha1.Certificate{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test-cert",
			Namespace:       system.Namespace(),
			OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(r)},
		},
		Spec: netv1alpha1.CertificateSpec{
			DNSNames:   dnsNames,
			SecretName: "test-secret",
		},
	}
}

func getContext() context.Context {
	cfg := ReconcilerTestConfig(false)
	return config.ToContext(context.Background(), cfg)
}
<commit_msg>Update test for checking last pinned timestamp (#5830)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package route

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"

	"knative.dev/pkg/kmeta"
	"knative.dev/pkg/ptr"
	"knative.dev/pkg/system"
	netv1alpha1 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/apis/serving"
	v1 "knative.dev/serving/pkg/apis/serving/v1"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	fakeservingclient "knative.dev/serving/pkg/client/injection/client/fake"
	fakecertinformer "knative.dev/serving/pkg/client/injection/informers/networking/v1alpha1/certificate/fake"
	fakeciinformer "knative.dev/serving/pkg/client/injection/informers/networking/v1alpha1/ingress/fake"
	fakerevisioninformer "knative.dev/serving/pkg/client/injection/informers/serving/v1alpha1/revision/fake"
	"knative.dev/serving/pkg/gc"
	"knative.dev/serving/pkg/reconciler/route/config"
	"knative.dev/serving/pkg/reconciler/route/resources"
	"knative.dev/serving/pkg/reconciler/route/traffic"

	. "knative.dev/pkg/logging/testing"
	. "knative.dev/serving/pkg/testing/v1alpha1"
)

func TestReconcileIngressInsert(t *testing.T) {
	_, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	ci := newTestIngress(t, r)

	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
}

func TestReconcileIngressUpdate(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")

	ci := newTestIngress(t, r)
	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated := getRouteIngressFromClient(ctx, t, r)
	fakeciinformer.Get(ctx).Informer().GetIndexer().Add(updated)

	ci2 := newTestIngress(t, r, func(tc *traffic.Config) {
		tc.Targets[traffic.DefaultTarget][0].TrafficTarget.Percent = ptr.Int64(50)
		tc.Targets[traffic.DefaultTarget] = append(tc.Targets[traffic.DefaultTarget], traffic.RevisionTarget{
			TrafficTarget: v1.TrafficTarget{
				Percent:      ptr.Int64(50),
				RevisionName: "revision2",
			},
		})
	})
	if _, err := reconciler.reconcileIngress(TestContextWithLogger(t), r, ci2); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated = getRouteIngressFromClient(ctx, t, r)
	if diff := cmp.Diff(ci2, updated); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %v", diff)
	}
	if diff := cmp.Diff(ci, updated); diff == "" {
		t.Error("Expected difference, but found none")
	}
}

func TestReconcileTargetValidRevision(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route", WithRouteLabel(map[string]string{"route": "test-route"}))
	rev := newTestRevision(r.Namespace, "revision")
	tc := traffic.Config{Targets: map[string]traffic.RevisionTargets{
		traffic.DefaultTarget: {{
			TrafficTarget: v1.TrafficTarget{
				RevisionName: "revision",
				Percent:      ptr.Int64(100),
			},
			Active: true,
		}}}}

	ctx = config.ToContext(ctx, &config.Config{
		GC: &gc.Config{
			StaleRevisionLastpinnedDebounce: time.Minute,
		},
	})

	fakeservingclient.Get(ctx).ServingV1alpha1().Revisions(r.Namespace).Create(rev)
	fakerevisioninformer.Get(ctx).Informer().GetIndexer().Add(rev)

	// Get timestamp before reconciling, so that we can compare this to the last pinned timestamp
	// after reconciliation
	beforeTimestamp, err := getLastPinnedTimestamp(t, rev)
	if err != nil {
		t.Fatalf("Error getting last pinned: %v", err)
	}

	if err = reconciler.reconcileTargetRevisions(ctx, &tc, r); err != nil {
		t.Fatalf("Error reconciling target revisions: %v", err)
	}

	// Verify last pinned annotation is updated correctly
	newRev, err := fakeservingclient.Get(ctx).ServingV1alpha1().Revisions(r.Namespace).Get(rev.Name, metav1.GetOptions{})
	afterTimestamp, err := getLastPinnedTimestamp(t, newRev)
	if err != nil {
		t.Fatalf("Error getting last pinned timestamps: %v", err)
	}
	if beforeTimestamp == afterTimestamp {
		t.Fatal("The last pinned timestamp is not updated")
	}
}

func TestReconcileRevisionTargetDoesNotExist(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route", WithRouteLabel(map[string]string{"route": "test-route"}))
	rev := newTestRevision(r.Namespace, "revision")
	tcInvalidRev := traffic.Config{Targets: map[string]traffic.RevisionTargets{
		traffic.DefaultTarget: {{
			TrafficTarget: v1.TrafficTarget{
				RevisionName: "invalid-revision",
				Percent:      ptr.Int64(100),
			},
			Active: true,
		}}}}
	ctx = config.ToContext(ctx, &config.Config{
		GC: &gc.Config{
			StaleRevisionLastpinnedDebounce: time.Minute,
		},
	})
	fakeservingclient.Get(ctx).ServingV1alpha1().Revisions(r.Namespace).Create(rev)
	fakerevisioninformer.Get(ctx).Informer().GetIndexer().Add(rev)

	// Try reconciling target revisions for a revision that does not exist. No err should be returned
	if err := reconciler.reconcileTargetRevisions(ctx, &tcInvalidRev, r); err != nil {
		t.Fatalf("Error reconciling target revisions: %v", err)
	}
}

func newTestRevision(namespace string, name string) *v1alpha1.Revision {
	return &v1alpha1.Revision{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
			Annotations: map[string]string{
				serving.RevisionLastPinnedAnnotationKey: v1alpha1.RevisionLastPinnedString(time.Now().Add(-1 * time.Hour)),
			},
		},
		Spec: v1alpha1.RevisionSpec{},
	}
}

func getLastPinnedTimestamp(t *testing.T, rev *v1alpha1.Revision) (string, error) {
	lastPinnedTime, ok := rev.ObjectMeta.Annotations[serving.RevisionLastPinnedAnnotationKey]
	if !ok {
		return "", errors.New("last pinned annotation not found")
	}
	return lastPinnedTime, nil
}

func newTestIngress(t *testing.T, r *v1alpha1.Route, trafficOpts ...func(tc *traffic.Config)) *netv1alpha1.Ingress {
	tc := &traffic.Config{Targets: map[string]traffic.RevisionTargets{
		traffic.DefaultTarget: {{
			TrafficTarget: v1.TrafficTarget{
				RevisionName: "revision",
				Percent:      ptr.Int64(100),
			},
			Active: true,
		}}}}

	for _, opt := range trafficOpts {
		opt(tc)
	}

	tls := []netv1alpha1.IngressTLS{{
		Hosts:           []string{"test-route.test-ns.example.com"},
		SecretName:      "test-secret",
		SecretNamespace: "test-ns",
	}}
	ingress, err := resources.MakeIngress(getContext(), r, tc, tls, sets.NewString(), "foo-ingress")
	if err != nil {
		t.Errorf("Unexpected error %v", err)
	}
	return ingress
}

func TestReconcileCertificatesInsert(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	certificate := newCerts([]string{"*.default.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, certificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	created := getCertificateFromClient(t, ctx, certificate)
	if diff := cmp.Diff(certificate, created); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %s", diff)
	}
}

func TestReconcileCertificateUpdate(t *testing.T) {
	ctx, _, reconciler, _, cancel := newTestReconciler(t)
	defer cancel()

	r := Route("test-ns", "test-route")
	certificate := newCerts([]string{"old.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, certificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	storedCert := getCertificateFromClient(t, ctx, certificate)
	fakecertinformer.Get(ctx).Informer().GetIndexer().Add(storedCert)

	newCertificate := newCerts([]string{"new.example.com"}, r)
	if _, err := reconciler.reconcileCertificate(TestContextWithLogger(t), r, newCertificate); err != nil {
		t.Errorf("Unexpected error: %v", err)
	}

	updated := getCertificateFromClient(t, ctx, newCertificate)
	if diff := cmp.Diff(newCertificate, updated); diff != "" {
		t.Errorf("Unexpected diff (-want +got): %s", diff)
	}
	if diff := cmp.Diff(certificate, updated); diff == "" {
		t.Error("Expected difference, but found none")
	}
}

func newCerts(dnsNames []string, r *v1alpha1.Route) *netv1alpha1.Certificate {
	return &netv1alpha1.Certificate{
		ObjectMeta: metav1.ObjectMeta{
			Name:            "test-cert",
			Namespace:       system.Namespace(),
			OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(r)},
		},
		Spec: netv1alpha1.CertificateSpec{
			DNSNames:   dnsNames,
			SecretName: "test-secret",
		},
	}
}

func getContext() context.Context {
	cfg := ReconcilerTestConfig(false)
	return config.ToContext(context.Background(), cfg)
}
<|endoftext|>


<commit_before>package cpp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type lexerState struct {
	pos  FilePos
	brdr *bufio.Reader
	//At the beginning on line not including whitespace
	bol bool
	//If this channel recieves a value, the lexing goroutines should close
	//its output channel and its error channel and terminate.
	cancel chan struct{}
	stream chan *Token
}

type breakout struct {
}

//Lex starts a goroutine which lexes the contents of the reader.
//fname is used for error messages when showing the source location.
//No preprocessing is done, this is just pure reading of the unprocessed
//source file.
//The goroutine will not stop until all tokens are read
func Lex(fname string, r io.Reader) chan *Token {
	ls := new(lexerState)
	ls.pos.File = fname
	ls.pos.Line = 1
	ls.pos.Col = 1
	ls.stream = make(chan *Token)
	ls.brdr = bufio.NewReader(r)
	ls.bol = true
	go ls.lex()
	return ls.stream
}

func (ls *lexerState) lexError(e error) {
	eWithPos := fmt.Sprintf("Error while reading %s. %s", ls.pos, e.Error())
	ls.sendTok(ERROR, eWithPos)
	close(ls.stream)
	//recover exits the lexer cleanly
	panic(&breakout{})
}

func (ls *lexerState) lex() {

	//This recovery happens if lexError is called.
	defer func() {
		//XXX is this correct way to retrigger non breakout?
		if e := recover(); e != nil {
			_ = e.(breakout) // Will re-panic if not a parse error.
		}

	}()

	first, _, err := ls.brdr.ReadRune()

	for {
		if err == io.EOF {
			break
		}
		if err != nil {
			ls.lexError(err)
		}
		switch {
		case isAlpha(first) || first == '_':
			ls.brdr.UnreadRune()
			ls.readIdentOrKeyword()
		case isNumeric(first):
			ls.brdr.UnreadRune()
			ls.readConstantInt()
		case isWhiteSpace(first):
			ls.brdr.UnreadRune()
			ls.skipWhiteSpace()
		default:
			switch first {
			case '#':
				ls.readDirective()
			case '!':
				ls.sendTok(NOT, "!")
			case '?':
				ls.sendTok('?', "?")
			case ':':
				ls.sendTok(COLON, ":")
			case '"':
				ls.brdr.UnreadRune()
				ls.readCString()
			case '(':
				ls.sendTok(LPAREN, "(")
			case ')':
				ls.sendTok(RPAREN, ")")
			case '{':
				ls.sendTok(LBRACE, "{")
			case '}':
				ls.sendTok(RBRACE, "}")
			case '[':
				ls.sendTok(LBRACK, "[")
			case ']':
				ls.sendTok(RBRACE, "]")
			case '<':
				ls.sendTok(LSS, "<")
			case '>':
				ls.sendTok(GTR, ">")
			case '+':
				second, _, _ := ls.brdr.ReadRune()
				if second == '+' {
					ls.sendTok(INC, "++")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ADD, "+")
			case '.':
				ls.sendTok(PERIOD, ".")
			case '-':
				second, _, _ := ls.brdr.ReadRune()
				if second == '>' {
					ls.sendTok(ARROW, "->")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(SUB, "-")
			case ',':
				ls.sendTok(COMMA, ",")
			case '*':
				ls.sendTok(MUL, "*")
			case '/':
				second, _, _ := ls.brdr.ReadRune()
				if second == '*' { // C comment.
					for {
						endChar, _, err := ls.brdr.ReadRune()
						if err == io.EOF {
							ls.lexError(fmt.Errorf("Unclosed comment."))
						}
						if err != nil {
							ls.lexError(err)
						}
						if endChar == '*' {
							closeBar, _, _ := ls.brdr.ReadRune()
							if closeBar == '/' {
								break
							}
						}
					}
				} else if second == '/' { // C++ comment.
					for {
						c, _, err := ls.brdr.ReadRune()
						if err != nil {
							break
						}
						if c == '\n' {
							break
						}
					}
				} else {
					ls.sendTok(QUO, "/")
				}
			case '|':
				second, _, _ := ls.brdr.ReadRune()
				if second == '|' {
					ls.sendTok(LOR, "||")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(OR, "|")
			case '&':
				second, _, _ := ls.brdr.ReadRune()
				if second == '&' {
					ls.sendTok(LAND, "&&")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(AND, "&")
			case '=':
				second, _, _ := ls.brdr.ReadRune()
				if second == '=' {
					ls.sendTok(EQL, "==")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ASSIGN, "=")
			case ';':
				ls.sendTok(SEMICOLON, ";")
			default:
				//ls.lexError(fmt.Errorf("Internal Error - bad char code '%d'", first))
			}
		}
		first, _, err = ls.brdr.ReadRune()
	}
	close(ls.stream)
}

func (ls *lexerState) sendTok(kind TokenKind, val string) {
	var tok Token
	tok.Kind = kind
	tok.Val = val
	tok.Pos.Line = ls.pos.Line
	tok.Pos.Col = ls.pos.Col
	tok.Pos.File = ls.pos.File
	ls.bol = false
	ls.stream <- &tok
	ls.pos.Col += len(val)
}

func (ls *lexerState) readDirective() {

	if !ls.isAtLineStart() {
		ls.lexError(fmt.Errorf("Directive not at beginning of line."))
	}

	//Skip
	for {
		ws, _, err := ls.brdr.ReadRune()
		if err != nil {
			ls.lexError(err)
		}

		if ws == ' ' || ws == '\t' {
			continue
		}

		if ws == '\n' {
			ls.lexError(fmt.Errorf("Empty directive"))
		}

		break
	}

}

func (ls *lexerState) readIdentOrKeyword() {
	var buff bytes.Buffer
	first, _, _ := ls.brdr.ReadRune()
	if !isValidIdentStart(first) {
		panic("internal error")
	}
	buff.WriteRune(first)
	for {
		b, _, err := ls.brdr.ReadRune()
		if isAlphaNumeric(b) || b == '_' {
			buff.WriteRune(b)
		} else {
			if err == nil {
				ls.brdr.UnreadByte()
			}
			str := buff.String()
			tokType, ok := keywordLUT[str]
			if !ok {
				tokType = IDENT
			}

			ls.sendTok(tokType, str)
			break
		}
	}
}

func (ls *lexerState) skipWhiteSpace() {
	for {
		r, _, err := ls.brdr.ReadRune()
		if !isWhiteSpace(r) {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			break
		}
		if r == '\n' {
			ls.pos.Line += 1
			ls.pos.Col = 1
		} else {
			ls.pos.Col += 1
		}
	}
}

func (ls *lexerState) readConstantInt() {
	var buff bytes.Buffer
	for {
		r, _, err := ls.brdr.ReadRune()
		if isNumeric(r) {
			buff.WriteRune(r)
		} else {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			str := buff.String()
			ls.sendTok(INT_CONSTANT, str)
			break
		}
	}
}

func (ls *lexerState) readCString() {
	var buff bytes.Buffer
	first, _, err := ls.brdr.ReadRune()
	if err != nil {
		ls.lexError(err)
	}

	if first != '"' {
		ls.lexError(fmt.Errorf("internal error"))
	}
	buff.WriteRune('"')

	escaped := false
	for {
		b, _, err := ls.brdr.ReadRune()
		if err == io.EOF {
			ls.lexError(fmt.Errorf("Unterminated string literal."))
		}
		if err != nil {
			ls.lexError(err)
		}
		if b == '"' && !escaped {
			break
		}
		if !escaped {
			if b == '\\' {
				escaped = true
			}
		} else {
			escaped = false
		}
	}
	ls.sendTok(STRING, "")
}

func (ls *lexerState) isAtLineStart() bool {
	return ls.bol
}

func isValidIdentStart(b rune) bool {
	return b == '_' || isAlpha(b)
}

func isAlpha(b rune) bool {
	if b >= 'a' && b <= 'z' {
		return true
	}
	if b >= 'A' && b <= 'Z' {
		return true
	}
	return false
}

func isWhiteSpace(b rune) bool {
	return b == ' ' || b == '\r' || b == '\n' || b == '\t'
}

func isNumeric(b rune) bool {
	if b >= '0' && b <= '9' {
		return true
	}
	return false
}

func isAlphaNumeric(b rune) bool {
	return isNumeric(b) || isAlpha(b)
}
<commit_msg>more work on header lexing<commit_after>package cpp

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type lexerState struct {
	pos  FilePos
	brdr *bufio.Reader
	//At the beginning on line not including whitespace
	bol bool
	//If this channel recieves a value, the lexing goroutines should close
	//its output channel and its error channel and terminate.
	cancel chan struct{}
	stream chan *Token
}

type breakout struct {
}

//Lex starts a goroutine which lexes the contents of the reader.
//fname is used for error messages when showing the source location.
//No preprocessing is done, this is just pure reading of the unprocessed
//source file.
//The goroutine will not stop until all tokens are read
func Lex(fname string, r io.Reader) chan *Token {
	ls := new(lexerState)
	ls.pos.File = fname
	ls.pos.Line = 1
	ls.pos.Col = 1
	ls.stream = make(chan *Token)
	ls.brdr = bufio.NewReader(r)
	ls.bol = true
	go ls.lex()
	return ls.stream
}

func (ls *lexerState) lexError(e string) {
	eWithPos := fmt.Sprintf("Error while reading %s. %s", ls.pos, e)
	ls.sendTok(ERROR, eWithPos)
	close(ls.stream)
	//recover exits the lexer cleanly
	panic(&breakout{})
}

func (ls *lexerState) lex() {

	//This recovery happens if lexError is called.
	defer func() {
		//XXX is this correct way to retrigger non breakout?
		if e := recover(); e != nil {
			_ = e.(breakout) // Will re-panic if not a parse error.
		}

	}()

	first, _, err := ls.brdr.ReadRune()

	for {
		if err == io.EOF {
			break
		}
		if err != nil {
			ls.lexError(err.Error())
		}
		switch {
		case isAlpha(first) || first == '_':
			ls.brdr.UnreadRune()
			ls.readIdentOrKeyword()
		case isNumeric(first):
			ls.brdr.UnreadRune()
			ls.readConstantInt()
		case isWhiteSpace(first):
			ls.brdr.UnreadRune()
			ls.skipWhiteSpace()
		default:
			switch first {
			case '#':
				ls.readDirective()
			case '!':
				ls.sendTok(NOT, "!")
			case '?':
				ls.sendTok('?', "?")
			case ':':
				ls.sendTok(COLON, ":")
			case '"':
				ls.brdr.UnreadRune()
				ls.readCString()
			case '(':
				ls.sendTok(LPAREN, "(")
			case ')':
				ls.sendTok(RPAREN, ")")
			case '{':
				ls.sendTok(LBRACE, "{")
			case '}':
				ls.sendTok(RBRACE, "}")
			case '[':
				ls.sendTok(LBRACK, "[")
			case ']':
				ls.sendTok(RBRACE, "]")
			case '<':
				ls.sendTok(LSS, "<")
			case '>':
				ls.sendTok(GTR, ">")
			case '+':
				second, _, _ := ls.brdr.ReadRune()
				if second == '+' {
					ls.sendTok(INC, "++")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ADD, "+")
			case '.':
				ls.sendTok(PERIOD, ".")
			case '-':
				second, _, _ := ls.brdr.ReadRune()
				if second == '>' {
					ls.sendTok(ARROW, "->")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(SUB, "-")
			case ',':
				ls.sendTok(COMMA, ",")
			case '*':
				ls.sendTok(MUL, "*")
			case '/':
				second, _, _ := ls.brdr.ReadRune()
				if second == '*' { // C comment.
					for {
						endChar, _, err := ls.brdr.ReadRune()
						if err == io.EOF {
							ls.lexError("unclosed comment.")
						}
						if err != nil {
							ls.lexError(err.Error())
						}
						if endChar == '*' {
							closeBar, _, _ := ls.brdr.ReadRune()
							if closeBar == '/' {
								break
							}
						}
					}
				} else if second == '/' { // C++ comment.
					for {
						c, _, err := ls.brdr.ReadRune()
						if err != nil {
							break
						}
						if c == '\n' {
							break
						}
					}
				} else {
					ls.sendTok(QUO, "/")
				}
			case '|':
				second, _, _ := ls.brdr.ReadRune()
				if second == '|' {
					ls.sendTok(LOR, "||")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(OR, "|")
			case '&':
				second, _, _ := ls.brdr.ReadRune()
				if second == '&' {
					ls.sendTok(LAND, "&&")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(AND, "&")
			case '=':
				second, _, _ := ls.brdr.ReadRune()
				if second == '=' {
					ls.sendTok(EQL, "==")
					break
				}
				ls.brdr.UnreadRune()
				ls.sendTok(ASSIGN, "=")
			case ';':
				ls.sendTok(SEMICOLON, ";")
			default:
				//ls.lexError(fmt.Errorf("Internal Error - bad char code '%d'", first))
			}
		}
		first, _, err = ls.brdr.ReadRune()
	}
	close(ls.stream)
}

func (ls *lexerState) sendTok(kind TokenKind, val string) {
	var tok Token
	tok.Kind = kind
	tok.Val = val
	tok.Pos.Line = ls.pos.Line
	tok.Pos.Col = ls.pos.Col
	tok.Pos.File = ls.pos.File
	ls.bol = false
	ls.stream <- &tok
	ls.pos.Col += len(val)
}

func (ls *lexerState) readDirective() {

	if !ls.isAtLineStart() {
		ls.lexError("Directive not at beginning of line.")
	}

	directiveLine := ls.pos.Line
	ls.skipWhiteSpace()
	if ls.pos.Line != directiveLine {
		ls.lexError("Empty directive")
	}
	var buff bytes.Buffer
	directiveChar, _, err := ls.brdr.ReadRune()
	if err != nil && err != io.EOF {
		ls.lexError(fmt.Sprintf("Error reading directive %s", err.Error()))
	}
	if isAlpha(directiveChar) {
		for isAlpha(directiveChar) {
			buff.WriteRune(directiveChar)
			directiveChar, _, err = ls.brdr.ReadRune()
		}
		directive := buff.String()
		ls.brdr.UnreadRune()
		if directive == "include" {
			ls.readHeaderInclude()
		}
	} else {
		//wasnt a directive, error will be caught by
		//cpp or parser.
		ls.brdr.UnreadRune()
	}

}

func (ls *lexerState) readHeaderInclude() {
	var buff bytes.Buffer
	line := ls.pos.Line
	ls.skipWhiteSpace()
	if ls.pos.Line != line {
		ls.lexError("No header after include.")
	}
	opening, _, _ := ls.brdr.ReadRune()
	var terminator rune
	if opening == '"' {
		terminator = '"'
	} else if opening == '<' {
		terminator = '>'
	} else {
		ls.lexError("bad start to header include.")
	}

	for {
		c, _, err := ls.brdr.ReadRune()
		if err == io.EOF {
			ls.lexError("EOF encountered in header include.")
		}
		if err != nil {
			ls.lexError(err.Error())
		}

		if c == '\n' {
			ls.lexError("new line in header include.")
		}
		if c == terminator {
			break
		}
		buff.WriteRune(c)
	}

}

func (ls *lexerState) readIdentOrKeyword() {
	var buff bytes.Buffer
	first, _, _ := ls.brdr.ReadRune()
	if !isValidIdentStart(first) {
		panic("internal error")
	}
	buff.WriteRune(first)
	for {
		b, _, err := ls.brdr.ReadRune()
		if isAlphaNumeric(b) || b == '_' {
			buff.WriteRune(b)
		} else {
			if err == nil {
				ls.brdr.UnreadByte()
			}
			str := buff.String()
			tokType, ok := keywordLUT[str]
			if !ok {
				tokType = IDENT
			}

			ls.sendTok(tokType, str)
			break
		}
	}
}

func (ls *lexerState) skipWhiteSpace() {
	for {
		r, _, err := ls.brdr.ReadRune()
		if !isWhiteSpace(r) {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			break
		}
		if r == '\n' {
			ls.pos.Line += 1
			ls.pos.Col = 1
		} else {
			ls.pos.Col += 1
		}
	}
}

func (ls *lexerState) readConstantInt() {
	var buff bytes.Buffer
	for {
		r, _, err := ls.brdr.ReadRune()
		if isNumeric(r) {
			buff.WriteRune(r)
		} else {
			if err == nil {
				ls.brdr.UnreadRune()
			}
			str := buff.String()
			ls.sendTok(INT_CONSTANT, str)
			break
		}
	}
}

func (ls *lexerState) readCString() {
	var buff bytes.Buffer
	first, _, err := ls.brdr.ReadRune()
	if err != nil {
		ls.lexError(err.Error())
	}

	if first != '"' {
		ls.lexError("internal error")
	}
	buff.WriteRune('"')

	escaped := false
	for {
		b, _, err := ls.brdr.ReadRune()
		if err == io.EOF {
			ls.lexError("Unterminated string literal.")
		}
		if err != nil {
			ls.lexError(err.Error())
		}
		if b == '"' && !escaped {
			break
		}
		if !escaped {
			if b == '\\' {
				escaped = true
			}
		} else {
			escaped = false
		}
	}
	ls.sendTok(STRING, "")
}

func (ls *lexerState) isAtLineStart() bool {
	return ls.bol
}

func isValidIdentStart(b rune) bool {
	return b == '_' || isAlpha(b)
}

func isAlpha(b rune) bool {
	if b >= 'a' && b <= 'z' {
		return true
	}
	if b >= 'A' && b <= 'Z' {
		return true
	}
	return false
}

func isWhiteSpace(b rune) bool {
	return b == ' ' || b == '\r' || b == '\n' || b == '\t'
}

func isNumeric(b rune) bool {
	if b >= '0' && b <= '9' {
		return true
	}
	return false
}

func isAlphaNumeric(b rune) bool {
	return isNumeric(b) || isAlpha(b)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x86_test

import (
	"bytes"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

const asmData = `
GLOBL zeros<>(SB),8,$64
TEXT ·testASM(SB),4,$0
VMOVDQU zeros<>(SB), Y8 // PC relative relocation is off by 1, for Y8-15
RET
`

const goData = `
package main

func testASM()

func main() {
	testASM()
}
`

func objdumpOutput(t *testing.T) []byte {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	tmpdir, err := ioutil.TempDir("", "19518")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)
	tmpfile, err := os.Create(filepath.Join(tmpdir, "input.s"))
	if err != nil {
		t.Fatal(err)
	}
	defer tmpfile.Close()
	_, err = tmpfile.WriteString(asmData)
	if err != nil {
		t.Fatal(err)
	}
	tmpfile2, err := os.Create(filepath.Join(tmpdir, "input.go"))
	if err != nil {
		t.Fatal(err)
	}
	defer tmpfile2.Close()
	_, err = tmpfile2.WriteString(goData)
	if err != nil {
		t.Fatal(err)
	}
	err = os.Chdir(tmpdir)
	if err != nil {
		t.Fatal(err)
	}
	cmd := exec.Command(
		testenv.GoToolPath(t), "build", "-o",
		filepath.Join(tmpdir, "output"))

	cmd.Env = append(os.Environ(), "GOARCH=amd64", "GOOS=linux")

	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("error %s output %s", err, out)
	}
	cmd2 := exec.Command(
		testenv.GoToolPath(t), "tool", "objdump", "-s", "testASM",
		filepath.Join(tmpdir, "output"))
	cmd2.Env = cmd.Env
	objout, err := cmd2.CombinedOutput()
	if err != nil {
		t.Fatalf("error %s output %s", err, objout)
	}
	err = os.Chdir(cwd)
	if err != nil {
		t.Fatal(err)
	}
	return objout
}

func TestVexPCrelative(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	objout := objdumpOutput(t)
	data := bytes.Split(objout, []byte("\n"))
	for idx := len(data) - 1; idx >= 0; idx-- {
		// OBJDUMP doesn't know about VMOVDQU,
		// so instead of checking that it was assembled correctly,
		// check that RET wasn't overwritten.
		if bytes.Index(data[idx], []byte("RET")) != -1 {
			return
		}
	}
	t.Fatal("RET was overwritten")
}
<commit_msg>cmd/internal/obj/x86: fix issue19518_test in module mode<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x86_test

import (
	"bytes"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

const asmData = `
GLOBL zeros<>(SB),8,$64
TEXT ·testASM(SB),4,$0
VMOVDQU zeros<>(SB), Y8 // PC relative relocation is off by 1, for Y8-15
RET
`

const goData = `
package main

func testASM()

func main() {
	testASM()
}
`

func objdumpOutput(t *testing.T) []byte {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	tmpdir, err := ioutil.TempDir("", "19518")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)
	err = ioutil.WriteFile(filepath.Join(tmpdir, "go.mod"), []byte("module issue19518\n"), 0666)
	if err != nil {
		t.Fatal(err)
	}
	tmpfile, err := os.Create(filepath.Join(tmpdir, "input.s"))
	if err != nil {
		t.Fatal(err)
	}
	defer tmpfile.Close()
	_, err = tmpfile.WriteString(asmData)
	if err != nil {
		t.Fatal(err)
	}
	tmpfile2, err := os.Create(filepath.Join(tmpdir, "input.go"))
	if err != nil {
		t.Fatal(err)
	}
	defer tmpfile2.Close()
	_, err = tmpfile2.WriteString(goData)
	if err != nil {
		t.Fatal(err)
	}
	err = os.Chdir(tmpdir)
	if err != nil {
		t.Fatal(err)
	}
	cmd := exec.Command(
		testenv.GoToolPath(t), "build", "-o",
		filepath.Join(tmpdir, "output"))

	cmd.Env = append(os.Environ(), "GOARCH=amd64", "GOOS=linux")

	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("error %s output %s", err, out)
	}
	cmd2 := exec.Command(
		testenv.GoToolPath(t), "tool", "objdump", "-s", "testASM",
		filepath.Join(tmpdir, "output"))
	cmd2.Env = cmd.Env
	objout, err := cmd2.CombinedOutput()
	if err != nil {
		t.Fatalf("error %s output %s", err, objout)
	}
	err = os.Chdir(cwd)
	if err != nil {
		t.Fatal(err)
	}
	return objout
}

func TestVexPCrelative(t *testing.T) {
	testenv.MustHaveGoBuild(t)
	objout := objdumpOutput(t)
	data := bytes.Split(objout, []byte("\n"))
	for idx := len(data) - 1; idx >= 0; idx-- {
		// OBJDUMP doesn't know about VMOVDQU,
		// so instead of checking that it was assembled correctly,
		// check that RET wasn't overwritten.
		if bytes.Index(data[idx], []byte("RET")) != -1 {
			return
		}
	}
	t.Fatal("RET was overwritten")
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package commands defines and manages the basic pprof commands
package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"cmd/pprof/internal/plugin"
	"cmd/pprof/internal/report"
	"cmd/pprof/internal/svg"
	"cmd/pprof/internal/tempfile"
)

// Commands describes the commands accepted by pprof.
type Commands map[string]*Command

// Command describes the actions for a pprof command. Includes a
// function for command-line completion, the report format to use
// during report generation, any postprocessing functions, and whether
// the command expects a regexp parameter (typically a function name).
type Command struct {
	Complete    Completer     // autocomplete for interactive mode
	Format      int           // report format to generate
	PostProcess PostProcessor // postprocessing to run on report
	HasParam    bool          // Collect a parameter from the CLI
	Usage       string        // Help text
}

// Completer is a function for command-line autocompletion
type Completer func(prefix string) string

// PostProcessor is a function that applies post-processing to the report output
type PostProcessor func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error

// PProf returns the basic pprof report-generation commands
func PProf(c Completer, interactive **bool, svgpan **string) Commands {
	return Commands{
		// Commands that require no post-processing.
		"tags":   {nil, report.Tags, nil, false, "Outputs all tags in the profile"},
		"raw":    {c, report.Raw, nil, false, "Outputs a text representation of the raw profile"},
		"dot":    {c, report.Dot, nil, false, "Outputs a graph in DOT format"},
		"top":    {c, report.Text, nil, false, "Outputs top entries in text form"},
		"tree":   {c, report.Tree, nil, false, "Outputs a text rendering of call graph"},
		"text":   {c, report.Text, nil, false, "Outputs top entries in text form"},
		"disasm": {c, report.Dis, nil, true, "Output annotated assembly for functions matching regexp or address"},
		"list":   {c, report.List, nil, true, "Output annotated source for functions matching regexp"},
		"peek":   {c, report.Tree, nil, true, "Output callers/callees of functions matching regexp"},

		// Save binary formats to a file
		"callgrind": {c, report.Callgrind, awayFromTTY("callgraph.out"), false, "Outputs a graph in callgrind format"},
		"proto":     {c, report.Proto, awayFromTTY("pb.gz"), false, "Outputs the profile in compressed protobuf format"},

		// Generate report in DOT format and postprocess with dot
		"gif": {c, report.Dot, invokeDot("gif"), false, "Outputs a graph image in GIF format"},
		"pdf": {c, report.Dot, invokeDot("pdf"), false, "Outputs a graph in PDF format"},
		"png": {c, report.Dot, invokeDot("png"), false, "Outputs a graph image in PNG format"},
		"ps":  {c, report.Dot, invokeDot("ps"), false, "Outputs a graph in PS format"},

		// Save SVG output into a file after including svgpan library
		"svg": {c, report.Dot, saveSVGToFile(svgpan), false, "Outputs a graph in SVG format"},

		// Visualize postprocessed dot output
		"eog":    {c, report.Dot, invokeVisualizer(interactive, invokeDot("svg"), "svg", []string{"eog"}), false, "Visualize graph through eog"},
		"evince": {c, report.Dot, invokeVisualizer(interactive, invokeDot("pdf"), "pdf", []string{"evince"}), false, "Visualize graph through evince"},
		"gv":     {c, report.Dot, invokeVisualizer(interactive, invokeDot("ps"), "ps", []string{"gv --noantialias"}), false, "Visualize graph through gv"},
		"web":    {c, report.Dot, invokeVisualizer(interactive, saveSVGToFile(svgpan), "svg", browsers), false, "Visualize graph through web browser"},

		// Visualize HTML directly generated by report.
		"weblist": {c, report.WebList, invokeVisualizer(interactive, awayFromTTY("html"), "html", browsers), true, "Output annotated source in HTML for functions matching regexp or address"},
	}
}

// List of web browsers to attempt for web visualization
var browsers = []string{"chrome", "google-chrome", "firefox", "/usr/bin/open"}

// NewCompleter creates an autocompletion function for a set of commands.
func NewCompleter(cs Commands) Completer {
	return func(line string) string {
		switch tokens := strings.Fields(line); len(tokens) {
		case 0:
			// Nothing to complete
		case 1:
			// Single token -- complete command name
			found := ""
			for c := range cs {
				if strings.HasPrefix(c, tokens[0]) {
					if found != "" {
						return line
					}
					found = c
				}
			}
			if found != "" {
				return found
			}
		default:
			// Multiple tokens -- complete using command completer
			if c, ok := cs[tokens[0]]; ok {
				if c.Complete != nil {
					lastTokenIdx := len(tokens) - 1
					lastToken := tokens[lastTokenIdx]
					if strings.HasPrefix(lastToken, "-") {
						lastToken = "-" + c.Complete(lastToken[1:])
					} else {
						lastToken = c.Complete(lastToken)
					}
					return strings.Join(append(tokens[:lastTokenIdx], lastToken), " ")
				}
			}
		}
		return line
	}
}

// awayFromTTY saves the output in a file if it would otherwise go to
// the terminal screen. This is used to avoid dumping binary data on
// the screen.
func awayFromTTY(format string) PostProcessor {
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		if output == os.Stdout && ui.IsTerminal() {
			tempFile, err := tempfile.New("", "profile", "."+format)
			if err != nil {
				return err
			}
			ui.PrintErr("Generating report in ", tempFile.Name())
			_, err = fmt.Fprint(tempFile, input)
			return err
		}
		_, err := fmt.Fprint(output, input)
		return err
	}
}

func invokeDot(format string) PostProcessor {
	divert := awayFromTTY(format)
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		cmd := exec.Command("dot", "-T"+format)
		var buf bytes.Buffer
		cmd.Stdin, cmd.Stdout, cmd.Stderr = input, &buf, os.Stderr
		if err := cmd.Run(); err != nil {
			return err
		}
		return divert(&buf, output, ui)
	}
}

func saveSVGToFile(svgpan **string) PostProcessor {
	generateSVG := invokeDot("svg")
	divert := awayFromTTY("svg")
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		baseSVG := &bytes.Buffer{}
		generateSVG(input, baseSVG, ui)
		massaged := &bytes.Buffer{}
		fmt.Fprint(massaged, svg.Massage(*baseSVG, **svgpan))
		return divert(massaged, output, ui)
	}
}

func invokeVisualizer(interactive **bool, format PostProcessor, suffix string, visualizers []string) PostProcessor {
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		tempFile, err := tempfile.New(os.Getenv("PPROF_TMPDIR"), "pprof", "."+suffix)
		if err != nil {
			return err
		}
		tempfile.DeferDelete(tempFile.Name())
		if err = format(input, tempFile, ui); err != nil {
			return err
		}
		// Try visualizers until one is successful
		for _, v := range visualizers {
			// Separate command and arguments for exec.Command.
			args := strings.Split(v, " ")
			if len(args) == 0 {
				continue
			}
			viewer := exec.Command(args[0], append(args[1:], tempFile.Name())...)
			viewer.Stderr = os.Stderr
			if err = viewer.Start(); err == nil {
				if !**interactive {
					// In command-line mode, wait for the viewer to be closed
					// before proceeding
					return viewer.Wait()
				}
				return nil
			}
		}
		return err
	}
}
<commit_msg>cmd/pprof/internal/commands: add command to open browser on windows<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package commands defines and manages the basic pprof commands
package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"cmd/pprof/internal/plugin"
	"cmd/pprof/internal/report"
	"cmd/pprof/internal/svg"
	"cmd/pprof/internal/tempfile"
)

// Commands describes the commands accepted by pprof.
type Commands map[string]*Command

// Command describes the actions for a pprof command. Includes a
// function for command-line completion, the report format to use
// during report generation, any postprocessing functions, and whether
// the command expects a regexp parameter (typically a function name).
type Command struct {
	Complete    Completer     // autocomplete for interactive mode
	Format      int           // report format to generate
	PostProcess PostProcessor // postprocessing to run on report
	HasParam    bool          // Collect a parameter from the CLI
	Usage       string        // Help text
}

// Completer is a function for command-line autocompletion
type Completer func(prefix string) string

// PostProcessor is a function that applies post-processing to the report output
type PostProcessor func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error

// PProf returns the basic pprof report-generation commands
func PProf(c Completer, interactive **bool, svgpan **string) Commands {
	return Commands{
		// Commands that require no post-processing.
		"tags":   {nil, report.Tags, nil, false, "Outputs all tags in the profile"},
		"raw":    {c, report.Raw, nil, false, "Outputs a text representation of the raw profile"},
		"dot":    {c, report.Dot, nil, false, "Outputs a graph in DOT format"},
		"top":    {c, report.Text, nil, false, "Outputs top entries in text form"},
		"tree":   {c, report.Tree, nil, false, "Outputs a text rendering of call graph"},
		"text":   {c, report.Text, nil, false, "Outputs top entries in text form"},
		"disasm": {c, report.Dis, nil, true, "Output annotated assembly for functions matching regexp or address"},
		"list":   {c, report.List, nil, true, "Output annotated source for functions matching regexp"},
		"peek":   {c, report.Tree, nil, true, "Output callers/callees of functions matching regexp"},

		// Save binary formats to a file
		"callgrind": {c, report.Callgrind, awayFromTTY("callgraph.out"), false, "Outputs a graph in callgrind format"},
		"proto":     {c, report.Proto, awayFromTTY("pb.gz"), false, "Outputs the profile in compressed protobuf format"},

		// Generate report in DOT format and postprocess with dot
		"gif": {c, report.Dot, invokeDot("gif"), false, "Outputs a graph image in GIF format"},
		"pdf": {c, report.Dot, invokeDot("pdf"), false, "Outputs a graph in PDF format"},
		"png": {c, report.Dot, invokeDot("png"), false, "Outputs a graph image in PNG format"},
		"ps":  {c, report.Dot, invokeDot("ps"), false, "Outputs a graph in PS format"},

		// Save SVG output into a file after including svgpan library
		"svg": {c, report.Dot, saveSVGToFile(svgpan), false, "Outputs a graph in SVG format"},

		// Visualize postprocessed dot output
		"eog":    {c, report.Dot, invokeVisualizer(interactive, invokeDot("svg"), "svg", []string{"eog"}), false, "Visualize graph through eog"},
		"evince": {c, report.Dot, invokeVisualizer(interactive, invokeDot("pdf"), "pdf", []string{"evince"}), false, "Visualize graph through evince"},
		"gv":     {c, report.Dot, invokeVisualizer(interactive, invokeDot("ps"), "ps", []string{"gv --noantialias"}), false, "Visualize graph through gv"},
		"web":    {c, report.Dot, invokeVisualizer(interactive, saveSVGToFile(svgpan), "svg", browsers()), false, "Visualize graph through web browser"},

		// Visualize HTML directly generated by report.
		"weblist": {c, report.WebList, invokeVisualizer(interactive, awayFromTTY("html"), "html", browsers()), true, "Output annotated source in HTML for functions matching regexp or address"},
	}
}

// browsers returns a list of commands to attempt for web visualization
// on the current platform
func browsers() []string {
	cmds := []string{"chrome", "google-chrome", "firefox"}
	switch runtime.GOOS {
	case "darwin":
		cmds = append(cmds, "/usr/bin/open")
	case "windows":
		cmds = append(cmds, "cmd /c start")
	default:
		cmds = append(cmds, "xdg-open")
	}
	return cmds
}

// NewCompleter creates an autocompletion function for a set of commands.
func NewCompleter(cs Commands) Completer {
	return func(line string) string {
		switch tokens := strings.Fields(line); len(tokens) {
		case 0:
			// Nothing to complete
		case 1:
			// Single token -- complete command name
			found := ""
			for c := range cs {
				if strings.HasPrefix(c, tokens[0]) {
					if found != "" {
						return line
					}
					found = c
				}
			}
			if found != "" {
				return found
			}
		default:
			// Multiple tokens -- complete using command completer
			if c, ok := cs[tokens[0]]; ok {
				if c.Complete != nil {
					lastTokenIdx := len(tokens) - 1
					lastToken := tokens[lastTokenIdx]
					if strings.HasPrefix(lastToken, "-") {
						lastToken = "-" + c.Complete(lastToken[1:])
					} else {
						lastToken = c.Complete(lastToken)
					}
					return strings.Join(append(tokens[:lastTokenIdx], lastToken), " ")
				}
			}
		}
		return line
	}
}

// awayFromTTY saves the output in a file if it would otherwise go to
// the terminal screen. This is used to avoid dumping binary data on
// the screen.
func awayFromTTY(format string) PostProcessor {
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		if output == os.Stdout && ui.IsTerminal() {
			tempFile, err := tempfile.New("", "profile", "."+format)
			if err != nil {
				return err
			}
			ui.PrintErr("Generating report in ", tempFile.Name())
			_, err = fmt.Fprint(tempFile, input)
			return err
		}
		_, err := fmt.Fprint(output, input)
		return err
	}
}

func invokeDot(format string) PostProcessor {
	divert := awayFromTTY(format)
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		if _, err := exec.LookPath("dot"); err != nil {
			ui.PrintErr("Cannot find dot, have you installed Graphviz?")
			return err
		}
		cmd := exec.Command("dot", "-T"+format)
		var buf bytes.Buffer
		cmd.Stdin, cmd.Stdout, cmd.Stderr = input, &buf, os.Stderr
		if err := cmd.Run(); err != nil {
			return err
		}
		return divert(&buf, output, ui)
	}
}

func saveSVGToFile(svgpan **string) PostProcessor {
	generateSVG := invokeDot("svg")
	divert := awayFromTTY("svg")
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		baseSVG := &bytes.Buffer{}
		generateSVG(input, baseSVG, ui)
		massaged := &bytes.Buffer{}
		fmt.Fprint(massaged, svg.Massage(*baseSVG, **svgpan))
		return divert(massaged, output, ui)
	}
}

func invokeVisualizer(interactive **bool, format PostProcessor, suffix string, visualizers []string) PostProcessor {
	return func(input *bytes.Buffer, output io.Writer, ui plugin.UI) error {
		tempFile, err := tempfile.New(os.Getenv("PPROF_TMPDIR"), "pprof", "."+suffix)
		if err != nil {
			return err
		}
		tempfile.DeferDelete(tempFile.Name())
		if err = format(input, tempFile, ui); err != nil {
			return err
		}
		tempFile.Close() // on windows, if the file is Open, start cannot access it.
		// Try visualizers until one is successful
		for _, v := range visualizers {
			// Separate command and arguments for exec.Command.
			args := strings.Split(v, " ")
			if len(args) == 0 {
				continue
			}
			viewer := exec.Command(args[0], append(args[1:], tempFile.Name())...)
			viewer.Stderr = os.Stderr
			if err = viewer.Start(); err == nil {
				if !**interactive {
					// In command-line mode, wait for the viewer to be closed
					// before proceeding
					return viewer.Wait()
				}
				return nil
			}
		}
		return err
	}
}
<|endoftext|>


<commit_before>package tree
/*
#cgo LDFLAGS: -lxml2
#cgo CFLAGS: -I/usr/include/libxml2
#include <libxml/xmlversion.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xmlstring.h>

xmlNode * GoNodeNext(xmlNode *node) { return node->next; }
xmlNode * GoNodePrev(xmlNode *node) { return node->prev; }
xmlNode * GoNodeChildren(xmlNode *node) { return node->children; }
xmlNode * GoNodeLast(xmlNode *node) { return node->last; }
xmlNode * GoNodeParent(xmlNode *node) { return node->parent; }
int NodeType(xmlNode *node) { return (int)node->type; }

char * GoNodeName(xmlNode *node) { return (char*)node->name; }

char *
DumpNodeToXmlChar(xmlNode *node, xmlDoc *doc) {
  xmlBuffer *buff = xmlBufferCreate();
  xmlNodeDump(buff, doc, node, 0, 0);
  return (char*)buff->content;
}
*/
import "C"
import "unsafe"

func xmlNodeType(node *C.xmlNode) int {
	return int(C.NodeType(node))
}

func (node *XmlNode) Doc() *Doc {
	return node.DocRef
}

func (node *XmlNode) Ptr() unsafe.Pointer {
	return unsafe.Pointer(node.NodePtr)
}

func (node *XmlNode) ptr() *C.xmlNode {
	return node.NodePtr
}

func (node *XmlNode) Type() int {
	return int(C.NodeType(node.ptr()))
}

// Used internally to the XmlNode to quickly create nodes
func (node *XmlNode) new(ptr *_Ctype_struct__xmlNode) Node {
	if ptr == nil {
		return nil
	}
	return NewNode(unsafe.Pointer(ptr), node.Doc())
}

func (node *XmlNode) Parent() Node {
	return node.new(node.ptr().parent)
}

func (node *XmlNode) Next() Node {
	return node.new(node.ptr().next)
}

func (node *XmlNode) Prev() Node {
	return node.new(node.ptr().prev)
}

func (node *XmlNode) First() Node {
	// xmlNode->children actually points to the first
	// element in an array, so we can just use this as the ptr for first
	return node.new(node.ptr().children)
}

func (node *XmlNode) Last() Node {
	return node.new(node.ptr().last)
}

func (node *XmlNode) Remove() bool {
	C.xmlUnlinkNode(node.ptr())
	return true // TODO: Return false if it was previously unlinked
}

func (node *XmlNode) Size() int {
	return int(C.xmlChildElementCount(node.ptr()))
}

func (node *XmlNode) Name() string {
	return C.GoString(C.GoNodeName(node.ptr()))
}

func (node *XmlNode) SetName(name string) {
	C.xmlNodeSetName(node.ptr(), C.xmlCharStrdup(C.CString(name)))
}

func (node *XmlNode) String() string {
	return C.GoString(C.DumpNodeToXmlChar(node.ptr(), node.Doc().DocPtr))
}

func (node *XmlNode) Attribute(name string) (*Attribute, bool) {
	cName := C.xmlCharStrdup(C.CString(name))
	xmlAttrPtr := C.xmlHasProp(node.NodePtr, cName)
	didCreate := false
	if xmlAttrPtr == nil {
		didCreate = true;
		xmlAttrPtr = C.xmlNewProp(node.NodePtr, cName, C.xmlCharStrdup(C.CString("")))
	}
	attribute := NewNode(unsafe.Pointer(xmlAttrPtr), node.Doc()).(*Attribute)
	return attribute, didCreate;
}
<commit_msg>Don't need these C methods anymore<commit_after>package tree
/*
#cgo LDFLAGS: -lxml2
#cgo CFLAGS: -I/usr/include/libxml2
#include <libxml/xmlversion.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xmlstring.h>

int NodeType(xmlNode *node) { return (int)node->type; }

char * GoNodeName(xmlNode *node) { return (char*)node->name; }

char *
DumpNodeToXmlChar(xmlNode *node, xmlDoc *doc) {
  xmlBuffer *buff = xmlBufferCreate();
  xmlNodeDump(buff, doc, node, 0, 0);
  return (char*)buff->content;
}
*/
import "C"
import "unsafe"

func xmlNodeType(node *C.xmlNode) int {
	return int(C.NodeType(node))
}

func (node *XmlNode) Doc() *Doc {
	return node.DocRef
}

func (node *XmlNode) Ptr() unsafe.Pointer {
	return unsafe.Pointer(node.NodePtr)
}

func (node *XmlNode) ptr() *C.xmlNode {
	return node.NodePtr
}

func (node *XmlNode) Type() int {
	return int(C.NodeType(node.ptr()))
}

// Used internally to the XmlNode to quickly create nodes
func (node *XmlNode) new(ptr *_Ctype_struct__xmlNode) Node {
	if ptr == nil {
		return nil
	}
	return NewNode(unsafe.Pointer(ptr), node.Doc())
}

func (node *XmlNode) Parent() Node {
	return node.new(node.ptr().parent)
}

func (node *XmlNode) Next() Node {
	return node.new(node.ptr().next)
}

func (node *XmlNode) Prev() Node {
	return node.new(node.ptr().prev)
}

func (node *XmlNode) First() Node {
	// xmlNode->children actually points to the first
	// element in an array, so we can just use this as the ptr for first
	return node.new(node.ptr().children)
}

func (node *XmlNode) Last() Node {
	return node.new(node.ptr().last)
}

func (node *XmlNode) Remove() bool {
	C.xmlUnlinkNode(node.ptr())
	return true // TODO: Return false if it was previously unlinked
}

func (node *XmlNode) Size() int {
	return int(C.xmlChildElementCount(node.ptr()))
}

func (node *XmlNode) Name() string {
	return C.GoString(C.GoNodeName(node.ptr()))
}

func (node *XmlNode) SetName(name string) {
	C.xmlNodeSetName(node.ptr(), C.xmlCharStrdup(C.CString(name)))
}

func (node *XmlNode) String() string {
	return C.GoString(C.DumpNodeToXmlChar(node.ptr(), node.Doc().DocPtr))
}

func (node *XmlNode) Attribute(name string) (*Attribute, bool) {
	cName := C.xmlCharStrdup(C.CString(name))
	xmlAttrPtr := C.xmlHasProp(node.NodePtr, cName)
	didCreate := false
	if xmlAttrPtr == nil {
		didCreate = true;
		xmlAttrPtr = C.xmlNewProp(node.NodePtr, cName, C.xmlCharStrdup(C.CString("")))
	}
	attribute := NewNode(unsafe.Pointer(xmlAttrPtr), node.Doc()).(*Attribute)
	return attribute, didCreate;
}
<|endoftext|>


<commit_before>package ui

import (
	"bufio"
	"strconv"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"github.com/BrianReallyMany/yomama/dozens"
	"github.com/BrianReallyMany/yomama/iomama"
	"github.com/BrianReallyMany/yomama/iomama/fastaqual"
	"github.com/BrianReallyMany/yomama/iomama/fastq"
	"github.com/BrianReallyMany/yomama/seq/sortseq"
)

type MamaController struct {
}

// Instantiate a new yomama controller
func MakeMamaController() *MamaController {
	return &MamaController{}
}

func (c *MamaController) Dozens() string {
	return dozens.RandomDozens()
}

func (c *MamaController) PrepFiles(args []string, ch chan string) {
	defer close(ch)

	if len(args) < 1 {
		return
	}

	myPath := args[0]

	ch <- "\nVerifying files...\n"

	// Verify folder exists, contains .fasta, .qual and .oligo files
	var fastqFileName, fastaFileName, qualFileName, oligoFileName string

	fastqfiles, err := filepath.Glob(myPath + "/*.fastq")
	if err != nil || len(fastqfiles) != 1 {
		ch <- "No fastq files found, checking for fasta and qual files...\n"

		fastafiles, err := filepath.Glob(myPath + "/*.fasta")
		if err != nil || len(fastafiles) != 1 {
			ch <- "PrepFiles: locating fasta file failed"
			ch <- err.Error()
			return
		}
		qualfiles, err := filepath.Glob(myPath + "/*.qual")
		if err != nil || len(qualfiles) != 1 {
			ch <- "PrepFiles: locating qual file failed"
			ch <- err.Error()
			return
		}

		fastaFileName = fastafiles[0]
		qualFileName = qualfiles[0]
	} else {
		fastqFileName = fastqfiles[0]
	}

	oligofiles, err := filepath.Glob(myPath + "/*.oligo")
	if err != nil || len(oligofiles) != 1 {
		ch <- "PrepFiles: locating oligo file failed"
		ch <- err.Error()
		return
	} else {
		oligoFileName = oligofiles[0]
	}

	var seqReader iomama.SeqReader

	if fastqFileName != "" {
		fastqfile, err := os.Open(fastqFileName)
		if err != nil {
			ch <- "PrepFiles: fastq open failed"
			ch <- err.Error()
			return
		}

		seqReader = fastq.NewFastqReader(bufio.NewReader(fastqfile))
	} else if fastaFileName != "" && qualFileName != "" {
		fastafile, err := os.Open(fastaFileName)
		if err != nil {
			ch <- "PrepFiles: fasta open failed"
			ch <- err.Error()
			return
		}
		qualfile, err := os.Open(qualFileName)
		if err != nil {
			return
		}

		// Make FastaQualReader, SeqSorter
		seqReader = fastaqual.NewFastaQualReader(fastafile, qualfile)
	}

	if err != nil {
		return
	}

	oligofile, err := os.Open(oligoFileName)
	temporaryDefaultSeqSorterOptions := sortseq.NewSeqSorterOptions(0, 0, 0, true)
	sorter, err := sortseq.NewSeqSorter(oligofile, temporaryDefaultSeqSorterOptions)
	if err != nil {
		return
	}

	// Make Store
	store, err := sortseq.NewStore(myPath + "/yomama.store")
	if err != nil {
		return
	}

	// Track where sorting fails and why
	errorsMap := make(map[string]int)

	ch <- "Reading in sequences..."

	for seqReader.HasNext() {
		seq := seqReader.Next()
		sortedseq, err := sorter.SortSeq(seq)
		if err != nil {
			// Add entry to errorsMap
			// (required type assertion)
			if e, ok := err.(*sortseq.SeqSorterError); ok {
				errorsMap[e.Where] += 1
			}
			continue
		}
		err = store.AddSeq(sortedseq)
		if err != nil {
			ch <- "PrepFiles: error storing seq--"
			ch <- sortedseq.ToString()
		}
	}
	ch <- "...done.\n"
	ch <- "Error summary:"
	ch <- "Type of error\tCount"
	for k, v := range errorsMap {
		ch <- k + "\t" + strconv.Itoa(v) + "\n"
	}
}

func (c *MamaController) System(args []string) string {
	out, err := exec.Command(args[0], args[1:]...).Output()
	if err != nil {
		log.Fatal(err)
	}
	return string(out)
}
<commit_msg>slight refactor of PrepFiles<commit_after>package ui

import (
	"bufio"
	"strconv"
	"log"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"github.com/BrianReallyMany/yomama/dozens"
	"github.com/BrianReallyMany/yomama/iomama"
	"github.com/BrianReallyMany/yomama/iomama/fastaqual"
	"github.com/BrianReallyMany/yomama/iomama/fastq"
	"github.com/BrianReallyMany/yomama/seq/sortseq"
)

type MamaController struct {
}

// Instantiate a new yomama controller
func MakeMamaController() *MamaController {
	return &MamaController{}
}

func (c *MamaController) Dozens() string {
	return dozens.RandomDozens()
}

func (c *MamaController) PrepFiles(args []string, ch chan string) {
	defer close(ch)

	if len(args) < 1 {
		return
	}

	myPath := args[0]

	ch <- "\nVerifying files...\n"
	var fastaFileName, qualFileName string
	fastqFileName, err := getFileByExtension(myPath, "fastq")
	if err != nil {
		ch <- err.Error()
		ch <- "Fastq search failed, checking for fasta and qual files...\n"

		fastaFileName, err = getFileByExtension(myPath, "fasta")
		if err != nil {
			ch <- err.Error()
			ch <- "Fasta search failed, exiting now.\n"
			return
		}

		qualFileName, err = getFileByExtension(myPath, "qual")
		if err != nil {
			ch <- err.Error()
			ch <- "Qual search failed, exiting now.\n"
			return
		}
	}

	oligoFileName, err := getFileByExtension(myPath, "oligo")
	if err != nil {
		ch <- err.Error()
		ch <- "Oligo search failed, exiting now.\n"
		return
	}

	var seqReader iomama.SeqReader

	if fastqFileName != "" {
		fastqfile, err := os.Open(fastqFileName)
		if err != nil {
			ch <- "PrepFiles: fastq open failed"
			ch <- err.Error()
			return
		}

		seqReader = fastq.NewFastqReader(bufio.NewReader(fastqfile))
	} else if fastaFileName != "" && qualFileName != "" {
		fastafile, err := os.Open(fastaFileName)
		if err != nil {
			ch <- "PrepFiles: fasta open failed"
			ch <- err.Error()
			return
		}
		qualfile, err := os.Open(qualFileName)
		if err != nil {
			return
		}

		// Make FastaQualReader, SeqSorter
		seqReader = fastaqual.NewFastaQualReader(fastafile, qualfile)
	}

	oligofile, err := os.Open(oligoFileName)
	// TODO this is a hack to hold things together until options module is complete
	// then PrepFiles should receive a *SeqSorterOptions as an argument.
	temporaryDefaultSeqSorterOptions := sortseq.NewSeqSorterOptions(0, 0, 0, true)
	sorter, err := sortseq.NewSeqSorter(oligofile, temporaryDefaultSeqSorterOptions)
	if err != nil {
		return
	}

	// Make Store
	store, err := sortseq.NewStore(myPath + "/yomama.store")
	if err != nil {
		return
	}

	// Track where sorting fails and why
	errorsMap := make(map[string]int)

	ch <- "Reading in sequences..."

	for seqReader.HasNext() {
		seq := seqReader.Next()
		sortedseq, err := sorter.SortSeq(seq)
		if err != nil {
			// Add entry to errorsMap
			// (required type assertion)
			if e, ok := err.(*sortseq.SeqSorterError); ok {
				errorsMap[e.Where] += 1
			}
			continue
		}
		err = store.AddSeq(sortedseq)
		if err != nil {
			ch <- "PrepFiles: error storing seq--"
			ch <- sortedseq.ToString()
		}
	}
	ch <- "...done.\n"
	ch <- "Error summary:"
	ch <- "Type of error\tCount"
	for k, v := range errorsMap {
		ch <- k + "\t" + strconv.Itoa(v) + "\n"
	}
}

func (c *MamaController) System(args []string) string {
	out, err := exec.Command(args[0], args[1:]...).Output()
	if err != nil {
		log.Fatal(err)
	}
	return string(out)
}

func getFileByExtension(path, extension string) (string, error) {
	files, err := filepath.Glob(path + "/*." + extension)
	if err != nil {
		return "", err
	}
	if len(files) != 1 {
		return "", errors.New("Multiple files found with extension ." + extension)
	}
	return files[0], nil
}

<|endoftext|>


<commit_before>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(filterIt).Invoke(js.InternalObject(v), v), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	boolKind       = 1
	intKind        = 2
	int8Kind       = 3
	int16Kind      = 4
	int32Kind      = 5
	int64Kind      = 6
	uintKind       = 7
	uint8Kind      = 8
	uint16Kind     = 9
	uint32Kind     = 10
	uint64Kind     = 11
	uintptrKind    = 12
	float32Kind    = 13
	float64Kind    = 14
	complex64Kind  = 15
	complex128Kind = 16
	arrayKind      = 17
	interfaceKind  = 20
	mapKind        = 21
	ptrKind        = 22
	sliceKind      = 23
	stringKind     = 24
	structKind     = 25
)

func filterIt(v, value *js.Object) *js.Object {
	filter(v, v.Get("constructor"), value)
	return value
}

func filter(v, t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(v.Get("$array").Index(i), t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", v)
		len := keys.Length()
		for i := 0; i < len; i++ {
			o := v.Get(keys.Index(i).String())
			filter(o.Get("v"), t.Get("elem"), value.Get(o.Get("k").String()))
		}
	case ptrKind:
		filter(v, t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			fieldType := field.Get("typ")
			tag, tagOptions := parseTag(getJSONTag(field.Get("tag")))
			if tag == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name").String()
			if tagOptions.Contains("omitempty") {
				if isEmpty(v.Get(name), fieldType) {
					value.Delete(name)
					continue
				}
			}
			if tagOptions.Contains("string") && stringable(fieldType) {
				if fieldType.Get("kind").Int() == stringKind {
					value.Set(name, js.Global.Get("JSON").Call("stringify", v.Get(name)))
				} else {
					value.Set(name, value.Get(name).Call("toString"))
				}
			} else {
				filter(v.Get(name), fieldType, value.Get(name))
			}
			if len(tag) != 0 && tag != name {
				value.Set(tag, value.Get(name))
				value.Delete(name)
			}
		}
	case interfaceKind:
		filter(v.Get("$val"), v.Get("constructor"), value)
	}
}

func isEmpty(v, t *js.Object) bool {
	switch t.Get("kind").Int() {
	case boolKind:
		return !v.Bool()
	case int8Kind, int16Kind, int32Kind, intKind, uint8Kind, uint16Kind, uint32Kind, uintKind:
		return v.Int() == 0
	case int64Kind, uint64Kind:
		return v.Int64() == 0
	case float32Kind, float64Kind:
		return v.Float() == 0
	case arrayKind, sliceKind, stringKind:
		return v.Length() == 0
	case ptrKind:
		if v == t.Get("nil") {
			return true
		}
		return isEmpty(v.Call("$get"), t.Get("elem"))
	case interfaceKind:
		if v.Get("$val") == js.Undefined {
			return true
		}
		return isEmpty(v.Get("$val"), v.Get("constructor"))
	case mapKind:
		return js.Global.Get("Object").Call("keys", v).Length() == 0
	}
	return false
}

func stringable(t *js.Object) bool {
	switch t.Get("kind").Int() {
	case boolKind, int8Kind, int16Kind, int32Kind, intKind, uint8Kind, uint16Kind, uint32Kind, uintKind, int64Kind, uint64Kind, float32Kind, float64Kind, stringKind:
		return true
	}
	return false
}
<commit_msg>Added isMarshal helper func<commit_after>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(filterIt).Invoke(js.InternalObject(v), v), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	boolKind       = 1
	intKind        = 2
	int8Kind       = 3
	int16Kind      = 4
	int32Kind      = 5
	int64Kind      = 6
	uintKind       = 7
	uint8Kind      = 8
	uint16Kind     = 9
	uint32Kind     = 10
	uint64Kind     = 11
	uintptrKind    = 12
	float32Kind    = 13
	float64Kind    = 14
	complex64Kind  = 15
	complex128Kind = 16
	arrayKind      = 17
	interfaceKind  = 20
	mapKind        = 21
	ptrKind        = 22
	sliceKind      = 23
	stringKind     = 24
	structKind     = 25
)

func filterIt(v, value *js.Object) *js.Object {
	filter(v, v.Get("constructor"), value)
	return value
}

func filter(v, t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(v.Get("$array").Index(i), t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", v)
		len := keys.Length()
		for i := 0; i < len; i++ {
			o := v.Get(keys.Index(i).String())
			filter(o.Get("v"), t.Get("elem"), value.Get(o.Get("k").String()))
		}
	case ptrKind:
		filter(v, t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			fieldType := field.Get("typ")
			tag, tagOptions := parseTag(getJSONTag(field.Get("tag")))
			if tag == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name").String()
			if tagOptions.Contains("omitempty") {
				if isEmpty(v.Get(name), fieldType) {
					value.Delete(name)
					continue
				}
			}
			if tagOptions.Contains("string") && stringable(fieldType) {
				if fieldType.Get("kind").Int() == stringKind {
					value.Set(name, js.Global.Get("JSON").Call("stringify", v.Get(name)))
				} else {
					value.Set(name, value.Get(name).Call("toString"))
				}
			} else {
				filter(v.Get(name), fieldType, value.Get(name))
			}
			if len(tag) != 0 && tag != name {
				value.Set(tag, value.Get(name))
				value.Delete(name)
			}
		}
	case interfaceKind:
		filter(v.Get("$val"), v.Get("constructor"), value)
	}
}

func isEmpty(v, t *js.Object) bool {
	switch t.Get("kind").Int() {
	case boolKind:
		return !v.Bool()
	case int8Kind, int16Kind, int32Kind, intKind, uint8Kind, uint16Kind, uint32Kind, uintKind:
		return v.Int() == 0
	case int64Kind, uint64Kind:
		return v.Int64() == 0
	case float32Kind, float64Kind:
		return v.Float() == 0
	case arrayKind, sliceKind, stringKind:
		return v.Length() == 0
	case ptrKind:
		if v == t.Get("nil") {
			return true
		}
		return isEmpty(v.Call("$get"), t.Get("elem"))
	case interfaceKind:
		if v.Get("$val") == js.Undefined {
			return true
		}
		return isEmpty(v.Get("$val"), v.Get("constructor"))
	case mapKind:
		return js.Global.Get("Object").Call("keys", v).Length() == 0
	}
	return false
}

func stringable(t *js.Object) bool {
	switch t.Get("kind").Int() {
	case boolKind, int8Kind, int16Kind, int32Kind, intKind, uint8Kind, uint16Kind, uint32Kind, uintKind, int64Kind, uint64Kind, float32Kind, float64Kind, stringKind:
		return true
	}
	return false
}

func isMarshaler(t *js.Object) bool {
	methodNum := t.Get("methods").Length()
	for i := 0; i < methodNum; i++ {
		method := t.Get("methods").Index(i)
		if method.Get("name").String() == "MarshalJSON" {
			mt := method.Get("typ")
			return mt.Get("params").Length() == 0 && mt.Get("results").Length() == 2 && mt.Get("results").Index(0).Get("string").String() == "[]uint8" && mt.Get("results").Index(1).Get("string").String() == "error"
		}
	}
	return false
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"fmt"
	"github.com/lpabon/godbc"
)

type PblCache struct {
                    stats        *CacheStats
	cachemap     *CacheMap
	blocksize    uint32
	cachesize    uint64
	blocks       uint64
	writethrough bool
}

// cachesize is in bytes
// blocksize is in bytes
func NewPblCache(cachesize uint64, writethrough bool, blocksize uint32) *PblCache {

	godbc.Require(cachesize > 0)

	cache := &PblCache{}
	cache.blocks = cachesize / uint64(blocksize)
	cache.cachesize = cachesize
	cache.blocksize = blocksize
	cache.writethrough = writethrough

	cache.stats = NewCacheStats()
	cache.cachemap = NewCacheMap(cache.blocks)

	//cache.db = NewKVIoDB("cache.iodb", cachesize, bcsize, blocksize)
	//godbc.Check(cache.db != nil)

	godbc.Ensure(cache.blocks > 0)
	godbc.Ensure(cache.cachemap != nil)
	godbc.Ensure(cache.stats != nil)

	return cache
}

func (c *PblCache) Close() {
}

func (c *PblCache) Invalidate(key AddressMapKey) {
	if c.cachemap.HasAddressMapKey(key) {
		c.stats.writehits++
		c.stats.invalidations++
		c.cachemap.FreeAddress(key)

		/*
			start := time.Now()
			c.db.Delete([]byte(key), index)
			end := time.Now()
			c.stats.tdeletions.Add(end.Sub(start))
		*/
	}
}

func (c *PblCache) Insert(key AddressMapKey) {
	c.stats.insertions++

	index, _ := c.cachemap.Alloc(key)

	// Insert new key in cache map
	c.cachemap.SetAddressMapKey(key, index)

}

func (c *PblCache) Write(obj, block uint64) {
	c.stats.writes++

	key := AddressMapKey{obj, block}

	// Invalidate
	c.Invalidate(key)

	// We would do back end IO here

	// Insert
	if c.writethrough {
		c.Insert(key)
	}
}

func (c *PblCache) Read(obj, block uint64) bool {
	c.stats.reads++

	if _, ok := c.cachemap.Get(obj, block); ok {
		// Read Hit
		c.stats.readhits++
		return true

	} else {
		// Read miss
		c.Insert(AddressMapKey{obj, block})
		return false
	}
}

func (c *PblCache) String() string {
	return fmt.Sprintf(
		"Cache Utilization: 0 \n" +
			c.stats.String())
}

func (c *PblCache) Stats() *CacheStats {
	return c.stats.Copy()
}

func (c *PblCache) StatsClear() {
	c.stats = NewCacheStats()
}
<commit_msg>Replace test fmt<commit_after>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"fmt"
	"github.com/lpabon/godbc"
)

type PblCache struct {
	stats        *CacheStats
	cachemap     *CacheMap
	blocksize    uint32
	cachesize    uint64
	blocks       uint64
	writethrough bool
}

// cachesize is in bytes
// blocksize is in bytes
func NewPblCache(cachesize uint64, writethrough bool, blocksize uint32) *PblCache {

	godbc.Require(cachesize > 0)

	cache := &PblCache{}
	cache.blocks = cachesize / uint64(blocksize)
	cache.cachesize = cachesize
	cache.blocksize = blocksize
	cache.writethrough = writethrough

	cache.stats = NewCacheStats()
	cache.cachemap = NewCacheMap(cache.blocks)

	//cache.db = NewKVIoDB("cache.iodb", cachesize, bcsize, blocksize)
	//godbc.Check(cache.db != nil)

	godbc.Ensure(cache.blocks > 0)
	godbc.Ensure(cache.cachemap != nil)
	godbc.Ensure(cache.stats != nil)

	return cache
}

func (c *PblCache) Close() {
}

func (c *PblCache) Invalidate(key AddressMapKey) {
	if c.cachemap.HasAddressMapKey(key) {
		c.stats.writehits++
		c.stats.invalidations++
		c.cachemap.FreeAddress(key)

		/*
			start := time.Now()
			c.db.Delete([]byte(key), index)
			end := time.Now()
			c.stats.tdeletions.Add(end.Sub(start))
		*/
	}
}

func (c *PblCache) Insert(key AddressMapKey) {
	c.stats.insertions++

	index, _ := c.cachemap.Alloc(key)

	// Insert new key in cache map
	c.cachemap.SetAddressMapKey(key, index)

}

func (c *PblCache) Write(obj, block uint64) {
	c.stats.writes++

	key := AddressMapKey{obj, block}

	// Invalidate
	c.Invalidate(key)

	// We would do back end IO here

	// Insert
	if c.writethrough {
		c.Insert(key)
	}
}

func (c *PblCache) Read(obj, block uint64) bool {
	c.stats.reads++

	if _, ok := c.cachemap.Get(obj, block); ok {
		// Read Hit
		c.stats.readhits++
		return true

	} else {
		// Read miss
		c.Insert(AddressMapKey{obj, block})
		return false
	}
}

func (c *PblCache) String() string {
	return fmt.Sprintf(
		"Cache Utilization: 0 \n" +
			c.stats.String())
}

func (c *PblCache) Stats() *CacheStats {
	return c.stats.Copy()
}

func (c *PblCache) StatsClear() {
	c.stats = NewCacheStats()
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"sort"
	//"sync"
	"fmt"
)

type WorkQueue struct {
	//sync.RWMutex
	//workMap  map[string]*Workunit //all parsed workunits
	all      WorkunitMap
	Queue    WorkunitMap // WORK_STAT_QUEUED - waiting workunits
	Checkout WorkunitMap // WORK_STAT_CHECKOUT - workunits being checked out
	Suspend  WorkunitMap // WORK_STAT_SUSPEND - suspended workunits
}

func NewWorkQueue() *WorkQueue {
	wq := &WorkQueue{
		all:      *NewWorkunitMap(),
		Queue:    *NewWorkunitMap(), // these workunits that are ready to be checked out
		Checkout: *NewWorkunitMap(), // workunits that are checked out right now
		Suspend:  *NewWorkunitMap(),
	}

	wq.all.Init("WorkQueue/workMap")
	wq.Queue.Init("WorkQueue/Queue")
	wq.Checkout.Init("WorkQueue/Checkout")
	wq.Suspend.Init("WorkQueue/Suspend")

	return wq
}

//--------accessor methods-------

func (wq *WorkQueue) Len() (int, error) {
	return wq.all.Len()
}

func (wq *WorkQueue) Add(workunit *Workunit) (err error) {
	if workunit.Id == "" {
		return errors.New("try to push a workunit with an empty id")
	}

	var work_str string
	work_str, err = workunit.String()
	if err != nil {
		err = fmt.Errorf("(WorkQueue/Add) will not add workunit, it has broken id")
		return
	}

	var ok bool
	var oldWorkunit *Workunit
	oldWorkunit, ok, err = wq.all.Get(workunit.Workunit_Unique_Identifier)
	if err != nil {
		return
	}
	if !ok {
		logger.Debug(3, "(WorkQueue/Add) workunit %s already in WorkQueue with state %s, deleting old pointer", work_str, oldWorkunit.State)
		err = wq.Delete(workunit.Workunit_Unique_Identifier)
		if err != nil {
			return
		}
	}

	logger.Debug(3, "(WorkQueue/Add) Adding workunit %s to WorkQueue", work_str)
	err = wq.all.Set(workunit)
	if err != nil {
		return
	}
	err = wq.StatusChange(Workunit_Unique_Identifier{}, workunit, WORK_STAT_QUEUED, "")
	if err != nil {
		return
	}
	return
}

func (wq *WorkQueue) Get(id Workunit_Unique_Identifier) (w *Workunit, ok bool, err error) {
	w, ok, err = wq.all.Get(id)
	return
}

func (wq *WorkQueue) GetForJob(jobid string) (worklist []*Workunit, err error) {

	workunits, err := wq.all.GetWorkunits()
	if err != nil {
		return
	}
	for _, work := range workunits {
		parentid := work.JobId
		if jobid == parentid {
			worklist = append(worklist, work)
		}
	}
	return
}

func (wq *WorkQueue) GetAll() (worklist []*Workunit, err error) {
	return wq.all.GetWorkunits()
}

func (wq *WorkQueue) Clean() (workunits []*Workunit) {
	workunt_list, err := wq.all.GetWorkunits()
	if err != nil {
		return
	}
	for _, work := range workunt_list {
		id := work.Workunit_Unique_Identifier
		if work == nil || work.Info == nil {
			workunits = append(workunits, work)
			wq.Queue.Delete(id)
			wq.Checkout.Delete(id)
			wq.Suspend.Delete(id)
			wq.all.Delete(id)
			logger.Error("error: in WorkQueue workunit %s is nil, deleted from queue", id)
		}
	}
	return
}

func (wq *WorkQueue) Delete(id Workunit_Unique_Identifier) (err error) {
	err = wq.Queue.Delete(id)
	if err != nil {
		return
	}
	err = wq.Checkout.Delete(id)
	if err != nil {
		return
	}
	err = wq.Suspend.Delete(id)
	if err != nil {
		return
	}
	err = wq.all.Delete(id)
	if err != nil {
		return
	}
	return

}

func (wq *WorkQueue) Has(id Workunit_Unique_Identifier) (has bool, err error) {
	_, has, err = wq.all.Get(id)
	return
}

//--------end of accessors-------

func (wq *WorkQueue) StatusChange(id Workunit_Unique_Identifier, workunit *Workunit, new_status string, reason string) (err error) {
	//move workunit id between maps. no need to care about the old status because
	//delete function will do nothing if the operated map has no such key.

	if workunit == nil {
		var ok bool
		workunit, ok, err = wq.all.Get(id)
		if err != nil {
			return
		}
		if !ok {
			var work_str string
			work_str, err = id.String()
			if err != nil {
				err = fmt.Errorf("() workid.String() returned: %s", err.Error())
				return
			}
			err = fmt.Errorf("WQueue.statusChange: invalid workunit id:" + work_str)
			return
		}
	} else {
		id = workunit.Workunit_Unique_Identifier
	}

	if workunit.State == new_status {
		return
	}
	if workunit.State != WORK_STAT_CHECKOUT && workunit.State != WORK_STAT_RESERVED {
		workunit.Client = ""
	}

	switch new_status {
	case WORK_STAT_CHECKOUT:
		wq.Queue.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Checkout.Set(workunit)

	case WORK_STAT_QUEUED:
		wq.Checkout.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Queue.Set(workunit)

	case WORK_STAT_SUSPEND:
		if reason == "" {
			err = fmt.Errorf("suspend workunit only with reason!")
			return
		}
		wq.Checkout.Delete(id)
		wq.Queue.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Suspend.Set(workunit)

	default:
		wq.Checkout.Delete(id)
		wq.Queue.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
	}

	return
}

//select workunits, return a slice of ids based on given queuing policy and requested count
//if available is a positive value, filter by workunit input size
func (wq *WorkQueue) selectWorkunits(workunits WorkList, policy string, available int64, count int) (selected []*Workunit, err error) {
	logger.Debug(3, "starting selectWorkunits")

	if policy == "FCFS" {
		sort.Sort(byFCFS{workunits})
	}
	added := 0
	for _, work := range workunits {
		if added == count {
			break
		}

		inputSize := int64(0)
		for _, input := range work.Inputs {
			inputSize = inputSize + input.Size
		}
		// skip work that is too large for client
		if (available < 0) || (available > inputSize) {
			selected = append(selected, work)
			added = added + 1
		}

	}

	if len(selected) == 0 {
		err = errors.New(e.NoEligibleWorkunitFound)
		return
	}

	logger.Debug(3, "done with selectWorkunits")
	return
}

//queuing/prioritizing related functions
type WorkList []*Workunit

func (wl WorkList) Len() int      { return len(wl) }
func (wl WorkList) Swap(i, j int) { wl[i], wl[j] = wl[j], wl[i] }

type byFCFS struct{ WorkList }

//compare priority first, then FCFS (if priorities are the same)
func (s byFCFS) Less(i, j int) (ret bool) {
	p_i := s.WorkList[i].Info.Priority
	p_j := s.WorkList[j].Info.Priority
	switch {
	case p_i > p_j:
		return true
	case p_i < p_j:
		return false
	case p_i == p_j:
		return s.WorkList[i].Info.SubmitTime.Before(s.WorkList[j].Info.SubmitTime)
	}
	return
}
<commit_msg>fix if statement<commit_after>package core

import (
	"errors"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"sort"
	//"sync"
	"fmt"
)

type WorkQueue struct {
	//sync.RWMutex
	//workMap  map[string]*Workunit //all parsed workunits
	all      WorkunitMap
	Queue    WorkunitMap // WORK_STAT_QUEUED - waiting workunits
	Checkout WorkunitMap // WORK_STAT_CHECKOUT - workunits being checked out
	Suspend  WorkunitMap // WORK_STAT_SUSPEND - suspended workunits
}

func NewWorkQueue() *WorkQueue {
	wq := &WorkQueue{
		all:      *NewWorkunitMap(),
		Queue:    *NewWorkunitMap(), // these workunits that are ready to be checked out
		Checkout: *NewWorkunitMap(), // workunits that are checked out right now
		Suspend:  *NewWorkunitMap(),
	}

	wq.all.Init("WorkQueue/workMap")
	wq.Queue.Init("WorkQueue/Queue")
	wq.Checkout.Init("WorkQueue/Checkout")
	wq.Suspend.Init("WorkQueue/Suspend")

	return wq
}

//--------accessor methods-------

func (wq *WorkQueue) Len() (int, error) {
	return wq.all.Len()
}

func (wq *WorkQueue) Add(workunit *Workunit) (err error) {
	if workunit.Id == "" {
		return errors.New("try to push a workunit with an empty id")
	}

	var work_str string
	work_str, err = workunit.String()
	if err != nil {
		err = fmt.Errorf("(WorkQueue/Add) will not add workunit, it has broken id")
		return
	}

	var ok bool
	var oldWorkunit *Workunit
	oldWorkunit, ok, err = wq.all.Get(workunit.Workunit_Unique_Identifier)
	if err != nil {
		return
	}
	if ok && oldWorkunit != nil {
		logger.Debug(3, "(WorkQueue/Add) workunit %s already in WorkQueue with state %s, deleting old pointer", work_str, oldWorkunit.State)
		err = wq.Delete(workunit.Workunit_Unique_Identifier)
		if err != nil {
			return
		}
	}

	logger.Debug(3, "(WorkQueue/Add) Adding workunit %s to WorkQueue", work_str)
	err = wq.all.Set(workunit)
	if err != nil {
		return
	}
	err = wq.StatusChange(Workunit_Unique_Identifier{}, workunit, WORK_STAT_QUEUED, "")
	if err != nil {
		return
	}
	return
}

func (wq *WorkQueue) Get(id Workunit_Unique_Identifier) (w *Workunit, ok bool, err error) {
	w, ok, err = wq.all.Get(id)
	return
}

func (wq *WorkQueue) GetForJob(jobid string) (worklist []*Workunit, err error) {

	workunits, err := wq.all.GetWorkunits()
	if err != nil {
		return
	}
	for _, work := range workunits {
		parentid := work.JobId
		if jobid == parentid {
			worklist = append(worklist, work)
		}
	}
	return
}

func (wq *WorkQueue) GetAll() (worklist []*Workunit, err error) {
	return wq.all.GetWorkunits()
}

func (wq *WorkQueue) Clean() (workunits []*Workunit) {
	workunt_list, err := wq.all.GetWorkunits()
	if err != nil {
		return
	}
	for _, work := range workunt_list {
		id := work.Workunit_Unique_Identifier
		if work == nil || work.Info == nil {
			workunits = append(workunits, work)
			wq.Queue.Delete(id)
			wq.Checkout.Delete(id)
			wq.Suspend.Delete(id)
			wq.all.Delete(id)
			logger.Error("error: in WorkQueue workunit %s is nil, deleted from queue", id)
		}
	}
	return
}

func (wq *WorkQueue) Delete(id Workunit_Unique_Identifier) (err error) {
	err = wq.Queue.Delete(id)
	if err != nil {
		return
	}
	err = wq.Checkout.Delete(id)
	if err != nil {
		return
	}
	err = wq.Suspend.Delete(id)
	if err != nil {
		return
	}
	err = wq.all.Delete(id)
	if err != nil {
		return
	}
	return

}

func (wq *WorkQueue) Has(id Workunit_Unique_Identifier) (has bool, err error) {
	_, has, err = wq.all.Get(id)
	return
}

//--------end of accessors-------

func (wq *WorkQueue) StatusChange(id Workunit_Unique_Identifier, workunit *Workunit, new_status string, reason string) (err error) {
	//move workunit id between maps. no need to care about the old status because
	//delete function will do nothing if the operated map has no such key.

	if workunit == nil {
		var ok bool
		workunit, ok, err = wq.all.Get(id)
		if err != nil {
			return
		}
		if !ok {
			var work_str string
			work_str, err = id.String()
			if err != nil {
				err = fmt.Errorf("() workid.String() returned: %s", err.Error())
				return
			}
			err = fmt.Errorf("WQueue.statusChange: invalid workunit id:" + work_str)
			return
		}
	} else {
		id = workunit.Workunit_Unique_Identifier
	}

	if workunit.State == new_status {
		return
	}
	if workunit.State != WORK_STAT_CHECKOUT && workunit.State != WORK_STAT_RESERVED {
		workunit.Client = ""
	}

	switch new_status {
	case WORK_STAT_CHECKOUT:
		wq.Queue.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Checkout.Set(workunit)

	case WORK_STAT_QUEUED:
		wq.Checkout.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Queue.Set(workunit)

	case WORK_STAT_SUSPEND:
		if reason == "" {
			err = fmt.Errorf("suspend workunit only with reason!")
			return
		}
		wq.Checkout.Delete(id)
		wq.Queue.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
		wq.Suspend.Set(workunit)

	default:
		wq.Checkout.Delete(id)
		wq.Queue.Delete(id)
		wq.Suspend.Delete(id)
		err = workunit.SetState(new_status, reason)
		if err != nil {
			return
		}
	}

	return
}

//select workunits, return a slice of ids based on given queuing policy and requested count
//if available is a positive value, filter by workunit input size
func (wq *WorkQueue) selectWorkunits(workunits WorkList, policy string, available int64, count int) (selected []*Workunit, err error) {
	logger.Debug(3, "starting selectWorkunits")

	if policy == "FCFS" {
		sort.Sort(byFCFS{workunits})
	}
	added := 0
	for _, work := range workunits {
		if added == count {
			break
		}

		inputSize := int64(0)
		for _, input := range work.Inputs {
			inputSize = inputSize + input.Size
		}
		// skip work that is too large for client
		if (available < 0) || (available > inputSize) {
			selected = append(selected, work)
			added = added + 1
		}

	}

	if len(selected) == 0 {
		err = errors.New(e.NoEligibleWorkunitFound)
		return
	}

	logger.Debug(3, "done with selectWorkunits")
	return
}

//queuing/prioritizing related functions
type WorkList []*Workunit

func (wl WorkList) Len() int      { return len(wl) }
func (wl WorkList) Swap(i, j int) { wl[i], wl[j] = wl[j], wl[i] }

type byFCFS struct{ WorkList }

//compare priority first, then FCFS (if priorities are the same)
func (s byFCFS) Less(i, j int) (ret bool) {
	p_i := s.WorkList[i].Info.Priority
	p_j := s.WorkList[j].Info.Priority
	switch {
	case p_i > p_j:
		return true
	case p_i < p_j:
		return false
	case p_i == p_j:
		return s.WorkList[i].Info.SubmitTime.Before(s.WorkList[j].Info.SubmitTime)
	}
	return
}
<|endoftext|>


<commit_before><commit_msg>Fix function PC displaying when creating fallback stacktrace<commit_after><|endoftext|>