<commit_before>// This file is generated - do not edit.

package jen

// standardLibraryHints contains package name hints
var standardLibraryHints = map[string]string{
	"archive/tar":                       "tar",
	"archive/zip":                       "zip",
	"bufio":                             "bufio",
	"builtin":                           "builtin",
	"bytes":                             "bytes",
	"cmd/asm/internal/arch":             "arch",
	"cmd/asm/internal/asm":              "asm",
	"cmd/asm/internal/flags":            "flags",
	"cmd/asm/internal/lex":              "lex",
	"cmd/compile/internal/amd64":        "amd64",
	"cmd/compile/internal/arm":          "arm",
	"cmd/compile/internal/arm64":        "arm64",
	"cmd/compile/internal/gc":           "gc",
	"cmd/compile/internal/mips":         "mips",
	"cmd/compile/internal/mips64":       "mips64",
	"cmd/compile/internal/ppc64":        "ppc64",
	"cmd/compile/internal/s390x":        "s390x",
	"cmd/compile/internal/ssa":          "ssa",
	"cmd/compile/internal/syntax":       "syntax",
	"cmd/compile/internal/test":         "test",
	"cmd/compile/internal/types":        "types",
	"cmd/compile/internal/x86":          "x86",
	"cmd/go/internal/base":              "base",
	"cmd/go/internal/bug":               "bug",
	"cmd/go/internal/cache":             "cache",
	"cmd/go/internal/cfg":               "cfg",
	"cmd/go/internal/clean":             "clean",
	"cmd/go/internal/cmdflag":           "cmdflag",
	"cmd/go/internal/doc":               "doc",
	"cmd/go/internal/envcmd":            "envcmd",
	"cmd/go/internal/fix":               "fix",
	"cmd/go/internal/fmtcmd":            "fmtcmd",
	"cmd/go/internal/generate":          "generate",
	"cmd/go/internal/get":               "get",
	"cmd/go/internal/help":              "help",
	"cmd/go/internal/list":              "list",
	"cmd/go/internal/load":              "load",
	"cmd/go/internal/run":               "run",
	"cmd/go/internal/str":               "str",
	"cmd/go/internal/test":              "test",
	"cmd/go/internal/tool":              "tool",
	"cmd/go/internal/version":           "version",
	"cmd/go/internal/vet":               "vet",
	"cmd/go/internal/web":               "web",
	"cmd/go/internal/work":              "work",
	"cmd/internal/bio":                  "bio",
	"cmd/internal/browser":              "browser",
	"cmd/internal/buildid":              "buildid",
	"cmd/internal/dwarf":                "dwarf",
	"cmd/internal/edit":                 "edit",
	"cmd/internal/gcprog":               "gcprog",
	"cmd/internal/goobj":                "goobj",
	"cmd/internal/obj":                  "obj",
	"cmd/internal/obj/arm":              "arm",
	"cmd/internal/obj/arm64":            "arm64",
	"cmd/internal/obj/mips":             "mips",
	"cmd/internal/obj/ppc64":            "ppc64",
	"cmd/internal/obj/s390x":            "s390x",
	"cmd/internal/obj/x86":              "x86",
	"cmd/internal/objabi":               "objabi",
	"cmd/internal/objfile":              "objfile",
	"cmd/internal/src":                  "src",
	"cmd/internal/sys":                  "sys",
	"cmd/internal/test2json":            "test2json",
	"cmd/link/internal/amd64":           "amd64",
	"cmd/link/internal/arm":             "arm",
	"cmd/link/internal/arm64":           "arm64",
	"cmd/link/internal/ld":              "ld",
	"cmd/link/internal/loadelf":         "loadelf",
	"cmd/link/internal/loadmacho":       "loadmacho",
	"cmd/link/internal/loadpe":          "loadpe",
	"cmd/link/internal/mips":            "mips",
	"cmd/link/internal/mips64":          "mips64",
	"cmd/link/internal/objfile":         "objfile",
	"cmd/link/internal/ppc64":           "ppc64",
	"cmd/link/internal/s390x":           "s390x",
	"cmd/link/internal/sym":             "sym",
	"cmd/link/internal/x86":             "x86",
	"cmd/vet/internal/cfg":              "cfg",
	"cmd/vet/internal/whitelist":        "whitelist",
	"compress/bzip2":                    "bzip2",
	"compress/flate":                    "flate",
	"compress/gzip":                     "gzip",
	"compress/lzw":                      "lzw",
	"compress/zlib":                     "zlib",
	"container/heap":                    "heap",
	"container/list":                    "list",
	"container/ring":                    "ring",
	"context":                           "context",
	"crypto":                            "crypto",
	"crypto/aes":                        "aes",
	"crypto/cipher":                     "cipher",
	"crypto/des":                        "des",
	"crypto/dsa":                        "dsa",
	"crypto/ecdsa":                      "ecdsa",
	"crypto/elliptic":                   "elliptic",
	"crypto/hmac":                       "hmac",
	"crypto/internal/cipherhw":          "cipherhw",
	"crypto/md5":                        "md5",
	"crypto/rand":                       "rand",
	"crypto/rc4":                        "rc4",
	"crypto/rsa":                        "rsa",
	"crypto/sha1":                       "sha1",
	"crypto/sha256":                     "sha256",
	"crypto/sha512":                     "sha512",
	"crypto/subtle":                     "subtle",
	"crypto/tls":                        "tls",
	"crypto/x509":                       "x509",
	"crypto/x509/pkix":                  "pkix",
	"database/sql":                      "sql",
	"database/sql/driver":               "driver",
	"debug/dwarf":                       "dwarf",
	"debug/elf":                         "elf",
	"debug/gosym":                       "gosym",
	"debug/macho":                       "macho",
	"debug/pe":                          "pe",
	"debug/plan9obj":                    "plan9obj",
	"encoding":                          "encoding",
	"encoding/ascii85":                  "ascii85",
	"encoding/asn1":                     "asn1",
	"encoding/base32":                   "base32",
	"encoding/base64":                   "base64",
	"encoding/binary":                   "binary",
	"encoding/csv":                      "csv",
	"encoding/gob":                      "gob",
	"encoding/hex":                      "hex",
	"encoding/json":                     "json",
	"encoding/pem":                      "pem",
	"encoding/xml":                      "xml",
	"errors":                            "errors",
	"expvar":                            "expvar",
	"flag":                              "flag",
	"fmt":                               "fmt",
	"go/ast":                            "ast",
	"go/build":                          "build",
	"go/constant":                       "constant",
	"go/doc":                            "doc",
	"go/format":                         "format",
	"go/importer":                       "importer",
	"go/internal/gccgoimporter":         "gccgoimporter",
	"go/internal/gcimporter":            "gcimporter",
	"go/internal/srcimporter":           "srcimporter",
	"go/parser":                         "parser",
	"go/printer":                        "printer",
	"go/scanner":                        "scanner",
	"go/token":                          "token",
	"go/types":                          "types",
	"hash":                              "hash",
	"hash/adler32":                      "adler32",
	"hash/crc32":                        "crc32",
	"hash/crc64":                        "crc64",
	"hash/fnv":                          "fnv",
	"html":                              "html",
	"html/template":                     "template",
	"image":                             "image",
	"image/color":                       "color",
	"image/color/palette":               "palette",
	"image/draw":                        "draw",
	"image/gif":                         "gif",
	"image/internal/imageutil":          "imageutil",
	"image/jpeg":                        "jpeg",
	"image/png":                         "png",
	"index/suffixarray":                 "suffixarray",
	"internal/cpu":                      "cpu",
	"internal/nettrace":                 "nettrace",
	"internal/poll":                     "poll",
	"internal/race":                     "race",
	"internal/singleflight":             "singleflight",
	"internal/syscall/windows":          "windows",
	"internal/syscall/windows/registry": "registry",
	"internal/syscall/windows/sysdll":   "sysdll",
	"internal/testenv":                  "testenv",
	"internal/testlog":                  "testlog",
	"internal/trace":                    "trace",
	"io":                                "io",
	"io/ioutil":                         "ioutil",
	"log":                               "log",
	"log/syslog":                        "syslog",
	"math":                              "math",
	"math/big":                          "big",
	"math/bits":                         "bits",
	"math/cmplx":                        "cmplx",
	"math/rand":                         "rand",
	"mime":                              "mime",
	"mime/multipart":                    "multipart",
	"mime/quotedprintable":              "quotedprintable",
	"net":                            "net",
	"net/http":                       "http",
	"net/http/cgi":                   "cgi",
	"net/http/cookiejar":             "cookiejar",
	"net/http/fcgi":                  "fcgi",
	"net/http/httptest":              "httptest",
	"net/http/httptrace":             "httptrace",
	"net/http/httputil":              "httputil",
	"net/http/internal":              "internal",
	"net/http/pprof":                 "pprof",
	"net/internal/socktest":          "socktest",
	"net/mail":                       "mail",
	"net/rpc":                        "rpc",
	"net/rpc/jsonrpc":                "jsonrpc",
	"net/smtp":                       "smtp",
	"net/textproto":                  "textproto",
	"net/url":                        "url",
	"os":                             "os",
	"os/exec":                        "exec",
	"os/signal":                      "signal",
	"os/signal/internal/pty":         "pty",
	"os/user":                        "user",
	"path":                           "path",
	"path/filepath":                  "filepath",
	"plugin":                         "plugin",
	"reflect":                        "reflect",
	"regexp":                         "regexp",
	"regexp/syntax":                  "syntax",
	"runtime":                        "runtime",
	"runtime/cgo":                    "cgo",
	"runtime/debug":                  "debug",
	"runtime/internal/atomic":        "atomic",
	"runtime/internal/sys":           "sys",
	"runtime/pprof":                  "pprof",
	"runtime/pprof/internal/profile": "profile",
	"runtime/race":                   "race",
	"runtime/trace":                  "trace",
	"sort":                           "sort",
	"strconv":                        "strconv",
	"strings":                        "strings",
	"sync":                           "sync",
	"sync/atomic":                    "atomic",
	"syscall":                        "syscall",
	"testing":                        "testing",
	"testing/internal/testdeps":      "testdeps",
	"testing/iotest":                 "iotest",
	"testing/quick":                  "quick",
	"text/scanner":                   "scanner",
	"text/tabwriter":                 "tabwriter",
	"text/template":                  "template",
	"text/template/parse":            "parse",
	"time":                           "time",
	"unicode":                        "unicode",
	"unicode/utf16":                  "utf16",
	"unicode/utf8":                   "utf8",
	"unsafe":                         "unsafe",
}
<commit_msg>Re-generate package names for go 1.11 standard library (only internal pacakges changed).<commit_after>// This file is generated - do not edit.

package jen

// standardLibraryHints contains package name hints
var standardLibraryHints = map[string]string{
	"archive/tar":                       "tar",
	"archive/zip":                       "zip",
	"bufio":                             "bufio",
	"builtin":                           "builtin",
	"bytes":                             "bytes",
	"cmd/asm/internal/arch":             "arch",
	"cmd/asm/internal/asm":              "asm",
	"cmd/asm/internal/flags":            "flags",
	"cmd/asm/internal/lex":              "lex",
	"cmd/compile/internal/amd64":        "amd64",
	"cmd/compile/internal/arm":          "arm",
	"cmd/compile/internal/arm64":        "arm64",
	"cmd/compile/internal/gc":           "gc",
	"cmd/compile/internal/mips":         "mips",
	"cmd/compile/internal/mips64":       "mips64",
	"cmd/compile/internal/ppc64":        "ppc64",
	"cmd/compile/internal/s390x":        "s390x",
	"cmd/compile/internal/ssa":          "ssa",
	"cmd/compile/internal/syntax":       "syntax",
	"cmd/compile/internal/test":         "test",
	"cmd/compile/internal/types":        "types",
	"cmd/compile/internal/wasm":         "wasm",
	"cmd/compile/internal/x86":          "x86",
	"cmd/go/internal/base":              "base",
	"cmd/go/internal/bug":               "bug",
	"cmd/go/internal/cache":             "cache",
	"cmd/go/internal/cfg":               "cfg",
	"cmd/go/internal/clean":             "clean",
	"cmd/go/internal/cmdflag":           "cmdflag",
	"cmd/go/internal/dirhash":           "dirhash",
	"cmd/go/internal/doc":               "doc",
	"cmd/go/internal/envcmd":            "envcmd",
	"cmd/go/internal/fix":               "fix",
	"cmd/go/internal/fmtcmd":            "fmtcmd",
	"cmd/go/internal/generate":          "generate",
	"cmd/go/internal/get":               "get",
	"cmd/go/internal/help":              "help",
	"cmd/go/internal/imports":           "imports",
	"cmd/go/internal/list":              "list",
	"cmd/go/internal/load":              "load",
	"cmd/go/internal/modcmd":            "modcmd",
	"cmd/go/internal/modconv":           "modconv",
	"cmd/go/internal/modfetch":          "modfetch",
	"cmd/go/internal/modfetch/codehost": "codehost",
	"cmd/go/internal/modfile":           "modfile",
	"cmd/go/internal/modget":            "modget",
	"cmd/go/internal/modinfo":           "modinfo",
	"cmd/go/internal/modload":           "modload",
	"cmd/go/internal/module":            "module",
	"cmd/go/internal/mvs":               "mvs",
	"cmd/go/internal/par":               "par",
	"cmd/go/internal/run":               "run",
	"cmd/go/internal/search":            "search",
	"cmd/go/internal/semver":            "semver",
	"cmd/go/internal/str":               "str",
	"cmd/go/internal/test":              "test",
	"cmd/go/internal/tool":              "tool",
	"cmd/go/internal/txtar":             "txtar",
	"cmd/go/internal/version":           "version",
	"cmd/go/internal/vet":               "vet",
	"cmd/go/internal/web":               "web",
	"cmd/go/internal/web2":              "web2",
	"cmd/go/internal/webtest":           "webtest",
	"cmd/go/internal/work":              "work",
	"cmd/internal/bio":                  "bio",
	"cmd/internal/browser":              "browser",
	"cmd/internal/buildid":              "buildid",
	"cmd/internal/dwarf":                "dwarf",
	"cmd/internal/edit":                 "edit",
	"cmd/internal/gcprog":               "gcprog",
	"cmd/internal/goobj":                "goobj",
	"cmd/internal/obj":                  "obj",
	"cmd/internal/obj/arm":              "arm",
	"cmd/internal/obj/arm64":            "arm64",
	"cmd/internal/obj/mips":             "mips",
	"cmd/internal/obj/ppc64":            "ppc64",
	"cmd/internal/obj/s390x":            "s390x",
	"cmd/internal/obj/wasm":             "wasm",
	"cmd/internal/obj/x86":              "x86",
	"cmd/internal/objabi":               "objabi",
	"cmd/internal/objfile":              "objfile",
	"cmd/internal/src":                  "src",
	"cmd/internal/sys":                  "sys",
	"cmd/internal/test2json":            "test2json",
	"cmd/link/internal/amd64":           "amd64",
	"cmd/link/internal/arm":             "arm",
	"cmd/link/internal/arm64":           "arm64",
	"cmd/link/internal/ld":              "ld",
	"cmd/link/internal/loadelf":         "loadelf",
	"cmd/link/internal/loadmacho":       "loadmacho",
	"cmd/link/internal/loadpe":          "loadpe",
	"cmd/link/internal/mips":            "mips",
	"cmd/link/internal/mips64":          "mips64",
	"cmd/link/internal/objfile":         "objfile",
	"cmd/link/internal/ppc64":           "ppc64",
	"cmd/link/internal/s390x":           "s390x",
	"cmd/link/internal/sym":             "sym",
	"cmd/link/internal/wasm":            "wasm",
	"cmd/link/internal/x86":             "x86",
	"cmd/vet/internal/cfg":              "cfg",
	"cmd/vet/internal/whitelist":        "whitelist",
	"compress/bzip2":                    "bzip2",
	"compress/flate":                    "flate",
	"compress/gzip":                     "gzip",
	"compress/lzw":                      "lzw",
	"compress/zlib":                     "zlib",
	"container/heap":                    "heap",
	"container/list":                    "list",
	"container/ring":                    "ring",
	"context":                           "context",
	"crypto":                            "crypto",
	"crypto/aes":                        "aes",
	"crypto/cipher":                     "cipher",
	"crypto/des":                        "des",
	"crypto/dsa":                        "dsa",
	"crypto/ecdsa":                      "ecdsa",
	"crypto/elliptic":                   "elliptic",
	"crypto/hmac":                       "hmac",
	"crypto/internal/randutil":          "randutil",
	"crypto/internal/subtle":            "subtle",
	"crypto/md5":                        "md5",
	"crypto/rand":                       "rand",
	"crypto/rc4":                        "rc4",
	"crypto/rsa":                        "rsa",
	"crypto/sha1":                       "sha1",
	"crypto/sha256":                     "sha256",
	"crypto/sha512":                     "sha512",
	"crypto/subtle":                     "subtle",
	"crypto/tls":                        "tls",
	"crypto/x509":                       "x509",
	"crypto/x509/pkix":                  "pkix",
	"database/sql":                      "sql",
	"database/sql/driver":               "driver",
	"debug/dwarf":                       "dwarf",
	"debug/elf":                         "elf",
	"debug/gosym":                       "gosym",
	"debug/macho":                       "macho",
	"debug/pe":                          "pe",
	"debug/plan9obj":                    "plan9obj",
	"encoding":                          "encoding",
	"encoding/ascii85":                  "ascii85",
	"encoding/asn1":                     "asn1",
	"encoding/base32":                   "base32",
	"encoding/base64":                   "base64",
	"encoding/binary":                   "binary",
	"encoding/csv":                      "csv",
	"encoding/gob":                      "gob",
	"encoding/hex":                      "hex",
	"encoding/json":                     "json",
	"encoding/pem":                      "pem",
	"encoding/xml":                      "xml",
	"errors":                            "errors",
	"expvar":                            "expvar",
	"flag":                              "flag",
	"fmt":                               "fmt",
	"go/ast":                            "ast",
	"go/build":                          "build",
	"go/constant":                       "constant",
	"go/doc":                            "doc",
	"go/format":                         "format",
	"go/importer":                       "importer",
	"go/internal/gccgoimporter":         "gccgoimporter",
	"go/internal/gcimporter":            "gcimporter",
	"go/internal/srcimporter":           "srcimporter",
	"go/parser":                         "parser",
	"go/printer":                        "printer",
	"go/scanner":                        "scanner",
	"go/token":                          "token",
	"go/types":                          "types",
	"hash":                              "hash",
	"hash/adler32":                      "adler32",
	"hash/crc32":                        "crc32",
	"hash/crc64":                        "crc64",
	"hash/fnv":                          "fnv",
	"html":                              "html",
	"html/template":                     "template",
	"image":                             "image",
	"image/color":                       "color",
	"image/color/palette":               "palette",
	"image/draw":                        "draw",
	"image/gif":                         "gif",
	"image/internal/imageutil":          "imageutil",
	"image/jpeg":                        "jpeg",
	"image/png":                         "png",
	"index/suffixarray":                 "suffixarray",
	"internal/bytealg":                  "bytealg",
	"internal/cpu":                      "cpu",
	"internal/nettrace":                 "nettrace",
	"internal/poll":                     "poll",
	"internal/race":                     "race",
	"internal/singleflight":             "singleflight",
	"internal/syscall/unix":             "unix",
	"internal/syscall/windows":          "windows",
	"internal/syscall/windows/registry": "registry",
	"internal/syscall/windows/sysdll":   "sysdll",
	"internal/testenv":                  "testenv",
	"internal/testlog":                  "testlog",
	"internal/trace":                    "trace",
	"io":                                "io",
	"io/ioutil":                         "ioutil",
	"log":                               "log",
	"log/syslog":                        "syslog",
	"math":                              "math",
	"math/big":                          "big",
	"math/bits":                         "bits",
	"math/cmplx":                        "cmplx",
	"math/rand":                         "rand",
	"mime":                              "mime",
	"mime/multipart":                    "multipart",
	"mime/quotedprintable":              "quotedprintable",
	"net":                               "net",
	"net/http":                          "http",
	"net/http/cgi":                      "cgi",
	"net/http/cookiejar":                "cookiejar",
	"net/http/fcgi":                     "fcgi",
	"net/http/httptest":                 "httptest",
	"net/http/httptrace":                "httptrace",
	"net/http/httputil":                 "httputil",
	"net/http/internal":                 "internal",
	"net/http/pprof":                    "pprof",
	"net/internal/socktest":             "socktest",
	"net/mail":                          "mail",
	"net/rpc":                           "rpc",
	"net/rpc/jsonrpc":                   "jsonrpc",
	"net/smtp":                          "smtp",
	"net/textproto":                     "textproto",
	"net/url":                           "url",
	"os":                                "os",
	"os/exec":                           "exec",
	"os/signal":                         "signal",
	"os/signal/internal/pty":            "pty",
	"os/user":                           "user",
	"path":                              "path",
	"path/filepath":                     "filepath",
	"plugin":                            "plugin",
	"reflect":                           "reflect",
	"regexp":                            "regexp",
	"regexp/syntax":                     "syntax",
	"runtime":                           "runtime",
	"runtime/cgo":                       "cgo",
	"runtime/debug":                     "debug",
	"runtime/internal/atomic":           "atomic",
	"runtime/internal/sys":              "sys",
	"runtime/pprof":                     "pprof",
	"runtime/pprof/internal/profile":    "profile",
	"runtime/race":                      "race",
	"runtime/trace":                     "trace",
	"sort":                              "sort",
	"strconv":                           "strconv",
	"strings":                           "strings",
	"sync":                              "sync",
	"sync/atomic":                       "atomic",
	"syscall":                           "syscall",
	"testing":                           "testing",
	"testing/internal/testdeps":         "testdeps",
	"testing/iotest":                    "iotest",
	"testing/quick":                     "quick",
	"text/scanner":                      "scanner",
	"text/tabwriter":                    "tabwriter",
	"text/template":                     "template",
	"text/template/parse":               "parse",
	"time":                              "time",
	"unicode":                           "unicode",
	"unicode/utf16":                     "utf16",
	"unicode/utf8":                      "utf8",
	"unsafe":                            "unsafe",
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package certconfig

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	ds "github.com/luci/gae/service/datastore"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/retry/transient"
	"github.com/luci/luci-go/server/auth"

	"github.com/luci/luci-go/tokenserver/api/admin/v1"
)

// List of OAuth scopes to use for token sent to CRL endpoint if config doesn't
// specify 'oauth_scopes' field.
var crlFetchDefaultScopes = []string{
	"https://www.googleapis.com/auth/userinfo.email",
}

// FetchCRLRPC implements CertificateAuthorities.FetchCRL RPC method.
type FetchCRLRPC struct {
}

// FetchCRL makes the server fetch a CRL for some CA.
func (r *FetchCRLRPC) FetchCRL(c context.Context, req *admin.FetchCRLRequest) (*admin.FetchCRLResponse, error) {
	// Grab a corresponding CA entity. It contains URL of CRL to fetch.
	ca := &CA{CN: req.Cn}
	switch err := ds.Get(c, ca); {
	case err == ds.ErrNoSuchEntity:
		return nil, grpc.Errorf(codes.NotFound, "no such CA %q", ca.CN)
	case err != nil:
		return nil, grpc.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Grab CRL URL from the CA config.
	cfg, err := ca.ParseConfig()
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "broken CA config in the datastore - %s", err)
	}
	if cfg.CrlUrl == "" {
		return nil, grpc.Errorf(codes.NotFound, "CA %q doesn't have CRL defined", ca.CN)
	}

	// Grab info about last processed CRL, if any.
	crl := &CRL{Parent: ds.KeyForObj(c, ca)}
	if err = ds.Get(c, crl); err != nil && err != ds.ErrNoSuchEntity {
		return nil, grpc.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Fetch latest CRL blob.
	logging.Infof(c, "Fetching CRL for %q from %s", ca.CN, cfg.CrlUrl)
	knownETag := crl.LastFetchETag
	if req.Force {
		knownETag = ""
	}
	fetchCtx, _ := clock.WithTimeout(c, time.Minute)
	crlDer, newEtag, err := fetchCRL(fetchCtx, cfg, knownETag)
	switch {
	case transient.Tag.In(err):
		return nil, grpc.Errorf(codes.Internal, "transient error when fetching CRL - %s", err)
	case err != nil:
		return nil, grpc.Errorf(codes.Unknown, "can't fetch CRL - %s", err)
	}

	// No changes?
	if knownETag != "" && knownETag == newEtag {
		logging.Infof(c, "No changes to CRL (etag is %s), skipping", knownETag)
	} else {
		logging.Infof(c, "Fetched CRL size is %d bytes, etag is %s", len(crlDer), newEtag)
		crl, err = validateAndStoreCRL(c, crlDer, newEtag, ca, crl)
		switch {
		case transient.Tag.In(err):
			return nil, grpc.Errorf(codes.Internal, "transient error when storing CRL - %s", err)
		case err != nil:
			return nil, grpc.Errorf(codes.Unknown, "bad CRL - %s", err)
		}
	}

	return &admin.FetchCRLResponse{CrlStatus: crl.GetStatusProto()}, nil
}

////////////////////////////////////////////////////////////////////////////////

// fetchCRL fetches a blob with pem or der-encoded CRL from the CRL endpoint.
//
// It knows how to use ETag headers to avoid fetching already known data.
// May return transient and fatal errors.
//
// It attempts to parse the fetched data as PEM first. If successful, returns
// decoded DER block. If the data doesn't look like a PEM block, returns it as
// is, assuming it is already in the DER form.
func fetchCRL(c context.Context, cfg *admin.CertificateAuthorityConfig, knownETag string) (blob []byte, etag string, err error) {
	// Pick auth or non-auth transport.
	var transport http.RoundTripper
	if cfg.UseOauth {
		var scopes []string
		if len(cfg.OauthScopes) != 0 {
			scopes = cfg.OauthScopes
		} else {
			scopes = crlFetchDefaultScopes
		}
		transport, err = auth.GetRPCTransport(c, auth.AsSelf, auth.WithScopes(scopes...))
	} else {
		transport, err = auth.GetRPCTransport(c, auth.NoAuth)
	}
	if err != nil {
		return nil, "", err
	}

	// Send the request with ETag related headers.
	req, err := http.NewRequest("GET", cfg.CrlUrl, nil)
	if err != nil {
		return nil, "", err
	}
	if knownETag != "" {
		req.Header.Set("If-None-Match", knownETag)
	}
	cl := http.Client{Transport: transport}
	resp, err := cl.Do(req)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	defer resp.Body.Close()

	// Nothing new?
	if resp.StatusCode == http.StatusNotModified {
		if knownETag == "" {
			return nil, "", errors.New("unexpected 304 status, no etag header was sent")
		}
		return nil, knownETag, nil
	}

	// Read the body in its entirety (plus new etag, if any).
	blob, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	etag = resp.Header.Get("ETag")

	// Transient error?
	if resp.StatusCode >= http.StatusInternalServerError {
		logging.Warningf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", errors.Reason("server replied with HTTP %d", resp.StatusCode).
			Tag(transient.Tag).Err()
	}

	// Something we don't support or expect?
	if resp.StatusCode != http.StatusOK {
		logging.Errorf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", fmt.Errorf("unexpected status HTTP %d", resp.StatusCode)
	}

	// Attempt to parse PEM. It's fine if it fails (block == nil), in that case we
	// assume the body is already in the DER form. If it's not really DER, it will
	// fail the subsequent validation step.
	block, rest := pem.Decode(blob)
	if block == nil {
		return blob, etag, nil
	}

	// It is a PEM, but maybe it's not a CRL, or it has multiple blocks. Reject
	// these with fatal errors.
	if block.Type != "X509 CRL" {
		logging.Errorf(c, "GET %s - bad CRL, expecting %q PEM, got %q", cfg.CrlUrl, "X509 CRL", block.Type)
		return nil, "", fmt.Errorf("got %q PEM, not a CRL PEM", block.Type)
	}
	if len(rest) != 0 {
		logging.Errorf(c, "GET %s - bad CRL, more than one PEM block", cfg.CrlUrl)
		return nil, "", fmt.Errorf("bad CRL, more than one PEM block")
	}

	// Have a valid X509 CRL PEM, return its decoded body.
	return block.Bytes, etag, nil
}

// validateAndStoreCRL handles incoming CRL blob fetched by 'fetchCRL'.
func validateAndStoreCRL(c context.Context, crlDer []byte, etag string, ca *CA, prev *CRL) (*CRL, error) {
	// Make sure it is signed by the CA.
	caCert, err := x509.ParseCertificate(ca.Cert)
	if err != nil {
		return nil, fmt.Errorf("cert in the datastore is broken - %s", err)
	}
	crl, err := x509.ParseDERCRL(crlDer)
	if err != nil {
		return nil, fmt.Errorf("not a valid x509 CRL - %s", err)
	}
	if err = caCert.CheckCRLSignature(crl); err != nil {
		return nil, fmt.Errorf("CRL is not signed by the CA - %s", err)
	}

	// The CRL is peachy. Update a sharded set of all revoked certs.
	logging.Infof(c, "CRL last updated %s", crl.TBSCertList.ThisUpdate)
	logging.Infof(c, "Found %d entries in the CRL", len(crl.TBSCertList.RevokedCertificates))
	if err = UpdateCRLSet(c, ca.CN, CRLShardCount, crl); err != nil {
		return nil, err
	}
	logging.Infof(c, "All CRL entries stored")

	// Update the CRL entity. Use EntityVersion to make sure we are not
	// overwriting someone else's changes.
	var updated *CRL
	err = ds.RunInTransaction(c, func(c context.Context) error {
		entity := *prev
		if err := ds.Get(c, &entity); err != nil && err != ds.ErrNoSuchEntity {
			return err
		}
		if entity.EntityVersion != prev.EntityVersion {
			return fmt.Errorf("CRL for %q was updated concurrently while we were working on it", ca.CN)
		}
		entity.EntityVersion++
		entity.LastUpdateTime = crl.TBSCertList.ThisUpdate.UTC()
		entity.LastFetchTime = clock.Now(c).UTC()
		entity.LastFetchETag = etag
		entity.RevokedCertsCount = len(crl.TBSCertList.RevokedCertificates)

		updated = &entity // used outside of this function
		toPut := []interface{}{updated}

		// Mark CA entity as ready for usage.
		curCA := CA{CN: ca.CN}
		switch err := ds.Get(c, &curCA); {
		case err == ds.ErrNoSuchEntity:
			return fmt.Errorf("CA entity for %q is unexpectedly gone", ca.CN)
		case err != nil:
			return err
		}
		if !curCA.Ready {
			logging.Infof(c, "CA %q is ready now", curCA.CN)
			curCA.Ready = true
			toPut = append(toPut, &curCA)
		}
		return ds.Put(c, toPut)
	}, nil)
	if err != nil {
		return nil, transient.Tag.Apply(err)
	}

	logging.Infof(c, "CRL for %q is updated, entity version is %d", ca.CN, updated.EntityVersion)
	return updated, nil
}
<commit_msg>tokenserver: Assume 'use_oauth' if 'oauth_scopes' is specified.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package certconfig

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	ds "github.com/luci/gae/service/datastore"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/retry/transient"
	"github.com/luci/luci-go/server/auth"

	"github.com/luci/luci-go/tokenserver/api/admin/v1"
)

// List of OAuth scopes to use for token sent to CRL endpoint if config doesn't
// specify 'oauth_scopes' field.
var crlFetchDefaultScopes = []string{
	"https://www.googleapis.com/auth/userinfo.email",
}

// FetchCRLRPC implements CertificateAuthorities.FetchCRL RPC method.
type FetchCRLRPC struct {
}

// FetchCRL makes the server fetch a CRL for some CA.
func (r *FetchCRLRPC) FetchCRL(c context.Context, req *admin.FetchCRLRequest) (*admin.FetchCRLResponse, error) {
	// Grab a corresponding CA entity. It contains URL of CRL to fetch.
	ca := &CA{CN: req.Cn}
	switch err := ds.Get(c, ca); {
	case err == ds.ErrNoSuchEntity:
		return nil, grpc.Errorf(codes.NotFound, "no such CA %q", ca.CN)
	case err != nil:
		return nil, grpc.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Grab CRL URL from the CA config.
	cfg, err := ca.ParseConfig()
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "broken CA config in the datastore - %s", err)
	}
	if cfg.CrlUrl == "" {
		return nil, grpc.Errorf(codes.NotFound, "CA %q doesn't have CRL defined", ca.CN)
	}

	// Grab info about last processed CRL, if any.
	crl := &CRL{Parent: ds.KeyForObj(c, ca)}
	if err = ds.Get(c, crl); err != nil && err != ds.ErrNoSuchEntity {
		return nil, grpc.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Fetch latest CRL blob.
	logging.Infof(c, "Fetching CRL for %q from %s", ca.CN, cfg.CrlUrl)
	knownETag := crl.LastFetchETag
	if req.Force {
		knownETag = ""
	}
	fetchCtx, _ := clock.WithTimeout(c, time.Minute)
	crlDer, newEtag, err := fetchCRL(fetchCtx, cfg, knownETag)
	switch {
	case transient.Tag.In(err):
		return nil, grpc.Errorf(codes.Internal, "transient error when fetching CRL - %s", err)
	case err != nil:
		return nil, grpc.Errorf(codes.Unknown, "can't fetch CRL - %s", err)
	}

	// No changes?
	if knownETag != "" && knownETag == newEtag {
		logging.Infof(c, "No changes to CRL (etag is %s), skipping", knownETag)
	} else {
		logging.Infof(c, "Fetched CRL size is %d bytes, etag is %s", len(crlDer), newEtag)
		crl, err = validateAndStoreCRL(c, crlDer, newEtag, ca, crl)
		switch {
		case transient.Tag.In(err):
			return nil, grpc.Errorf(codes.Internal, "transient error when storing CRL - %s", err)
		case err != nil:
			return nil, grpc.Errorf(codes.Unknown, "bad CRL - %s", err)
		}
	}

	return &admin.FetchCRLResponse{CrlStatus: crl.GetStatusProto()}, nil
}

////////////////////////////////////////////////////////////////////////////////

// fetchCRL fetches a blob with pem or der-encoded CRL from the CRL endpoint.
//
// It knows how to use ETag headers to avoid fetching already known data.
// May return transient and fatal errors.
//
// It attempts to parse the fetched data as PEM first. If successful, returns
// decoded DER block. If the data doesn't look like a PEM block, returns it as
// is, assuming it is already in the DER form.
func fetchCRL(c context.Context, cfg *admin.CertificateAuthorityConfig, knownETag string) (blob []byte, etag string, err error) {
	// Pick auth or non-auth transport.
	var transport http.RoundTripper
	if cfg.UseOauth || len(cfg.OauthScopes) != 0 {
		var scopes []string
		if len(cfg.OauthScopes) != 0 {
			scopes = cfg.OauthScopes
		} else {
			scopes = crlFetchDefaultScopes
		}
		transport, err = auth.GetRPCTransport(c, auth.AsSelf, auth.WithScopes(scopes...))
	} else {
		transport, err = auth.GetRPCTransport(c, auth.NoAuth)
	}
	if err != nil {
		return nil, "", err
	}

	// Send the request with ETag related headers.
	req, err := http.NewRequest("GET", cfg.CrlUrl, nil)
	if err != nil {
		return nil, "", err
	}
	if knownETag != "" {
		req.Header.Set("If-None-Match", knownETag)
	}
	cl := http.Client{Transport: transport}
	resp, err := cl.Do(req)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	defer resp.Body.Close()

	// Nothing new?
	if resp.StatusCode == http.StatusNotModified {
		if knownETag == "" {
			return nil, "", errors.New("unexpected 304 status, no etag header was sent")
		}
		return nil, knownETag, nil
	}

	// Read the body in its entirety (plus new etag, if any).
	blob, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	etag = resp.Header.Get("ETag")

	// Transient error?
	if resp.StatusCode >= http.StatusInternalServerError {
		logging.Warningf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", errors.Reason("server replied with HTTP %d", resp.StatusCode).
			Tag(transient.Tag).Err()
	}

	// Something we don't support or expect?
	if resp.StatusCode != http.StatusOK {
		logging.Errorf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", fmt.Errorf("unexpected status HTTP %d", resp.StatusCode)
	}

	// Attempt to parse PEM. It's fine if it fails (block == nil), in that case we
	// assume the body is already in the DER form. If it's not really DER, it will
	// fail the subsequent validation step.
	block, rest := pem.Decode(blob)
	if block == nil {
		return blob, etag, nil
	}

	// It is a PEM, but maybe it's not a CRL, or it has multiple blocks. Reject
	// these with fatal errors.
	if block.Type != "X509 CRL" {
		logging.Errorf(c, "GET %s - bad CRL, expecting %q PEM, got %q", cfg.CrlUrl, "X509 CRL", block.Type)
		return nil, "", fmt.Errorf("got %q PEM, not a CRL PEM", block.Type)
	}
	if len(rest) != 0 {
		logging.Errorf(c, "GET %s - bad CRL, more than one PEM block", cfg.CrlUrl)
		return nil, "", fmt.Errorf("bad CRL, more than one PEM block")
	}

	// Have a valid X509 CRL PEM, return its decoded body.
	return block.Bytes, etag, nil
}

// validateAndStoreCRL handles incoming CRL blob fetched by 'fetchCRL'.
func validateAndStoreCRL(c context.Context, crlDer []byte, etag string, ca *CA, prev *CRL) (*CRL, error) {
	// Make sure it is signed by the CA.
	caCert, err := x509.ParseCertificate(ca.Cert)
	if err != nil {
		return nil, fmt.Errorf("cert in the datastore is broken - %s", err)
	}
	crl, err := x509.ParseDERCRL(crlDer)
	if err != nil {
		return nil, fmt.Errorf("not a valid x509 CRL - %s", err)
	}
	if err = caCert.CheckCRLSignature(crl); err != nil {
		return nil, fmt.Errorf("CRL is not signed by the CA - %s", err)
	}

	// The CRL is peachy. Update a sharded set of all revoked certs.
	logging.Infof(c, "CRL last updated %s", crl.TBSCertList.ThisUpdate)
	logging.Infof(c, "Found %d entries in the CRL", len(crl.TBSCertList.RevokedCertificates))
	if err = UpdateCRLSet(c, ca.CN, CRLShardCount, crl); err != nil {
		return nil, err
	}
	logging.Infof(c, "All CRL entries stored")

	// Update the CRL entity. Use EntityVersion to make sure we are not
	// overwriting someone else's changes.
	var updated *CRL
	err = ds.RunInTransaction(c, func(c context.Context) error {
		entity := *prev
		if err := ds.Get(c, &entity); err != nil && err != ds.ErrNoSuchEntity {
			return err
		}
		if entity.EntityVersion != prev.EntityVersion {
			return fmt.Errorf("CRL for %q was updated concurrently while we were working on it", ca.CN)
		}
		entity.EntityVersion++
		entity.LastUpdateTime = crl.TBSCertList.ThisUpdate.UTC()
		entity.LastFetchTime = clock.Now(c).UTC()
		entity.LastFetchETag = etag
		entity.RevokedCertsCount = len(crl.TBSCertList.RevokedCertificates)

		updated = &entity // used outside of this function
		toPut := []interface{}{updated}

		// Mark CA entity as ready for usage.
		curCA := CA{CN: ca.CN}
		switch err := ds.Get(c, &curCA); {
		case err == ds.ErrNoSuchEntity:
			return fmt.Errorf("CA entity for %q is unexpectedly gone", ca.CN)
		case err != nil:
			return err
		}
		if !curCA.Ready {
			logging.Infof(c, "CA %q is ready now", curCA.CN)
			curCA.Ready = true
			toPut = append(toPut, &curCA)
		}
		return ds.Put(c, toPut)
	}, nil)
	if err != nil {
		return nil, transient.Tag.Apply(err)
	}

	logging.Infof(c, "CRL for %q is updated, entity version is %d", ca.CN, updated.EntityVersion)
	return updated, nil
}
<|endoftext|>


<commit_before>package casbin

import (
	"testing"
	"log"
)

func testEnforce(t *testing.T, e *Enforcer, sub string, obj string, act string, res bool) {
	if e.enforce(sub, obj, act) != res {
		t.Errorf("%s, %s, %s: %t, supposed to be %t", sub, obj, act, !res, res)
	}
}

func TestBasicModel(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/basic_model.conf", "examples/basic_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", false)
	testEnforce(t, e, "alice", "data2", "write", false)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
}

func TestBasicModelWithRoot(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/basic_model_with_root.conf", "examples/basic_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", false)
	testEnforce(t, e, "alice", "data2", "write", false)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
	testEnforce(t, e, "root", "data1", "read", true)
	testEnforce(t, e, "root", "data1", "write", true)
	testEnforce(t, e, "root", "data2", "read", true)
	testEnforce(t, e, "root", "data2", "write", true)
}

func TestRBACModel(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", true)
	testEnforce(t, e, "alice", "data2", "write", true)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
}

func testKeyMatch(t *testing.T, e *Enforcer, key1 string, key2 string, res bool) {
	my_res := e.keyMatch(key1, key2)
	log.Printf("%s, %s: %t", key1, key2, my_res)

	if my_res != res {
		t.Errorf("%s < %s: %t, supposed to be %t", key1, key2, !res, res)
	}
}

func TestKeyMatch(t *testing.T) {
	e := &Enforcer{}

	testKeyMatch(t, e, "/foo", "/foo", true)
	testKeyMatch(t, e, "/foo", "/foo*", true)
	testKeyMatch(t, e, "/foo", "/foo/*", false)
	testKeyMatch(t, e, "/foo/bar", "/foo", false)
	testKeyMatch(t, e, "/foo/bar", "/foo*", true)
	testKeyMatch(t, e, "/foo/bar", "/foo/*", true)
	testKeyMatch(t, e, "/foobar", "/foo", false)
	testKeyMatch(t, e, "/foobar", "/foo*", true)
	testKeyMatch(t, e, "/foobar", "/foo/*", false)
}

func testGetRoles(e *Enforcer, name string) {
	log.Print("Roles for ", name, ": ", e.getRoles(name))
}

func TestGetRoles(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	testGetRoles(e, "alice")
	testGetRoles(e, "bob")
	testGetRoles(e, "data2_admin")
	testGetRoles(e, "non_exist")
}

func testGetPolicy(e *Enforcer, fieldIndex int, fieldValue string) {
	log.Print("Policy for ", fieldValue, ": ", e.getFilteredPolicy(fieldIndex, fieldValue))
}

func TestGetPolicy(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	log.Print("Policy: ", e.getPolicy())
	testGetPolicy(e, 0, "alice")
	testGetPolicy(e, 0, "bob")
	testGetPolicy(e, 0, "data2_admin")
	testGetPolicy(e, 1, "data1")
	testGetPolicy(e, 1, "data2")
	testGetPolicy(e, 2, "read")
	testGetPolicy(e, 2, "write")
}
<commit_msg>Check the testing status of getting roles and policy.<commit_after>package casbin

import (
	"testing"
	"log"
)

func testEnforce(t *testing.T, e *Enforcer, sub string, obj string, act string, res bool) {
	if e.enforce(sub, obj, act) != res {
		t.Errorf("%s, %s, %s: %t, supposed to be %t", sub, obj, act, !res, res)
	}
}

func TestBasicModel(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/basic_model.conf", "examples/basic_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", false)
	testEnforce(t, e, "alice", "data2", "write", false)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
}

func TestBasicModelWithRoot(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/basic_model_with_root.conf", "examples/basic_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", false)
	testEnforce(t, e, "alice", "data2", "write", false)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
	testEnforce(t, e, "root", "data1", "read", true)
	testEnforce(t, e, "root", "data1", "write", true)
	testEnforce(t, e, "root", "data2", "read", true)
	testEnforce(t, e, "root", "data2", "write", true)
}

func TestRBACModel(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	testEnforce(t, e, "alice", "data1", "read", true)
	testEnforce(t, e, "alice", "data1", "write", false)
	testEnforce(t, e, "alice", "data2", "read", true)
	testEnforce(t, e, "alice", "data2", "write", true)
	testEnforce(t, e, "bob", "data1", "read", false)
	testEnforce(t, e, "bob", "data1", "write", false)
	testEnforce(t, e, "bob", "data2", "read", false)
	testEnforce(t, e, "bob", "data2", "write", true)
}

func testKeyMatch(t *testing.T, e *Enforcer, key1 string, key2 string, res bool) {
	myRes := e.keyMatch(key1, key2)
	log.Printf("%s < %s: %t", key1, key2, myRes)

	if myRes != res {
		t.Errorf("%s < %s: %t, supposed to be %t", key1, key2, !res, res)
	}
}

func TestKeyMatch(t *testing.T) {
	e := &Enforcer{}

	testKeyMatch(t, e, "/foo", "/foo", true)
	testKeyMatch(t, e, "/foo", "/foo*", true)
	testKeyMatch(t, e, "/foo", "/foo/*", false)
	testKeyMatch(t, e, "/foo/bar", "/foo", false)
	testKeyMatch(t, e, "/foo/bar", "/foo*", true)
	testKeyMatch(t, e, "/foo/bar", "/foo/*", true)
	testKeyMatch(t, e, "/foobar", "/foo", false)
	testKeyMatch(t, e, "/foobar", "/foo*", true)
	testKeyMatch(t, e, "/foobar", "/foo/*", false)
}

func arrayEquals(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	for i, v := range a {
		if v != b[i] {
			return false
		}
	}
	return true
}

func array2DEquals(a [][]string, b [][]string) bool {
	if len(a) != len(b) {
		return false
	}

	for i, v := range a {
		if !arrayEquals(v, b[i]) {
			return false
		}
	}
	return true
}

func testGetRoles(t *testing.T, e *Enforcer, name string, res []string) {
	myRes := e.getRoles(name)
	log.Print("Roles for ", name, ": ", myRes)

	if !arrayEquals(res, myRes) {
		t.Error("Roles for ", name, ": ", myRes, ", supposed to be ", res)
	}
}

func TestGetRoles(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	testGetRoles(t, e, "alice", []string{"data2_admin"})
	testGetRoles(t, e, "bob", []string{})
	testGetRoles(t, e, "data2_admin", []string{})
	testGetRoles(t, e, "non_exist", []string{})
}

func testGetPolicy(t *testing.T, e *Enforcer, res [][]string) {
	myRes := e.getPolicy()
	log.Print("Policy: ", myRes)

	if !array2DEquals(res, myRes) {
		t.Error("Policy: ", myRes, ", supposed to be ", res)
	}
}

func testGetFilteredPolicy(t *testing.T, e *Enforcer, fieldIndex int, fieldValue string, res [][]string) {
	myRes := e.getFilteredPolicy(fieldIndex, fieldValue)
	log.Print("Policy for ", fieldValue, ": ", myRes)

	if !array2DEquals(res, myRes) {
		t.Error("Policy for ", fieldValue, ": ", myRes, ", supposed to be ", res)
	}
}

func TestGetPolicy(t *testing.T) {
	e := &Enforcer{}
	e.init("examples/rbac_model.conf", "examples/rbac_policy.csv")

	testGetPolicy(t, e, [][]string{{"alice", "data1", "read"}, {"bob", "data2", "write"}, {"data2_admin", "data2", "read"}, {"data2_admin", "data2", "write"}})
	testGetFilteredPolicy(t, e, 0, "alice", [][]string{{"alice", "data1", "read"}})
	testGetFilteredPolicy(t, e, 0, "bob", [][]string{{"bob", "data2", "write"}})
	testGetFilteredPolicy(t, e, 0, "data2_admin", [][]string{{"data2_admin", "data2", "read"}, {"data2_admin", "data2", "write"}})
	testGetFilteredPolicy(t, e, 1, "data1", [][]string{{"alice", "data1", "read"}})
	testGetFilteredPolicy(t, e, 1, "data2", [][]string{{"bob", "data2", "write"}, {"data2_admin", "data2", "read"}, {"data2_admin", "data2", "write"}})
	testGetFilteredPolicy(t, e, 2, "read", [][]string{{"alice", "data1", "read"}, {"data2_admin", "data2", "read"}})
	testGetFilteredPolicy(t, e, 2, "write", [][]string{{"bob", "data2", "write"}, {"data2_admin", "data2", "write"}})
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
)

func EmptyCommonLogFormatLine() string {
	// goaccess did not accept this as an "empty" line
	//timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	//return "- - - [" + timestamp + "] \"- - -\" - -\n"
	return "\n"
}

func EmptyCombinedLogFormatLine() string {
	// goaccess did not accept this as an "empty" line
	//timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	//return "- - - [" + timestamp + "] \"- - -\" - - \"-\" \"-\"\n"
	return "\n"
}

// CommonLogFormat returns a line with the data that is available at the start
// of a request handler. The log line is in NCSA format, the same log format
// used by Apache. Fields where data is not available are indicated by a "-".
// See also: https://en.wikipedia.org/wiki/Common_Log_Format
func (ac *Config) CommonLogFormat(req *http.Request, statusCode, byteSize int) string {
	username := "-"
	if ac.perm != nil {
		username = ac.perm.UserState().Username(req)
	}
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	ip := host
	if err != nil {
		ip = req.RemoteAddr
	}
	statusCodeString := "-"
	if statusCode > 0 {
		statusCodeString = strconv.Itoa(statusCode)
	}
	byteSizeString := "0"
	if byteSize > 0 {
		byteSizeString = strconv.Itoa(byteSize)
	}
	timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %s %s", ip, username, timestamp, req.Method, req.RequestURI, req.Proto, statusCodeString, byteSizeString)
}

// CombinedLogFormat returns a line with the data that is available at the start
// of a request handler. The log line is in CLF, similar to the Common log format,
// but with two extra fields.
// See also: https://httpd.apache.org/docs/1.3/logs.html#combined
func (ac *Config) CombinedLogFormat(req *http.Request, statusCode, byteSize int) string {
	username := "-"
	if ac.perm != nil {
		username = ac.perm.UserState().Username(req)
	}
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	ip := host
	if err != nil {
		ip = req.RemoteAddr
	}
	statusCodeString := "-"
	if statusCode > 0 {
		statusCodeString = strconv.Itoa(statusCode)
	}
	byteSizeString := "0"
	if byteSize > 0 {
		byteSizeString = strconv.Itoa(byteSize)
	}
	timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	referer := req.Header.Get("Referer")
	userAgent := req.Header.Get("User-Agent")
	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %s %s \"%s\" \"%s\"", ip, username, timestamp, req.Method, req.RequestURI, req.Proto, statusCodeString, byteSizeString, referer, userAgent)
}

func (ac *Config) LogAccess(req *http.Request, statusCode, byteSize int) {
	if ac.commonAccessLogFilename != "" {
		f, err := os.OpenFile(ac.commonAccessLogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			log.Warn("Can not open " + ac.commonAccessLogFilename + ": " + err.Error())
			return
		}
		defer f.Close()
		_, err = f.WriteString(ac.CommonLogFormat(req, statusCode, byteSize) + "\n")
		if err != nil {
			log.Warn("Can not write to " + ac.commonAccessLogFilename + ": " + err.Error())
			return
		}
	}
	if ac.combinedAccessLogFilename != "" {
		f, err := os.OpenFile(ac.combinedAccessLogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			log.Warn("Can not open " + ac.combinedAccessLogFilename + ": " + err.Error())
			return
		}
		defer f.Close()
		_, err = f.WriteString(ac.CombinedLogFormat(req, statusCode, byteSize) + "\n")
		if err != nil {
			log.Warn("Can not write to " + ac.combinedAccessLogFilename + ": " + err.Error())
			return
		}
	}
}
<commit_msg>Don't start access logs with an empty line<commit_after>package engine

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
)

func EmptyCommonLogFormatLine() string {
	// goaccess did not accept this as an "empty" line
	//timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	//return "- - - [" + timestamp + "] \"- - -\" - -\n"
	return ""
}

func EmptyCombinedLogFormatLine() string {
	// goaccess did not accept this as an "empty" line
	//timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	//return "- - - [" + timestamp + "] \"- - -\" - - \"-\" \"-\"\n"
	return ""
}

// CommonLogFormat returns a line with the data that is available at the start
// of a request handler. The log line is in NCSA format, the same log format
// used by Apache. Fields where data is not available are indicated by a "-".
// See also: https://en.wikipedia.org/wiki/Common_Log_Format
func (ac *Config) CommonLogFormat(req *http.Request, statusCode, byteSize int) string {
	username := "-"
	if ac.perm != nil {
		username = ac.perm.UserState().Username(req)
	}
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	ip := host
	if err != nil {
		ip = req.RemoteAddr
	}
	statusCodeString := "-"
	if statusCode > 0 {
		statusCodeString = strconv.Itoa(statusCode)
	}
	byteSizeString := "0"
	if byteSize > 0 {
		byteSizeString = strconv.Itoa(byteSize)
	}
	timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %s %s", ip, username, timestamp, req.Method, req.RequestURI, req.Proto, statusCodeString, byteSizeString)
}

// CombinedLogFormat returns a line with the data that is available at the start
// of a request handler. The log line is in CLF, similar to the Common log format,
// but with two extra fields.
// See also: https://httpd.apache.org/docs/1.3/logs.html#combined
func (ac *Config) CombinedLogFormat(req *http.Request, statusCode, byteSize int) string {
	username := "-"
	if ac.perm != nil {
		username = ac.perm.UserState().Username(req)
	}
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	ip := host
	if err != nil {
		ip = req.RemoteAddr
	}
	statusCodeString := "-"
	if statusCode > 0 {
		statusCodeString = strconv.Itoa(statusCode)
	}
	byteSizeString := "0"
	if byteSize > 0 {
		byteSizeString = strconv.Itoa(byteSize)
	}
	timestamp := strings.Replace(time.Now().Format("02/Jan/2006 15:04:05 -0700"), " ", ":", 1)
	referer := req.Header.Get("Referer")
	userAgent := req.Header.Get("User-Agent")
	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %s %s \"%s\" \"%s\"", ip, username, timestamp, req.Method, req.RequestURI, req.Proto, statusCodeString, byteSizeString, referer, userAgent)
}

func (ac *Config) LogAccess(req *http.Request, statusCode, byteSize int) {
	if ac.commonAccessLogFilename != "" {
		f, err := os.OpenFile(ac.commonAccessLogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			log.Warn("Can not open " + ac.commonAccessLogFilename + ": " + err.Error())
			return
		}
		defer f.Close()
		_, err = f.WriteString(ac.CommonLogFormat(req, statusCode, byteSize) + "\n")
		if err != nil {
			log.Warn("Can not write to " + ac.commonAccessLogFilename + ": " + err.Error())
			return
		}
	}
	if ac.combinedAccessLogFilename != "" {
		f, err := os.OpenFile(ac.combinedAccessLogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			log.Warn("Can not open " + ac.combinedAccessLogFilename + ": " + err.Error())
			return
		}
		defer f.Close()
		_, err = f.WriteString(ac.CombinedLogFormat(req, statusCode, byteSize) + "\n")
		if err != nil {
			log.Warn("Can not write to " + ac.combinedAccessLogFilename + ": " + err.Error())
			return
		}
	}
}
<|endoftext|>


<commit_before>package engine

import (
	"flag"
	"fmt"
	"github.com/dropbox/changes-client/client"
	"github.com/dropbox/changes-client/client/adapter"
	"github.com/dropbox/changes-client/reporter"
	"log"
	"os"
	"os/signal"
	"sync"

	_ "github.com/dropbox/changes-client/adapter/basic"
	_ "github.com/dropbox/changes-client/adapter/lxc"
)

const (
	STATUS_QUEUED      = "queued"
	STATUS_IN_PROGRESS = "in_progress"
	STATUS_FINISHED    = "finished"

	RESULT_PASSED  = "passed"
	RESULT_FAILED  = "failed"
	RESULT_ABORTED = "aborted"

	SNAPSHOT_ACTIVE = "active"
	SNAPSHOT_FAILED = "failed'"
)

var (
	selectedAdapter string
	outputSnapshot  string
)

type Engine struct {
	config    *client.Config
	clientLog *client.Log
	adapter   adapter.Adapter
}

func RunBuildPlan(config *client.Config) error {
	var err error

	currentAdapter, err := adapter.Get(selectedAdapter)
	if err != nil {
		// TODO(dcramer): handle this error
		return err
	}

	engine := &Engine{
		config:    config,
		clientLog: client.NewLog(),
		adapter:   currentAdapter,
	}

	return engine.Run()
}

func (e *Engine) Run() error {
	var err error

	r := reporter.NewReporter(e.config.Server, e.config.JobstepID, e.config.Debug)
	defer r.Shutdown()

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		reportLogChunks("console", e.clientLog, r)
		wg.Done()
	}()

	r.PushJobstepStatus(STATUS_IN_PROGRESS, "")

	result := e.runBuildPlan(r)

	if result == RESULT_PASSED && outputSnapshot != "" {
		err = e.captureSnapshot()
		if err != nil {
			r.PushSnapshotImageStatus(outputSnapshot, SNAPSHOT_FAILED)
		} else {
			r.PushSnapshotImageStatus(outputSnapshot, SNAPSHOT_ACTIVE)
		}
	}

	r.PushJobstepStatus(STATUS_FINISHED, result)

	e.clientLog.Close()

	wg.Wait()

	return err
}

func (e *Engine) executeCommands(r *reporter.Reporter) string {
	var result string = RESULT_PASSED

	wg := sync.WaitGroup{}

	for _, cmdConfig := range e.config.Cmds {
		cmd, err := client.NewCommand(cmdConfig.ID, cmdConfig.Script)
		if err != nil {
			r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 255)
			result = RESULT_FAILED
			break
		}
		r.PushCommandStatus(cmd.ID, STATUS_IN_PROGRESS, -1)

		cmd.CaptureOutput = cmdConfig.CaptureOutput

		env := os.Environ()
		for k, v := range cmdConfig.Env {
			env = append(env, k+"="+v)
		}
		cmd.Env = env

		if len(cmdConfig.Cwd) > 0 {
			cmd.Cwd = cmdConfig.Cwd
		}

		cmdResult, err := e.adapter.Run(cmd, e.clientLog)

		if err != nil {
			r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 255)
			result = RESULT_FAILED
		} else {
			if cmdResult.Success {
				if cmd.CaptureOutput {
					r.PushCommandOutput(cmd.ID, STATUS_FINISHED, 0, cmdResult.Output)
				} else {
					r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 0)
				}
			} else {
				r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 1)
				result = RESULT_FAILED
			}
		}

		wg.Add(1)
		go func(artifacts []string) {
			// publishArtifacts is a synchronous operation and doesnt follow the normal queue flow of
			// other operations
			e.publishArtifacts(r, artifacts)
			wg.Done()
		}(cmdConfig.Artifacts)

		if result == RESULT_FAILED {
			break
		}
	}

	wg.Wait()

	return result
}

func (e *Engine) captureSnapshot() error {
	log.Printf("[adapter] Capturing snapshot %s", outputSnapshot)
	err := e.adapter.CaptureSnapshot(outputSnapshot, e.clientLog)
	if err != nil {
		log.Printf("[adapter] Failed to capture snapshot: %s", err.Error())
		return err
	}
	return nil
}

func (e *Engine) runBuildPlan(r *reporter.Reporter) string {
	var (
		result string
		err error
	)

	// cancellation signal
	cancel := make(chan struct{})

	// capture ctrl+c and enforce a clean shutdown
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)
	go func() {
		shuttingDown := false
		for _ = range sigchan {
			if shuttingDown {
				log.Printf("Second interrupt received. Terminating!")
				os.Exit(1)
			}

			shuttingDown = true

			log.Printf("Interrupted! Cancelling execution and cleaning up..")
			cancel <- struct{}{}
		}
	}()

	// We need to ensure that we're able to abort the build if upstream suggests
	// that it's been cancelled.
	if !e.config.Debug {
		go func() {
			um := &UpstreamMonitor{
				Config: e.config,
			}
			err := um.WaitUntilAbort()
			if err != nil {
				cancel <- struct{}{}
			}
		}()
	}

	err = e.adapter.Init(e.config)
	if err != nil {
		log.Print(fmt.Sprintf("[adapter] %s", err.Error()))
		return RESULT_FAILED
	}

	err = e.adapter.Prepare(e.clientLog)
	defer e.adapter.Shutdown(e.clientLog)
	if err != nil {
		log.Print(fmt.Sprintf("[adapter] %s", err.Error()))
		return RESULT_FAILED
	}

	// actually begin executing the build plan
	finished := make(chan struct{})
	go func() {
		result = e.executeCommands(r)
		finished <- struct{}{}
	}()

	select {
	case <-finished:
	case <-cancel:
		e.clientLog.Writeln("==> ERROR: Build was aborted by upstream")
		result = RESULT_ABORTED
	}

	return result

}

func (e *Engine) publishArtifacts(r *reporter.Reporter, artifacts []string) {
	if len(artifacts) == 0 {
		e.clientLog.Writeln("==> Skipping artifact collection")
		return
	}

	e.clientLog.Writeln(fmt.Sprintf("==> Collecting artifacts matching %s", artifacts))

	matches, err := e.adapter.CollectArtifacts(artifacts, e.clientLog)
	if err != nil {
		e.clientLog.Writeln(fmt.Sprintf("==> ERROR: " + err.Error()))
		return
	}

	e.clientLog.Writeln(fmt.Sprintf("==> Found %d matching artifacts", len(matches)))

	r.PushArtifacts(matches)
}

func reportLogChunks(name string, clientLog *client.Log, r *reporter.Reporter) {
	for chunk := range clientLog.Chan {
		r.PushLogChunk(name, chunk)
	}
}

func init() {
	flag.StringVar(&selectedAdapter, "adapter", "basic", "Adapter to run build against")
	flag.StringVar(&outputSnapshot, "save-snapshot", "", "Save the resulting container snapshot")
}
<commit_msg>Correct SNAPSHOT_FAILED value<commit_after>package engine

import (
	"flag"
	"fmt"
	"github.com/dropbox/changes-client/client"
	"github.com/dropbox/changes-client/client/adapter"
	"github.com/dropbox/changes-client/reporter"
	"log"
	"os"
	"os/signal"
	"sync"

	_ "github.com/dropbox/changes-client/adapter/basic"
	_ "github.com/dropbox/changes-client/adapter/lxc"
)

const (
	STATUS_QUEUED      = "queued"
	STATUS_IN_PROGRESS = "in_progress"
	STATUS_FINISHED    = "finished"

	RESULT_PASSED  = "passed"
	RESULT_FAILED  = "failed"
	RESULT_ABORTED = "aborted"

	SNAPSHOT_ACTIVE = "active"
	SNAPSHOT_FAILED = "failed"
)

var (
	selectedAdapter string
	outputSnapshot  string
)

type Engine struct {
	config    *client.Config
	clientLog *client.Log
	adapter   adapter.Adapter
}

func RunBuildPlan(config *client.Config) error {
	var err error

	currentAdapter, err := adapter.Get(selectedAdapter)
	if err != nil {
		// TODO(dcramer): handle this error
		return err
	}

	engine := &Engine{
		config:    config,
		clientLog: client.NewLog(),
		adapter:   currentAdapter,
	}

	return engine.Run()
}

func (e *Engine) Run() error {
	var err error

	r := reporter.NewReporter(e.config.Server, e.config.JobstepID, e.config.Debug)
	defer r.Shutdown()

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		reportLogChunks("console", e.clientLog, r)
		wg.Done()
	}()

	r.PushJobstepStatus(STATUS_IN_PROGRESS, "")

	result := e.runBuildPlan(r)

	if result == RESULT_PASSED && outputSnapshot != "" {
		err = e.captureSnapshot()
		if err != nil {
			r.PushSnapshotImageStatus(outputSnapshot, SNAPSHOT_FAILED)
		} else {
			r.PushSnapshotImageStatus(outputSnapshot, SNAPSHOT_ACTIVE)
		}
	}

	r.PushJobstepStatus(STATUS_FINISHED, result)

	e.clientLog.Close()

	wg.Wait()

	return err
}

func (e *Engine) executeCommands(r *reporter.Reporter) string {
	var result string = RESULT_PASSED

	wg := sync.WaitGroup{}

	for _, cmdConfig := range e.config.Cmds {
		cmd, err := client.NewCommand(cmdConfig.ID, cmdConfig.Script)
		if err != nil {
			r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 255)
			result = RESULT_FAILED
			break
		}
		r.PushCommandStatus(cmd.ID, STATUS_IN_PROGRESS, -1)

		cmd.CaptureOutput = cmdConfig.CaptureOutput

		env := os.Environ()
		for k, v := range cmdConfig.Env {
			env = append(env, k+"="+v)
		}
		cmd.Env = env

		if len(cmdConfig.Cwd) > 0 {
			cmd.Cwd = cmdConfig.Cwd
		}

		cmdResult, err := e.adapter.Run(cmd, e.clientLog)

		if err != nil {
			r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 255)
			result = RESULT_FAILED
		} else {
			if cmdResult.Success {
				if cmd.CaptureOutput {
					r.PushCommandOutput(cmd.ID, STATUS_FINISHED, 0, cmdResult.Output)
				} else {
					r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 0)
				}
			} else {
				r.PushCommandStatus(cmd.ID, STATUS_FINISHED, 1)
				result = RESULT_FAILED
			}
		}

		wg.Add(1)
		go func(artifacts []string) {
			// publishArtifacts is a synchronous operation and doesnt follow the normal queue flow of
			// other operations
			e.publishArtifacts(r, artifacts)
			wg.Done()
		}(cmdConfig.Artifacts)

		if result == RESULT_FAILED {
			break
		}
	}

	wg.Wait()

	return result
}

func (e *Engine) captureSnapshot() error {
	log.Printf("[adapter] Capturing snapshot %s", outputSnapshot)
	err := e.adapter.CaptureSnapshot(outputSnapshot, e.clientLog)
	if err != nil {
		log.Printf("[adapter] Failed to capture snapshot: %s", err.Error())
		return err
	}
	return nil
}

func (e *Engine) runBuildPlan(r *reporter.Reporter) string {
	var (
		result string
		err error
	)

	// cancellation signal
	cancel := make(chan struct{})

	// capture ctrl+c and enforce a clean shutdown
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)
	go func() {
		shuttingDown := false
		for _ = range sigchan {
			if shuttingDown {
				log.Printf("Second interrupt received. Terminating!")
				os.Exit(1)
			}

			shuttingDown = true

			log.Printf("Interrupted! Cancelling execution and cleaning up..")
			cancel <- struct{}{}
		}
	}()

	// We need to ensure that we're able to abort the build if upstream suggests
	// that it's been cancelled.
	if !e.config.Debug {
		go func() {
			um := &UpstreamMonitor{
				Config: e.config,
			}
			err := um.WaitUntilAbort()
			if err != nil {
				cancel <- struct{}{}
			}
		}()
	}

	err = e.adapter.Init(e.config)
	if err != nil {
		log.Print(fmt.Sprintf("[adapter] %s", err.Error()))
		return RESULT_FAILED
	}

	err = e.adapter.Prepare(e.clientLog)
	defer e.adapter.Shutdown(e.clientLog)
	if err != nil {
		log.Print(fmt.Sprintf("[adapter] %s", err.Error()))
		return RESULT_FAILED
	}

	// actually begin executing the build plan
	finished := make(chan struct{})
	go func() {
		result = e.executeCommands(r)
		finished <- struct{}{}
	}()

	select {
	case <-finished:
	case <-cancel:
		e.clientLog.Writeln("==> ERROR: Build was aborted by upstream")
		result = RESULT_ABORTED
	}

	return result

}

func (e *Engine) publishArtifacts(r *reporter.Reporter, artifacts []string) {
	if len(artifacts) == 0 {
		e.clientLog.Writeln("==> Skipping artifact collection")
		return
	}

	e.clientLog.Writeln(fmt.Sprintf("==> Collecting artifacts matching %s", artifacts))

	matches, err := e.adapter.CollectArtifacts(artifacts, e.clientLog)
	if err != nil {
		e.clientLog.Writeln(fmt.Sprintf("==> ERROR: " + err.Error()))
		return
	}

	e.clientLog.Writeln(fmt.Sprintf("==> Found %d matching artifacts", len(matches)))

	r.PushArtifacts(matches)
}

func reportLogChunks(name string, clientLog *client.Log, r *reporter.Reporter) {
	for chunk := range clientLog.Chan {
		r.PushLogChunk(name, chunk)
	}
}

func init() {
	flag.StringVar(&selectedAdapter, "adapter", "basic", "Adapter to run build against")
	flag.StringVar(&outputSnapshot, "save-snapshot", "", "Save the resulting container snapshot")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

var quit = make(chan int)

func makeDebugServer(name string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, name, ":", r.URL.Path)
	}
}

func main() {
	debug1 := http.NewServeMux()
	debug1.HandleFunc("/", makeDebugServer("first"))
	debug2 := http.NewServeMux()
	debug2.HandleFunc("/", makeDebugServer("second"))

	go http.ListenAndServe(":6061", debug1)
	go http.ListenAndServe(":6062", debug2)

	<-quit
}
<commit_msg>Name test server instances<commit_after>package main

import (
	"fmt"
	"net/http"
)

var quit = make(chan int)

func makeDebugServer(name string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, name, ":", r.URL.Path)
	}
}

func main() {
	debug1 := http.NewServeMux()
	debug1.HandleFunc("/", makeDebugServer("whitehall"))
	debug2 := http.NewServeMux()
	debug2.HandleFunc("/", makeDebugServer("frontend"))

	go http.ListenAndServe(":6061", debug1)
	go http.ListenAndServe(":6062", debug2)

	<-quit
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"syscall"
	"time"

	"github.com/aerogo/log"
	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

var colorPool = []*color.Color{
	color.New(color.FgHiGreen),
	color.New(color.FgHiYellow),
	color.New(color.FgHiBlue),
	color.New(color.FgHiMagenta),
	color.New(color.FgHiCyan),
	color.New(color.FgGreen),
}

var jobs = map[string]time.Duration{
	"anime-ratings": 10 * time.Minute,
	"test":          1 * time.Hour,
	"twist":         4 * time.Hour,
	// "refresh-episodes": 10 * time.Hour,
	// "refresh-osu":      12 * time.Hour,
	// "sync-anime":       12 * time.Hour,
	// "sync-shoboi":      24 * time.Hour,
}

func main() {
	// Start all jobs defined in the map above
	startJobs()

	// Wait for program termination
	wait()
}

func startJobs() {
	// Log paths
	logsPath := path.Join(arn.Root, "logs")
	jobLogsPath := path.Join(arn.Root, "logs", "jobs")
	os.Mkdir(jobLogsPath, 0777)

	// Scheduler log
	mainLog := log.New()
	mainLog.AddOutput(os.Stdout)
	mainLog.AddOutput(log.File(path.Join(logsPath, "scheduler.log")))
	schedulerLog := mainLog

	// Color index
	colorIndex := 0

	// Start each job
	for job, interval := range jobs {
		jobName := job
		jobInterval := interval
		executable := path.Join(arn.Root, "jobs", jobName, jobName)
		jobColor := colorPool[colorIndex].SprintFunc()

		jobLog := log.New()
		jobLog.AddOutput(log.File(path.Join(jobLogsPath, jobName+".log")))

		fmt.Printf("Registered job %s for execution every %v\n", jobColor(jobName), interval)

		go func() {
			ticker := time.NewTicker(jobInterval)
			defer ticker.Stop()

			for {
				// Wait for the given interval first
				<-ticker.C

				// Now start
				schedulerLog.Info("Starting " + jobColor(jobName))

				cmd := exec.Command(executable)
				cmd.Stdout = jobLog
				cmd.Stderr = jobLog

				err := cmd.Start()

				if err != nil {
					schedulerLog.Error("Error starting job", jobColor(jobName), err)
				}

				err = cmd.Wait()

				if err != nil {
					schedulerLog.Error("Job exited with error", jobColor(jobName), err)
				}

				schedulerLog.Info("Finished " + jobColor(jobName))
				jobLog.Info("--------------------------------------------------------------------------------")
			}
		}()

		colorIndex = (colorIndex + 1) % len(colorPool)
	}

	// Finished job registration
	println("--------------------------------------------------------------------------------")
}

func wait() {
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
	<-stop
}
<commit_msg>Updated job scheduler<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"syscall"
	"time"

	"github.com/aerogo/log"
	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

var colorPool = []*color.Color{
	color.New(color.FgHiGreen),
	color.New(color.FgHiYellow),
	color.New(color.FgHiBlue),
	color.New(color.FgHiMagenta),
	color.New(color.FgHiCyan),
	color.New(color.FgGreen),
}

var jobs = map[string]time.Duration{
	"anime-ratings": 10 * time.Minute,
	"twist":         2 * time.Hour,
	// "test":             1 * time.Hour,
	// "refresh-episodes": 10 * time.Hour,
	// "refresh-osu":      12 * time.Hour,
	// "sync-anime":       12 * time.Hour,
	// "sync-shoboi":      24 * time.Hour,
}

func main() {
	// Start all jobs defined in the map above
	startJobs()

	// Wait for program termination
	wait()
}

func startJobs() {
	// Log paths
	logsPath := path.Join(arn.Root, "logs")
	jobLogsPath := path.Join(arn.Root, "logs", "jobs")
	os.Mkdir(jobLogsPath, 0777)

	// Scheduler log
	mainLog := log.New()
	mainLog.AddOutput(os.Stdout)
	mainLog.AddOutput(log.File(path.Join(logsPath, "scheduler.log")))
	schedulerLog := mainLog

	// Color index
	colorIndex := 0

	// Start each job
	for job, interval := range jobs {
		jobName := job
		jobInterval := interval
		executable := path.Join(arn.Root, "jobs", jobName, jobName)
		jobColor := colorPool[colorIndex].SprintFunc()

		jobLog := log.New()
		jobLog.AddOutput(log.File(path.Join(jobLogsPath, jobName+".log")))

		fmt.Printf("Registered job %s for execution every %v\n", jobColor(jobName), interval)

		go func() {
			ticker := time.NewTicker(jobInterval)
			defer ticker.Stop()

			for {
				// Wait for the given interval first
				<-ticker.C

				// Now start
				schedulerLog.Info("Starting " + jobColor(jobName))

				cmd := exec.Command(executable)
				cmd.Stdout = jobLog
				cmd.Stderr = jobLog

				err := cmd.Start()

				if err != nil {
					schedulerLog.Error("Error starting job", jobColor(jobName), err)
				}

				err = cmd.Wait()

				if err != nil {
					schedulerLog.Error("Job exited with error", jobColor(jobName), err)
				}

				schedulerLog.Info("Finished " + jobColor(jobName))
				jobLog.Info("--------------------------------------------------------------------------------")
			}
		}()

		colorIndex = (colorIndex + 1) % len(colorPool)
	}

	// Finished job registration
	println("--------------------------------------------------------------------------------")
}

func wait() {
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
	<-stop
}
<|endoftext|>


<commit_before>package consensus

import (
	"errors"

	"github.com/boltdb/bolt"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errMissingFileContract = errors.New("storage proof submitted for non existing file contract")
	errOutputAlreadyMature = errors.New("delayed siacoin output is already in the matured outputs set")
	errPayoutsAlreadyPaid  = errors.New("payouts are already in the consensus set")
	errStorageProofTiming  = errors.New("missed proof triggered for file contract that is not expiring")
)

// applyMinerPayouts adds a block's miner payouts to the consensus set as
// delayed siacoin outputs.
func applyMinerPayouts(tx *bolt.Tx, pb *processedBlock) {
	for i := range pb.Block.MinerPayouts {
		mpid := pb.Block.MinerPayoutID(uint64(i))
		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffApply,
			ID:             mpid,
			SiacoinOutput:  pb.Block.MinerPayouts[i],
			MaturityHeight: pb.Height + types.MaturityDelay,
		}
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}
}

// applyMaturedSiacoinOutputs goes through the list of siacoin outputs that
// have matured and adds them to the consensus set. This also updates the block
// node diff set.
func applyMaturedSiacoinOutputs(tx *bolt.Tx, pb *processedBlock) {
	// Skip this step if the blockchain is not old enough to have maturing
	// outputs.
	if !(pb.Height >= types.MaturityDelay) {
		return
	}

	// Iterate through the list of delayed siacoin outputs. Sometimes boltdb
	// has trouble if you delete elements in a bucket while iterating through
	// the bucket (and sometimes not - nondeterministic), so all of the
	// elements are collected into an array and then deleted after the bucket
	// scan is complete.
	bucketID := append(prefixDSCO, encoding.Marshal(pb.Height)...)
	var dscods []modules.DelayedSiacoinOutputDiff
	dbErr := tx.Bucket(bucketID).ForEach(func(idBytes, scoBytes []byte) error {
		// Decode the key-value pair into an id and a siacoin output.
		var id types.SiacoinOutputID
		var sco types.SiacoinOutput
		encErr := encoding.Unmarshal(idBytes, &id)
		if build.DEBUG && encErr != nil {
			panic(encErr)
		}
		encErr = encoding.Unmarshal(scoBytes, &sco)
		if build.DEBUG && encErr != nil {
			panic(encErr)
		}

		// Sanity check - the output should not already be in siacoinOuptuts.
		if build.DEBUG && isSiacoinOutput(tx, id) {
			panic(errOutputAlreadyMature)
		}

		// Add the output to the ConsensusSet and record the diff in the
		// blockNode.
		scod := modules.SiacoinOutputDiff{
			Direction:     modules.DiffApply,
			ID:            id,
			SiacoinOutput: sco,
		}
		pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod)
		commitSiacoinOutputDiff(tx, scod, modules.DiffApply)

		// Create the dscod and add it to the list of dscods that should be
		// deleted.
		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffRevert,
			ID:             id,
			SiacoinOutput:  sco,
			MaturityHeight: pb.Height,
		}
		dscods = append(dscods, dscod)
		return nil
	})
	if build.DEBUG && dbErr != nil {
		panic(dbErr)
	}
	for _, dscod := range dscods {
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}
	deleteDSCOBucket(tx, pb.Height)
}

// applyMissedStorageProof adds the outputs and diffs that result from a file
// contract expiring.
func applyMissedStorageProof(tx *bolt.Tx, pb *processedBlock, fcid types.FileContractID) {
	// Sanity checks.
	fc, err := getFileContract(tx, fcid)
	if build.DEBUG && err != nil {
		panic(err)
	}
	if build.DEBUG {
		// Check that the file contract in question expires at pb.Height.
		if fc.WindowEnd != pb.Height {
			panic(errStorageProofTiming)
		}
	}

	// Add all of the outputs in the missed proof outputs to the consensus set.
	for i, mpo := range fc.MissedProofOutputs {
		// Sanity check - output should not already exist.
		spoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(i))
		if build.DEBUG && isSiacoinOutput(tx, spoid) {
			panic(errPayoutsAlreadyPaid)
		}

		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffApply,
			ID:             spoid,
			SiacoinOutput:  mpo,
			MaturityHeight: pb.Height + types.MaturityDelay,
		}
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}

	// Remove the file contract from the consensus set and record the diff in
	// the blockNode.
	fcd := modules.FileContractDiff{
		Direction:    modules.DiffRevert,
		ID:           fcid,
		FileContract: fc,
	}
	pb.FileContractDiffs = append(pb.FileContractDiffs, fcd)
	commitFileContractDiff(tx, fcd, modules.DiffApply)
}

// applyFileContractMaintenance looks for all of the file contracts that have
// expired without an appropriate storage proof, and calls 'applyMissedProof'
// for the file contract.
func applyFileContractMaintenance(tx *bolt.Tx, pb *processedBlock) {
	// Get the bucket pointing to all of the expiring file contracts.
	fceBucketID := append(prefixFCEX, encoding.Marshal(pb.Height)...)
	fceBucket := tx.Bucket(fceBucketID)
	// Finish if there are no expiring file contracts.
	if fceBucket == nil {
		return
	}

	err := fceBucket.ForEach(func(keyBytes, valBytes []byte) error {
		var id types.FileContractID
		copy(id[:], keyBytes)
		applyMissedStorageProof(tx, pb, id)
		return nil
	})
	if build.DEBUG && err != nil {
		panic(err)
	}
	err = tx.DeleteBucket(fceBucketID)
	if build.DEBUG && err != nil {
		panic(err)
	}
}

// applyMaintenance applies block-level alterations to the consensus set.
// Maintenance is applied after all of the transcations for the block have been
// applied.
func applyMaintenance(tx *bolt.Tx, pb *processedBlock) {
	applyMinerPayouts(tx, pb)
	applyMaturedSiacoinOutputs(tx, pb)
	applyFileContractMaintenance(tx, pb)
}
<commit_msg>correct use of ForEach for file contract deletion<commit_after>package consensus

import (
	"errors"

	"github.com/boltdb/bolt"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errMissingFileContract = errors.New("storage proof submitted for non existing file contract")
	errOutputAlreadyMature = errors.New("delayed siacoin output is already in the matured outputs set")
	errPayoutsAlreadyPaid  = errors.New("payouts are already in the consensus set")
	errStorageProofTiming  = errors.New("missed proof triggered for file contract that is not expiring")
)

// applyMinerPayouts adds a block's miner payouts to the consensus set as
// delayed siacoin outputs.
func applyMinerPayouts(tx *bolt.Tx, pb *processedBlock) {
	for i := range pb.Block.MinerPayouts {
		mpid := pb.Block.MinerPayoutID(uint64(i))
		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffApply,
			ID:             mpid,
			SiacoinOutput:  pb.Block.MinerPayouts[i],
			MaturityHeight: pb.Height + types.MaturityDelay,
		}
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}
}

// applyMaturedSiacoinOutputs goes through the list of siacoin outputs that
// have matured and adds them to the consensus set. This also updates the block
// node diff set.
func applyMaturedSiacoinOutputs(tx *bolt.Tx, pb *processedBlock) {
	// Skip this step if the blockchain is not old enough to have maturing
	// outputs.
	if !(pb.Height >= types.MaturityDelay) {
		return
	}

	// Iterate through the list of delayed siacoin outputs. Sometimes boltdb
	// has trouble if you delete elements in a bucket while iterating through
	// the bucket (and sometimes not - nondeterministic), so all of the
	// elements are collected into an array and then deleted after the bucket
	// scan is complete.
	bucketID := append(prefixDSCO, encoding.Marshal(pb.Height)...)
	var dscods []modules.DelayedSiacoinOutputDiff
	dbErr := tx.Bucket(bucketID).ForEach(func(idBytes, scoBytes []byte) error {
		// Decode the key-value pair into an id and a siacoin output.
		var id types.SiacoinOutputID
		var sco types.SiacoinOutput
		copy(id[:], idBytes)
		encErr := encoding.Unmarshal(scoBytes, &sco)
		if build.DEBUG && encErr != nil {
			panic(encErr)
		}

		// Sanity check - the output should not already be in siacoinOuptuts.
		if build.DEBUG && isSiacoinOutput(tx, id) {
			panic(errOutputAlreadyMature)
		}

		// Add the output to the ConsensusSet and record the diff in the
		// blockNode.
		scod := modules.SiacoinOutputDiff{
			Direction:     modules.DiffApply,
			ID:            id,
			SiacoinOutput: sco,
		}
		pb.SiacoinOutputDiffs = append(pb.SiacoinOutputDiffs, scod)
		commitSiacoinOutputDiff(tx, scod, modules.DiffApply)

		// Create the dscod and add it to the list of dscods that should be
		// deleted.
		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffRevert,
			ID:             id,
			SiacoinOutput:  sco,
			MaturityHeight: pb.Height,
		}
		dscods = append(dscods, dscod)
		return nil
	})
	if build.DEBUG && dbErr != nil {
		panic(dbErr)
	}
	for _, dscod := range dscods {
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}
	deleteDSCOBucket(tx, pb.Height)
}

// applyMissedStorageProof adds the outputs and diffs that result from a file
// contract expiring.
func applyMissedStorageProof(tx *bolt.Tx, pb *processedBlock, fcid types.FileContractID) (fcd modules.FileContractDiff) {
	// Sanity checks.
	fc, err := getFileContract(tx, fcid)
	if build.DEBUG && err != nil {
		panic(err)
	}
	if build.DEBUG {
		// Check that the file contract in question expires at pb.Height.
		if fc.WindowEnd != pb.Height {
			panic(errStorageProofTiming)
		}
	}

	// Add all of the outputs in the missed proof outputs to the consensus set.
	for i, mpo := range fc.MissedProofOutputs {
		// Sanity check - output should not already exist.
		spoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(i))
		if build.DEBUG && isSiacoinOutput(tx, spoid) {
			panic(errPayoutsAlreadyPaid)
		}

		dscod := modules.DelayedSiacoinOutputDiff{
			Direction:      modules.DiffApply,
			ID:             spoid,
			SiacoinOutput:  mpo,
			MaturityHeight: pb.Height + types.MaturityDelay,
		}
		pb.DelayedSiacoinOutputDiffs = append(pb.DelayedSiacoinOutputDiffs, dscod)
		commitDelayedSiacoinOutputDiff(tx, dscod, modules.DiffApply)
	}

	// Remove the file contract from the consensus set and record the diff in
	// the blockNode.
	return modules.FileContractDiff{
		Direction:    modules.DiffRevert,
		ID:           fcid,
		FileContract: fc,
	}
}

// applyFileContractMaintenance looks for all of the file contracts that have
// expired without an appropriate storage proof, and calls 'applyMissedProof'
// for the file contract.
func applyFileContractMaintenance(tx *bolt.Tx, pb *processedBlock) {
	// Get the bucket pointing to all of the expiring file contracts.
	fceBucketID := append(prefixFCEX, encoding.Marshal(pb.Height)...)
	fceBucket := tx.Bucket(fceBucketID)
	// Finish if there are no expiring file contracts.
	if fceBucket == nil {
		return
	}

	var fcds []modules.FileContractDiff
	err := fceBucket.ForEach(func(keyBytes, valBytes []byte) error {
		var id types.FileContractID
		copy(id[:], keyBytes)
		fcds = append(fcds, applyMissedStorageProof(tx, pb, id))
		return nil
	})
	if build.DEBUG && err != nil {
		panic(err)
	}
	for _, fcd := range fcds {
		pb.FileContractDiffs = append(pb.FileContractDiffs, fcd)
		commitFileContractDiff(tx, fcd, modules.DiffApply)
	}
	err = tx.DeleteBucket(fceBucketID)
	if build.DEBUG && err != nil {
		panic(err)
	}
}

// applyMaintenance applies block-level alterations to the consensus set.
// Maintenance is applied after all of the transcations for the block have been
// applied.
func applyMaintenance(tx *bolt.Tx, pb *processedBlock) {
	applyMinerPayouts(tx, pb)
	applyMaturedSiacoinOutputs(tx, pb)
	applyFileContractMaintenance(tx, pb)
}
<|endoftext|>