<commit_before>// Copyright 2016 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:generate go run gen_helper.go

package ptyy

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode/utf8"
)

var _ = fmt.Sprintf

// 医院信息
type HospitalInfo struct {
	Name      string   // 名字
	City      string   // 城市
	Keywords  []string // 关键字(可能含城市名)
	Addr      []string // 地址
	WebSite   []string // 网站
	WeiXin    []string // 微信
	WeiXinPub []string // 微信公众号
	Tel       []string // 电话
	Comment   []string // 注释
	Feedback  []string // 用户反馈
}

// 医院列表
var All []HospitalInfo = _AllHospitalInfoList

// 查询列表
func Search(query string, limits int) []HospitalInfo {
	// 规范化: 删除前后空白
	query = strings.TrimSpace(query)
	query = strings.ToLower(query)

	// 如果为空的话, 返回全部
	if query == "" {
		if limits <= 0 || limits > len(All) {
			return All
		}
		return All[:limits]
	}

	if !utf8.ValidString(query) {
		return nil
	}

	// 根据关键字查询
	if results := searchByKeywords(query, limits); len(results) > 0 {
		return results
	}

	// 如果没有匹配的, 则尝试正则查询
	if re, err := regexp.Compile(goodRegexpString(query)); err == nil {
		if results := searchByRegexp(re, limits); len(results) > 0 {
			return results
		}
	}

	// 没有匹配
	return nil
}

// 根据关键字查询
func SearchByKeywords(keywords string, limits int) []HospitalInfo {
	if !utf8.ValidString(keywords) {
		return nil
	}

	// 规范化: 删除前后空白
	keywords = strings.TrimSpace(keywords)
	keywords = strings.ToLower(keywords)

	// 如果为空的话, 返回全部
	if keywords == "" {
		if limits <= 0 || limits > len(All) {
			return All
		}
		return All[:limits]
	}

	// 根据关键字查询
	if results := searchByKeywords(keywords, limits); len(results) > 0 {
		return results
	}

	// 没有匹配
	return nil
}

// 根据关键字查询
func SearchByRegexp(query string, limits int) ([]HospitalInfo, error) {
	if !utf8.ValidString(query) {
		return nil, fmt.Errorf("ptzz: bad query: %q", query)
	}

	// 规范化: 删除前后空白
	query = strings.TrimSpace(query)

	// 如果为空的话, 返回全部
	if query == "" || query == ".*" {
		if limits <= 0 || limits > len(All) {
			return All, nil
		}
		return All[:limits], nil
	}

	// 如果没有匹配的, 则尝试正则查询
	re, err := regexp.Compile(goodRegexpString(query))
	if err != nil {
		return nil, err
	}
	if results := searchByRegexp(re, limits); len(results) > 0 {
		return results, nil
	}

	// 没有匹配
	return nil, nil
}

// 根据关键字查询
// TODO: 以后可扩展为多个关键字, 采用非字符字符分隔
func searchByKeywords(key string, limits int) (results []HospitalInfo) {
	result0Map := make(map[string]HospitalInfo)
	result1Map := make(map[string]HospitalInfo)

	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.HasPrefix(v.Name, key) || strings.HasPrefix(v.City, key) {
			result0Map[v.Name] = v
		}
		if strings.HasPrefix(_NamePinyinLongMap[v.Name], key) || strings.HasPrefix(_NamePinyinShortMap[v.Name], key) {
			result0Map[v.Name] = v
		}
		if strings.HasPrefix(_NamePinyinLongMap[v.City], key) || strings.HasPrefix(_NamePinyinShortMap[v.City], key) {
			result0Map[v.Name] = v
		}
	}
	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.Contains(v.Name, key) || strings.Contains(v.City, key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
		if strings.Contains(_NamePinyinLongMap[v.Name], key) || strings.Contains(_NamePinyinShortMap[v.Name], key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
		if strings.Contains(_NamePinyinLongMap[v.City], key) || strings.Contains(_NamePinyinShortMap[v.City], key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
	}

	var result0List []HospitalInfo
	var result1List []HospitalInfo

	for _, v := range result0Map {
		result0List = append(result0List, v)
	}
	for _, v := range result1Map {
		result1List = append(result1List, v)
	}

	sort.Sort(byHospitalInfo(result0List))
	sort.Sort(byHospitalInfo(result1List))

	results = append(results, result0List...)
	results = append(results, result1List...)

	return
}

// 根据正则表达式查询
func searchByRegexp(re *regexp.Regexp, limits int) []HospitalInfo {
	resultMap := make(map[string]HospitalInfo)

	for _, v := range All {
		if limits > 0 && len(resultMap) >= limits {
			break
		}
		if re.MatchString(v.Name) || re.MatchString(v.City) {
			resultMap[v.Name] = v
		}
		if re.MatchString(_NamePinyinLongMap[v.Name]) || re.MatchString(_NamePinyinShortMap[v.Name]) {
			resultMap[v.Name] = v
		}
		if re.MatchString(_NamePinyinLongMap[v.City]) || re.MatchString(_NamePinyinShortMap[v.City]) {
			resultMap[v.Name] = v
		}
	}

	var resultList []HospitalInfo
	for _, v := range resultMap {
		resultList = append(resultList, v)
	}
	sort.Sort(byHospitalInfo(resultList))
	return resultList
}

// 规范化正则
func goodRegexpString(re string) string {
	mapRune := [][2]rune{
		{'，', ','},
		{'。', '.'},
		{'：', ':'},
		{'－', '-'},
		{'＋', '+'},
		{'＊', '*'},
		{'？', '?'},
		{'^', '^'},
		{'¥', '$'},
		{'、', '\\'},
		{'｜', '|'},
		{'［', '['},
		{'］', ']'},
		{'｛', '{'},
		{'｝', '}'},
		{'（', '('},
		{'）', ')'},
		{'《', '<'},
		{'》', '>'},
	}
	runes := []rune(re)
	for i := 0; i < len(runes); i++ {
		for j := 0; j < len(mapRune); j++ {
			if runes[i] == mapRune[j][0] {
				runes[i] = mapRune[j][1]
			}
		}
	}
	return string(runes)
}

// 按unicode排序
type byHospitalInfo []HospitalInfo

func (d byHospitalInfo) Len() int           { return len(d) }
func (d byHospitalInfo) Less(i, j int) bool { return d[i].Name < d[j].Name }
func (d byHospitalInfo) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
<commit_msg>优化含数字的查询<commit_after>// Copyright 2016 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:generate go run gen_helper.go

package ptyy

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"unicode/utf8"
)

var _ = fmt.Sprintf

// 医院信息
type HospitalInfo struct {
	Name      string   // 名字
	City      string   // 城市
	Keywords  []string // 关键字(可能含城市名)
	Addr      []string // 地址
	WebSite   []string // 网站
	WeiXin    []string // 微信
	WeiXinPub []string // 微信公众号
	Tel       []string // 电话
	Comment   []string // 注释
	Feedback  []string // 用户反馈
}

// 医院列表
var All []HospitalInfo = _AllHospitalInfoList

// 查询列表
func Search(query string, limits int) []HospitalInfo {
	// 规范化: 删除前后空白
	query = strings.TrimSpace(query)
	query = strings.ToLower(query)

	// 如果为空的话, 返回全部
	if query == "" {
		if limits <= 0 || limits > len(All) {
			return All
		}
		return All[:limits]
	}

	if !utf8.ValidString(query) {
		return nil
	}

	// 如果是纯数字, 单独处理
	if sNum, sNumZh, ok := adjustDigitString(query); ok {
		return searchByDigitString(sNum, sNumZh, limits)
	}

	// 根据关键字查询
	if results := searchByKeywords(query, limits); len(results) > 0 {
		return results
	}

	// 如果没有匹配的, 则尝试正则查询
	if re, err := regexp.Compile(goodRegexpString(query)); err == nil {
		if results := searchByRegexp(re, limits); len(results) > 0 {
			return results
		}
	}

	// 没有匹配
	return nil
}

// 根据关键字查询
func SearchByKeywords(keywords string, limits int) []HospitalInfo {
	if !utf8.ValidString(keywords) {
		return nil
	}

	// 规范化: 删除前后空白
	keywords = strings.TrimSpace(keywords)
	keywords = strings.ToLower(keywords)

	// 如果为空的话, 返回全部
	if keywords == "" {
		if limits <= 0 || limits > len(All) {
			return All
		}
		return All[:limits]
	}

	// 根据关键字查询
	if results := searchByKeywords(keywords, limits); len(results) > 0 {
		return results
	}

	// 没有匹配
	return nil
}

// 根据关键字查询
func SearchByRegexp(query string, limits int) ([]HospitalInfo, error) {
	if !utf8.ValidString(query) {
		return nil, fmt.Errorf("ptzz: bad query: %q", query)
	}

	// 规范化: 删除前后空白
	query = strings.TrimSpace(query)

	// 如果为空的话, 返回全部
	if query == "" || query == ".*" {
		if limits <= 0 || limits > len(All) {
			return All, nil
		}
		return All[:limits], nil
	}

	// 如果没有匹配的, 则尝试正则查询
	re, err := regexp.Compile(goodRegexpString(query))
	if err != nil {
		return nil, err
	}
	if results := searchByRegexp(re, limits); len(results) > 0 {
		return results, nil
	}

	// 没有匹配
	return nil, nil
}

// 根据关键字查询
// TODO: 以后可扩展为多个关键字, 采用非字符字符分隔
func searchByKeywords(key string, limits int) (results []HospitalInfo) {
	result0Map := make(map[string]HospitalInfo)
	result1Map := make(map[string]HospitalInfo)

	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.HasPrefix(v.Name, key) || strings.HasPrefix(v.City, key) {
			result0Map[v.Name] = v
		}
		if strings.HasPrefix(_NamePinyinLongMap[v.Name], key) || strings.HasPrefix(_NamePinyinShortMap[v.Name], key) {
			result0Map[v.Name] = v
		}
		if strings.HasPrefix(_NamePinyinLongMap[v.City], key) || strings.HasPrefix(_NamePinyinShortMap[v.City], key) {
			result0Map[v.Name] = v
		}
	}
	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.Contains(v.Name, key) || strings.Contains(v.City, key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
		if strings.Contains(_NamePinyinLongMap[v.Name], key) || strings.Contains(_NamePinyinShortMap[v.Name], key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
		if strings.Contains(_NamePinyinLongMap[v.City], key) || strings.Contains(_NamePinyinShortMap[v.City], key) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
	}

	var result0List []HospitalInfo
	var result1List []HospitalInfo

	for _, v := range result0Map {
		result0List = append(result0List, v)
	}
	for _, v := range result1Map {
		result1List = append(result1List, v)
	}

	sort.Sort(byHospitalInfo(result0List))
	sort.Sort(byHospitalInfo(result1List))

	results = append(results, result0List...)
	results = append(results, result1List...)

	return
}

// 数字查询
func searchByDigitString(sNum, sNumZh string, limits int) (results []HospitalInfo) {
	result0Map := make(map[string]HospitalInfo)
	result1Map := make(map[string]HospitalInfo)

	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.HasPrefix(v.Name, sNum) || strings.HasPrefix(v.Name, sNumZh) {
			result0Map[v.Name] = v
		}
	}
	for _, v := range All {
		if limits > 0 && len(result0Map)+len(result1Map) >= limits {
			break
		}
		if strings.Contains(v.Name, sNum) || strings.Contains(v.Name, sNumZh) {
			if _, ok := result0Map[v.Name]; !ok {
				result1Map[v.Name] = v
			}
		}
	}

	var result0List []HospitalInfo
	var result1List []HospitalInfo

	for _, v := range result0Map {
		result0List = append(result0List, v)
	}
	for _, v := range result1Map {
		result1List = append(result1List, v)
	}

	sort.Sort(byHospitalInfo(result0List))
	sort.Sort(byHospitalInfo(result1List))

	results = append(results, result0List...)
	results = append(results, result1List...)

	return
}

// 根据正则表达式查询
func searchByRegexp(re *regexp.Regexp, limits int) []HospitalInfo {
	resultMap := make(map[string]HospitalInfo)

	for _, v := range All {
		if limits > 0 && len(resultMap) >= limits {
			break
		}
		if re.MatchString(v.Name) || re.MatchString(v.City) {
			resultMap[v.Name] = v
		}
		if re.MatchString(_NamePinyinLongMap[v.Name]) || re.MatchString(_NamePinyinShortMap[v.Name]) {
			resultMap[v.Name] = v
		}
		if re.MatchString(_NamePinyinLongMap[v.City]) || re.MatchString(_NamePinyinShortMap[v.City]) {
			resultMap[v.Name] = v
		}
	}

	var resultList []HospitalInfo
	for _, v := range resultMap {
		resultList = append(resultList, v)
	}
	sort.Sort(byHospitalInfo(resultList))
	return resultList
}

// 规范化正则
func goodRegexpString(re string) string {
	mapRune := [][2]rune{
		{'，', ','},
		{'。', '.'},
		{'：', ':'},
		{'－', '-'},
		{'＋', '+'},
		{'＊', '*'},
		{'？', '?'},
		{'^', '^'},
		{'¥', '$'},
		{'、', '\\'},
		{'｜', '|'},
		{'［', '['},
		{'］', ']'},
		{'｛', '{'},
		{'｝', '}'},
		{'（', '('},
		{'）', ')'},
		{'《', '<'},
		{'》', '>'},
	}
	runes := []rune(re)
	for i := 0; i < len(runes); i++ {
		for j := 0; j < len(mapRune); j++ {
			if runes[i] == mapRune[j][0] {
				runes[i] = mapRune[j][1]
			}
		}
	}
	return string(runes)
}

// 按unicode排序
type byHospitalInfo []HospitalInfo

func (d byHospitalInfo) Len() int           { return len(d) }
func (d byHospitalInfo) Less(i, j int) bool { return d[i].Name < d[j].Name }
func (d byHospitalInfo) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"strings"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/uploader"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

func doPush(buildPath string, spec string) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFileID)
		if err != nil {
			return err
		}

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewMultipartUpload(newPatchRes.File.UploadURL,
		newPatchRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	signatureWriter, err := uploader.NewMultipartUpload(newSignatureRes.File.UploadURL,
		newSignatureRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container

	comm.Debugf("Waiting for source container")
	select {
	case err := <-walkErrs:
		return err
	case sourceContainer = <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		break
	}

	comm.Logf("")
	comm.Opf("Pushing %s (%s)", humanize.Bytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes) * 0.97)

		// input bytes that aren't in output, for esxample:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > 10*1024 {
			comm.ProgressLabel(fmt.Sprintf("%s left", humanize.Bytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)
	}
	patchWriter.OnProgress = updateProgress

	stateConsumer := &pwr.StateConsumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		SourcePath:      buildPath,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return err
	}

	err = patchWriter.Close()
	if err != nil {
		return err
	}

	err = signatureWriter.Close()
	if err != nil {
		return err
	}

	for c := 0; c < 2; c++ {
		select {
		case err := <-uploadErrs:
			return err
		case <-uploadDone:
		}
	}
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- err
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return err
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

		comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
	}
	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType)
		if err != nil {
			errs <- err
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)
		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	container, err := tlc.Walk(path, filterPaths)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Don't deadlock while trying to flush buffered uploader<commit_after>package main

import (
	"fmt"
	"io"
	"strings"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/uploader"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

func doPush(buildPath string, spec string) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFileID)
		if err != nil {
			return err
		}

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewMultipartUpload(newPatchRes.File.UploadURL,
		newPatchRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	signatureWriter, err := uploader.NewMultipartUpload(newSignatureRes.File.UploadURL,
		newSignatureRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container

	comm.Debugf("Waiting for source container")
	select {
	case err := <-walkErrs:
		return err
	case sourceContainer = <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		break
	}

	comm.Logf("")
	comm.Opf("Pushing %s (%s)", humanize.Bytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes) * 0.97)

		// input bytes that aren't in output, for esxample:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > 10*1024 {
			comm.ProgressLabel(fmt.Sprintf("%s left", humanize.Bytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)
	}
	patchWriter.OnProgress = updateProgress

	stateConsumer := &pwr.StateConsumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		SourcePath:      buildPath,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return err
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		err := c.Close()
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case err := <-uploadErrs:
			return err
		case <-uploadDone:
		}
	}
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return err
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

		comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
	}
	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType)
		if err != nil {
			errs <- err
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)
		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	container, err := tlc.Walk(path, filterPaths)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/websocket"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/akrennmair/cabinet/data"
	"github.com/golang/protobuf/proto"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"
)

func dispatchEvents(events <-chan *data.Event, replRequests <-chan replRequest) {
	subscribers := make(map[chan<- *data.Event]struct{})

	for {
		select {
		case e := <-events:
			log.Printf("event: %v", e)
			for ch := range subscribers {
				ch <- e
			}
			log.Printf("finished notifying subscribers")
		case r := <-replRequests:
			switch r.Type {
			case Subscribe:
				subscribers[r.Events] = struct{}{}
			case Unsubscribe:
				delete(subscribers, r.Events)
			}
		}
	}
}

type replicator struct {
	ParentServer string
	DB           *leveldb.DB
	Username     string
	Password     string
}

func (r *replicator) replicate() {
	count := 0
	for {
		ts := time.Now()
		err := r.replicateUntilError()
		if err != nil {
			log.Printf("Replication error: %v", err)
			if time.Since(ts) < 5*time.Second {
				if count < 5 {
					count++
				}
			} else {
				count = 0
			}
			if count > 0 {
				backoffTime := time.Duration(math.Pow(2, float64(count))) * time.Second
				log.Printf("Backing off for %s", backoffTime)
				time.Sleep(backoffTime)
			}
		}
	}
}

func (r *replicator) replicateUntilError() error {
	uri := r.ParentServer + "/api/repl"
	parsedURL, err := url.Parse(uri)
	if err != nil {
		log.Printf("parsing %s failed: %v", uri, err)
		return err
	}
	switch parsedURL.Scheme {
	case "http":
		parsedURL.Scheme = "ws"
	case "https":
		parsedURL.Scheme = "wss"
	default:
		return fmt.Errorf("unknown URL scheme %q", parsedURL.Scheme)
	}

	cfg, err := websocket.NewConfig(parsedURL.String(), "http://localhost/")
	if err != nil {
		log.Printf("creating websocket config failed: %v", err)
		return err
	}
	cfg.Location = parsedURL
	cfg.Header.Set("Authorization", "Basic "+basicAuthEncode(r.Username, r.Password))

	ws, err := websocket.DialConfig(cfg)
	if err != nil {
		log.Printf("connecting to %s failed: %v", parsedURL, err)
		return err
	}
	defer ws.Close()

	latestEvent, err := r.DB.Get([]byte("latest_event"), nil)
	if err != nil {
		latestEvent = []byte("event:0")
	}

	var replStart data.ReplicationStart
	replStart.Event = proto.String(string(latestEvent))

	rawReplStartMsg, err := proto.Marshal(&replStart)
	if err != nil {
		log.Printf("marshalling replication start message failed: %v", err)
		return err
	}

	if err := websocket.Message.Send(ws, rawReplStartMsg); err != nil {
		log.Printf("sending replication start message failed: %v", err)
		return err
	}

	for {
		var rawMsg []byte
		if err := websocket.Message.Receive(ws, &rawMsg); err != nil {
			log.Printf("receiving event failed: %v", err)
			return err
		}

		var event data.Event
		if err := proto.Unmarshal(rawMsg, &event); err != nil {
			log.Printf("unmarshalling event failed: %v", err)
			return err
		}

		if haveEvent, _ := r.DB.Has([]byte(event.GetId()), nil); haveEvent {
			log.Printf("ignoring duplicate event %s", event.GetId())
			continue
		}

		batch := new(leveldb.Batch)
		batch.Put([]byte(event.GetId()), rawMsg)
		batch.Put([]byte("latest_event"), []byte(event.GetId()))

		switch event.GetType() {
		case data.Event_UPLOAD:
			fileContent, mimeType, err := r.downloadFile(r.ParentServer + "/" + event.GetDrawer() + "/" + event.GetFilename())
			if err != nil {
				log.Printf("Error downloading %s:%s, ignoring file: %v", event.GetDrawer(), event.GetFilename(), err)
			} else {
				batch.Put([]byte("file:"+event.GetDrawer()+":"+event.GetFilename()), fileContent)

				var metadata data.MetaData
				metadata.ContentType = proto.String(mimeType)
				rawMetaData, err := proto.Marshal(&metadata)
				if err != nil {
					log.Printf("marshalling meta data failed: %v", err)
					return err
				}

				batch.Put([]byte("meta:"+event.GetDrawer()+":"+event.GetFilename()), rawMetaData)
			}
		case data.Event_DELETE:
			batch.Delete([]byte("file:" + event.GetDrawer() + ":" + event.GetFilename()))
		default:
			return fmt.Errorf("unknown event type %d", event.GetType())
		}

		if err := r.DB.Write(batch, nil); err != nil {
			log.Printf("writing replicated event to database failed: %v", err)
			return err
		}

		log.Printf("replicated %s to %s:%s", event.GetId(), event.GetDrawer(), event.GetFilename())
	}

	return nil
}

func (r *replicator) downloadFile(uri string) (content []byte, contentType string, err error) {
	resp, err := http.Get(uri)
	if err != nil {
		return nil, "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("%s returned %d", uri, resp.StatusCode)
	}
	content, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", err
	}
	contentType = resp.Header.Get("Content-Type")
	return content, contentType, nil
}

type replHandler struct {
	DB         *leveldb.DB
	Username   string
	Password   string
	Replicator chan<- replRequest
}

type replRequest struct {
	Type   replRequestType
	Events chan<- *data.Event
}

type replRequestType int

const (
	Subscribe replRequestType = iota
	Unsubscribe
)

func (h *replHandler) handleWebsocket(conn *websocket.Conn) {
	if !basicAuth(nil, conn.Request(), h.Username, h.Password) {
		return
	}

	var msg []byte

	if err := websocket.Message.Receive(conn, &msg); err != nil {
		log.Printf("Receiving from replication websocket failed: %v", err)
		return
	}

	var replStart data.ReplicationStart
	if err := proto.Unmarshal(msg, &replStart); err != nil {
		log.Printf("Decoding ReplicationStart failed: %v", err)
		return
	}

	if !strings.HasPrefix(replStart.GetEvent(), "event:") {
		log.Printf("Got invalid event: %s", replStart.GetEvent())
		return
	}

	/*
		this whole replication code works like this:

		We first register to receive events to replicate them to this connected child.

		We then start a goroutine to cache incoming events (later referred to as
		"caching goroutine") until we're finished with catching
		up missing events.

		We then start another goroutine ("forwarding goroutine") that first catches
		up with missing events by iterating through all events since the last event
		the child got. We then switch over to forwarding events that have been
		cached by the caching goroutine.

		We then start a third goroutine ("receiving goroutine") that waits for the
		websocket to receive a message (which will never happen) or for the
		connection to close. If the connection is closed, the receivin goroutine
		closes the quit channel to indicate that replication work needs to be
		stopped.

		The handleWebsocket method waits for the quit signal, and then returns. Due
		to a defer, the channel through which we receive events gets unregistered,
		and the rawEvents channel gets closed, as well. This makes the forwarding
		goroutine end its operation. The quit signal also ends the caching goroutine.

	*/
	events := make(chan *data.Event, 1)
	rawEvents := make(chan []byte)

	defer close(rawEvents)

	h.Replicator <- replRequest{Type: Subscribe, Events: events}

	defer func() {
		h.Replicator <- replRequest{Type: Unsubscribe, Events: events}
		close(events)
	}()

	quit := make(chan bool)

	go cacheEvents(events, rawEvents, quit)

	go func() {
		iterator := h.DB.NewIterator(&util.Range{Start: []byte(replStart.GetEvent()), Limit: []byte("f")}, nil)

		for iterator.Next() {
			eventData := iterator.Value()
			if err := websocket.Message.Send(conn, eventData); err != nil {
				log.Printf("Sending event failed: %v", err)
				return
			}
		}

		for event := range rawEvents {
			if err := websocket.Message.Send(conn, event); err != nil {
				log.Printf("Sending event failed: %v", err)
				return
			}
		}

		log.Printf("stopped sending events to client")
	}()

	go func() {
		defer close(quit)
		for {
			var inbuf []byte
			if err := websocket.Message.Receive(conn, &inbuf); err != nil {
				log.Printf("client has disconnected: %v", err)
				return
			}
		}
	}()

	<-quit
	log.Printf("handleWebsocket: received signal to stop replicating to client")
}

func cacheEvents(incomingEvents <-chan *data.Event, outgoingEvents chan<- []byte, quit <-chan bool) {
	cachedEvents := [][]byte{}

	for {
		// if there are currently no cached events, only attempt to receive and listen for
		// the quit signal.
		if len(cachedEvents) == 0 {
			select {
			case e := <-incomingEvents:
				rawData, err := proto.Marshal(e)
				if err != nil {
					return
				}
				cachedEvents = append(cachedEvents, rawData)
			case <-quit:
				return
			}
		} else {
			// if there are cached events, attempt to receive, send and listen for the quit
			// signal.
			select {
			case e := <-incomingEvents:
				rawData, err := proto.Marshal(e)
				if err != nil {
					return
				}
				cachedEvents = append(cachedEvents, rawData)
			case outgoingEvents <- cachedEvents[0]:
				cachedEvents = cachedEvents[1:]
			case <-quit:
				return
			}
		}
	}
}
<commit_msg>fix go vet and golint warnings.<commit_after>package main

import (
	"fmt"
	"golang.org/x/net/websocket"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/akrennmair/cabinet/data"
	"github.com/golang/protobuf/proto"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"
)

func dispatchEvents(events <-chan *data.Event, replRequests <-chan replRequest) {
	subscribers := make(map[chan<- *data.Event]struct{})

	for {
		select {
		case e := <-events:
			log.Printf("event: %v", e)
			for ch := range subscribers {
				ch <- e
			}
			log.Printf("finished notifying subscribers")
		case r := <-replRequests:
			switch r.Type {
			case subscribe:
				subscribers[r.Events] = struct{}{}
			case unsubscribe:
				delete(subscribers, r.Events)
			}
		}
	}
}

type replicator struct {
	ParentServer string
	DB           *leveldb.DB
	Username     string
	Password     string
}

func (r *replicator) replicate() {
	count := 0
	for {
		ts := time.Now()
		err := r.replicateUntilError()
		if err != nil {
			log.Printf("Replication error: %v", err)
			if time.Since(ts) < 5*time.Second {
				if count < 5 {
					count++
				}
			} else {
				count = 0
			}
			if count > 0 {
				backoffTime := time.Duration(math.Pow(2, float64(count))) * time.Second
				log.Printf("Backing off for %s", backoffTime)
				time.Sleep(backoffTime)
			}
		}
	}
}

func (r *replicator) replicateUntilError() error {
	uri := r.ParentServer + "/api/repl"
	parsedURL, err := url.Parse(uri)
	if err != nil {
		log.Printf("parsing %s failed: %v", uri, err)
		return err
	}
	switch parsedURL.Scheme {
	case "http":
		parsedURL.Scheme = "ws"
	case "https":
		parsedURL.Scheme = "wss"
	default:
		return fmt.Errorf("unknown URL scheme %q", parsedURL.Scheme)
	}

	cfg, err := websocket.NewConfig(parsedURL.String(), "http://localhost/")
	if err != nil {
		log.Printf("creating websocket config failed: %v", err)
		return err
	}
	cfg.Location = parsedURL
	cfg.Header.Set("Authorization", "Basic "+basicAuthEncode(r.Username, r.Password))

	ws, err := websocket.DialConfig(cfg)
	if err != nil {
		log.Printf("connecting to %s failed: %v", parsedURL, err)
		return err
	}
	defer ws.Close()

	latestEvent, err := r.DB.Get([]byte("latest_event"), nil)
	if err != nil {
		latestEvent = []byte("event:0")
	}

	var replStart data.ReplicationStart
	replStart.Event = proto.String(string(latestEvent))

	rawReplStartMsg, err := proto.Marshal(&replStart)
	if err != nil {
		log.Printf("marshalling replication start message failed: %v", err)
		return err
	}

	if err := websocket.Message.Send(ws, rawReplStartMsg); err != nil {
		log.Printf("sending replication start message failed: %v", err)
		return err
	}

	for {
		var rawMsg []byte
		if err := websocket.Message.Receive(ws, &rawMsg); err != nil {
			log.Printf("receiving event failed: %v", err)
			return err
		}

		var event data.Event
		if err := proto.Unmarshal(rawMsg, &event); err != nil {
			log.Printf("unmarshalling event failed: %v", err)
			return err
		}

		if haveEvent, _ := r.DB.Has([]byte(event.GetId()), nil); haveEvent {
			log.Printf("ignoring duplicate event %s", event.GetId())
			continue
		}

		batch := new(leveldb.Batch)
		batch.Put([]byte(event.GetId()), rawMsg)
		batch.Put([]byte("latest_event"), []byte(event.GetId()))

		switch event.GetType() {
		case data.Event_UPLOAD:
			fileContent, mimeType, err := r.downloadFile(r.ParentServer + "/" + event.GetDrawer() + "/" + event.GetFilename())
			if err != nil {
				log.Printf("Error downloading %s:%s, ignoring file: %v", event.GetDrawer(), event.GetFilename(), err)
			} else {
				batch.Put([]byte("file:"+event.GetDrawer()+":"+event.GetFilename()), fileContent)

				var metadata data.MetaData
				metadata.ContentType = proto.String(mimeType)
				rawMetaData, err := proto.Marshal(&metadata)
				if err != nil {
					log.Printf("marshalling meta data failed: %v", err)
					return err
				}

				batch.Put([]byte("meta:"+event.GetDrawer()+":"+event.GetFilename()), rawMetaData)
			}
		case data.Event_DELETE:
			batch.Delete([]byte("file:" + event.GetDrawer() + ":" + event.GetFilename()))
		default:
			return fmt.Errorf("unknown event type %d", event.GetType())
		}

		if err := r.DB.Write(batch, nil); err != nil {
			log.Printf("writing replicated event to database failed: %v", err)
			return err
		}

		log.Printf("replicated %s to %s:%s", event.GetId(), event.GetDrawer(), event.GetFilename())
	}
}

func (r *replicator) downloadFile(uri string) (content []byte, contentType string, err error) {
	resp, err := http.Get(uri)
	if err != nil {
		return nil, "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("%s returned %d", uri, resp.StatusCode)
	}
	content, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", err
	}
	contentType = resp.Header.Get("Content-Type")
	return content, contentType, nil
}

type replHandler struct {
	DB         *leveldb.DB
	Username   string
	Password   string
	Replicator chan<- replRequest
}

type replRequest struct {
	Type   replRequestType
	Events chan<- *data.Event
}

type replRequestType int

const (
	subscribe replRequestType = iota
	unsubscribe
)

func (h *replHandler) handleWebsocket(conn *websocket.Conn) {
	if !basicAuth(nil, conn.Request(), h.Username, h.Password) {
		return
	}

	var msg []byte

	if err := websocket.Message.Receive(conn, &msg); err != nil {
		log.Printf("Receiving from replication websocket failed: %v", err)
		return
	}

	var replStart data.ReplicationStart
	if err := proto.Unmarshal(msg, &replStart); err != nil {
		log.Printf("Decoding ReplicationStart failed: %v", err)
		return
	}

	if !strings.HasPrefix(replStart.GetEvent(), "event:") {
		log.Printf("Got invalid event: %s", replStart.GetEvent())
		return
	}

	/*
		this whole replication code works like this:

		We first register to receive events to replicate them to this connected child.

		We then start a goroutine to cache incoming events (later referred to as
		"caching goroutine") until we're finished with catching
		up missing events.

		We then start another goroutine ("forwarding goroutine") that first catches
		up with missing events by iterating through all events since the last event
		the child got. We then switch over to forwarding events that have been
		cached by the caching goroutine.

		We then start a third goroutine ("receiving goroutine") that waits for the
		websocket to receive a message (which will never happen) or for the
		connection to close. If the connection is closed, the receivin goroutine
		closes the quit channel to indicate that replication work needs to be
		stopped.

		The handleWebsocket method waits for the quit signal, and then returns. Due
		to a defer, the channel through which we receive events gets unregistered,
		and the rawEvents channel gets closed, as well. This makes the forwarding
		goroutine end its operation. The quit signal also ends the caching goroutine.

	*/
	events := make(chan *data.Event, 1)
	rawEvents := make(chan []byte)

	defer close(rawEvents)

	h.Replicator <- replRequest{Type: subscribe, Events: events}

	defer func() {
		h.Replicator <- replRequest{Type: unsubscribe, Events: events}
		close(events)
	}()

	quit := make(chan bool)

	go cacheEvents(events, rawEvents, quit)

	go func() {
		iterator := h.DB.NewIterator(&util.Range{Start: []byte(replStart.GetEvent()), Limit: []byte("f")}, nil)

		for iterator.Next() {
			eventData := iterator.Value()
			if err := websocket.Message.Send(conn, eventData); err != nil {
				log.Printf("Sending event failed: %v", err)
				return
			}
		}

		for event := range rawEvents {
			if err := websocket.Message.Send(conn, event); err != nil {
				log.Printf("Sending event failed: %v", err)
				return
			}
		}

		log.Printf("stopped sending events to client")
	}()

	go func() {
		defer close(quit)
		for {
			var inbuf []byte
			if err := websocket.Message.Receive(conn, &inbuf); err != nil {
				log.Printf("client has disconnected: %v", err)
				return
			}
		}
	}()

	<-quit
	log.Printf("handleWebsocket: received signal to stop replicating to client")
}

func cacheEvents(incomingEvents <-chan *data.Event, outgoingEvents chan<- []byte, quit <-chan bool) {
	cachedEvents := [][]byte{}

	for {
		// if there are currently no cached events, only attempt to receive and listen for
		// the quit signal.
		if len(cachedEvents) == 0 {
			select {
			case e := <-incomingEvents:
				rawData, err := proto.Marshal(e)
				if err != nil {
					return
				}
				cachedEvents = append(cachedEvents, rawData)
			case <-quit:
				return
			}
		} else {
			// if there are cached events, attempt to receive, send and listen for the quit
			// signal.
			select {
			case e := <-incomingEvents:
				rawData, err := proto.Marshal(e)
				if err != nil {
					return
				}
				cachedEvents = append(cachedEvents, rawData)
			case outgoingEvents <- cachedEvents[0]:
				cachedEvents = cachedEvents[1:]
			case <-quit:
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package req

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"time"
)

// Req represents a request with it's response
type Resp struct {
	r        *Req
	req      *http.Request
	resp     *http.Response
	client   *http.Client
	reqBody  []byte
	respBody []byte
	cost     time.Duration
}

func (r *Resp) getReqBody() io.ReadCloser {
	if r.reqBody == nil {
		return nil
	}
	return ioutil.NopCloser(bytes.NewReader(r.reqBody))
}

// Cost returns time spent by the request
func (r *Resp) Cost() time.Duration {
	return r.cost
}

// Request returns *http.Request
func (r *Resp) Request() *http.Request {
	return r.req
}

// Response returns *http.Response
func (r *Resp) Response() *http.Response {
	return r.resp
}

// Bytes returns response body as []byte
func (r *Resp) Bytes() []byte {
	return r.respBody
}

// String returns response body as string
func (r *Resp) String() string {
	return string(r.respBody)
}

// ToJSON convert json response body to struct or map
func (r *Resp) ToJSON(v interface{}) error {
	return json.Unmarshal(r.respBody, v)
}

// ToXML convert xml response body to struct or map
func (r *Resp) ToXML(v interface{}) error {
	return xml.Unmarshal(r.respBody, v)
}

// ToFile download the response body to file
func (r *Resp) ToFile(name string) error {
	file, err := os.Create(name)
	if err != nil {
		return err
	}
	_, err = io.Copy(file, r.resp.Body)
	if err != nil {
		return err
	}
	return nil
}

var regNewline = regexp.MustCompile(`\n|\r`)

func (r *Resp) Format(s fmt.State, verb rune) {
	if r == nil || r.req == nil {
		return
	}
	req := r.req
	if s.Flag('+') { // include header and format pretty.
		fmt.Fprint(s, r.dump())
	} else if s.Flag('-') { // keep all informations in one line.
		fmt.Fprint(s, req.Method, " ", req.URL.String())
		if r.r.flag&Lcost != 0 {
			fmt.Fprint(s, " ", r.cost.String())
		}
		if r.r.flag&LreqBody != 0 {
			if len(r.reqBody) > 0 {
				str := regNewline.ReplaceAllString(string(r.reqBody), " ")
				fmt.Fprint(s, " ", str)
			} else {
				fmt.Fprint(s, " ******")
			}
		}
		if r.r.flag&LrespBody != 0 {
			if len(r.respBody) > 0 {
				str := regNewline.ReplaceAllString(string(r.respBody), " ")
				fmt.Fprint(s, " ", str)
			} else {
				fmt.Fprint(s, " ******")
			}
		}
	} else { // auto
		fmt.Fprint(s, req.Method, " ", req.URL.String())
		if r.r.flag&Lcost != 0 {
			fmt.Fprint(s, " ", r.cost.String())
		}
		respBody := r.respBody
		if (len(r.reqBody) > 0 && regNewline.Match(r.reqBody)) || (len(respBody) > 0 && regNewline.Match(respBody)) { // pretty format
			if len(r.reqBody) > 0 {
				fmt.Fprint(s, "\n", string(r.reqBody))
			}
			if len(respBody) > 0 {
				fmt.Fprint(s, "\n", string(respBody))
			}
		} else {
			if len(r.reqBody) > 0 {
				fmt.Fprint(s, " ", string(r.reqBody))
			}
			if len(respBody) > 0 {
				fmt.Fprint(s, " ", string(respBody))
			}
		}
	}

}
<commit_msg>Fix typo<commit_after>package req

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"time"
)

// Resp represents a request with it's response
type Resp struct {
	r        *Req
	req      *http.Request
	resp     *http.Response
	client   *http.Client
	reqBody  []byte
	respBody []byte
	cost     time.Duration
}

func (r *Resp) getReqBody() io.ReadCloser {
	if r.reqBody == nil {
		return nil
	}
	return ioutil.NopCloser(bytes.NewReader(r.reqBody))
}

// Cost returns time spent by the request
func (r *Resp) Cost() time.Duration {
	return r.cost
}

// Request returns *http.Request
func (r *Resp) Request() *http.Request {
	return r.req
}

// Response returns *http.Response
func (r *Resp) Response() *http.Response {
	return r.resp
}

// Bytes returns response body as []byte
func (r *Resp) Bytes() []byte {
	return r.respBody
}

// String returns response body as string
func (r *Resp) String() string {
	return string(r.respBody)
}

// ToJSON convert json response body to struct or map
func (r *Resp) ToJSON(v interface{}) error {
	return json.Unmarshal(r.respBody, v)
}

// ToXML convert xml response body to struct or map
func (r *Resp) ToXML(v interface{}) error {
	return xml.Unmarshal(r.respBody, v)
}

// ToFile download the response body to file
func (r *Resp) ToFile(name string) error {
	file, err := os.Create(name)
	if err != nil {
		return err
	}
	_, err = io.Copy(file, r.resp.Body)
	if err != nil {
		return err
	}
	return nil
}

var regNewline = regexp.MustCompile(`\n|\r`)

func (r *Resp) Format(s fmt.State, verb rune) {
	if r == nil || r.req == nil {
		return
	}
	req := r.req
	if s.Flag('+') { // include header and format pretty.
		fmt.Fprint(s, r.dump())
	} else if s.Flag('-') { // keep all informations in one line.
		fmt.Fprint(s, req.Method, " ", req.URL.String())
		if r.r.flag&Lcost != 0 {
			fmt.Fprint(s, " ", r.cost.String())
		}
		if r.r.flag&LreqBody != 0 {
			if len(r.reqBody) > 0 {
				str := regNewline.ReplaceAllString(string(r.reqBody), " ")
				fmt.Fprint(s, " ", str)
			} else {
				fmt.Fprint(s, " ******")
			}
		}
		if r.r.flag&LrespBody != 0 {
			if len(r.respBody) > 0 {
				str := regNewline.ReplaceAllString(string(r.respBody), " ")
				fmt.Fprint(s, " ", str)
			} else {
				fmt.Fprint(s, " ******")
			}
		}
	} else { // auto
		fmt.Fprint(s, req.Method, " ", req.URL.String())
		if r.r.flag&Lcost != 0 {
			fmt.Fprint(s, " ", r.cost.String())
		}
		respBody := r.respBody
		if (len(r.reqBody) > 0 && regNewline.Match(r.reqBody)) || (len(respBody) > 0 && regNewline.Match(respBody)) { // pretty format
			if len(r.reqBody) > 0 {
				fmt.Fprint(s, "\n", string(r.reqBody))
			}
			if len(respBody) > 0 {
				fmt.Fprint(s, "\n", string(respBody))
			}
		} else {
			if len(r.reqBody) > 0 {
				fmt.Fprint(s, " ", string(r.reqBody))
			}
			if len(respBody) > 0 {
				fmt.Fprint(s, " ", string(respBody))
			}
		}
	}

}
<|endoftext|>


<commit_before>package dicebot

import (
	"fmt"
	"github.com/arkie/dicebot/roll"
	"github.com/arkie/dicebot/slack"
	"math/rand"
	"os"
	"strconv"
	"time"
)

func init() {
	r := os.Getenv("ROLL_COMMAND")
	if r == "" {
		r = "roll"
	}
	slack.Register(r, command)
}

func formatRoll(id string, mini bool, results []*roll.Dice) slack.D {
	var (
		color    string
		fields   []slack.D
		final    int
		text     string
		fallback string
	)
	for i, result := range results {
		if i == 0 {
			if result.Operator == "-" {
				final -= result.Total
			} else {
				final = result.Total
			}
			text = fmt.Sprint("*", final, "*")
			fallback = fmt.Sprint(final)
			if result.For != "" {
				if result.Sides > 1 {
					text += fmt.Sprint(" = *", result.Total, "*")
					fallback += fmt.Sprint(" = ", result.Total)
				}
				text += fmt.Sprint(" for *", result.For, "*")
				fallback += fmt.Sprint(" for ", result.For)
			}
		} else {
			op := result.Operator
			switch result.Operator {
			case roll.Add:
				final += result.Total
			case roll.Subtract:
				final -= result.Total
			case roll.Multiply:
				final *= result.Total
				op = "×"
			case roll.Divide:
				final /= result.Total
			case roll.Max:
				if result.Total > final {
					final = result.Total
				}
			case roll.Min:
				if result.Total < final {
					final = result.Total
				}
			}
			text += fmt.Sprint(" ", op, " *", result.Total, "*")
			fallback += fmt.Sprint(" ", result.Operator, " ", result.Total)
			if result.For != "" {
				text += fmt.Sprint(" for *", result.For, "*")
				fallback += fmt.Sprint(" for ", result.For)
			}
			if i == len(results)-1 {
				text += fmt.Sprint(" = *", final, "*")
				fallback += fmt.Sprint(" = ", final)
			}
		}
		if result.Sides <= 1 {
			continue
		}
		rollText := fmt.Sprint(result.Rolls)
		if result.Fudge {
			if result.Total > 0 {
				color = "good"
			} else if result.Total == 0 {
				color = "warning"
			} else {
				color = "danger"
			}
		} else {
			single := result.Number * result.Sides / 3.0
			if result.Total > single*2 {
				color = "good"
			} else if result.Total > single+result.Number-1 {
				color = "warning"
			} else {
				color = "danger"
			}
		}
		dice := fmt.Sprint(result.Number, "d", result.Sides)
		if result.Fudge {
			dice = fmt.Sprint(result.Number, "df")
		}
		fields = append(fields, slack.D{
			"title": "Dice",
			"value": dice,
			"short": true,
		}, slack.D{
			"title": "Rolls",
			"value": rollText[1 : len(rollText)-1],
			"short": true,
		})
		if result.Minimum != 0 {
			count := 0
			for _, r := range result.Rolls {
				if r > result.Minimum {
					count++
				}
			}
			fields = append(fields, slack.D{
				"title": "Minimum",
				"value": strconv.Itoa(result.Minimum),
				"short": true,
			}, slack.D{
				"title": "Over",
				"value": strconv.Itoa(count),
				"short": true,
			})
		}
		if result.Maximum != 0 {
			count := 0
			for _, r := range result.Rolls {
				if r < result.Maximum {
					count++
				}
			}
			fields = append(fields, slack.D{
				"title": "Maximum",
				"value": strconv.Itoa(result.Maximum),
				"short": true,
			}, slack.D{
				"title": "Under",
				"value": strconv.Itoa(count),
				"short": true,
			})
		}
		if result.Keep != 0 {
			removed := fmt.Sprint(result.Removed)
			fields = append(fields, slack.D{
				"title": "Keep",
				"value": strconv.Itoa(result.Keep),
				"short": true,
			}, slack.D{
				"title": "Removed",
				"value": removed[1 : len(removed)-1],
				"short": true,
			})
		}
	}
	if mini {
		fields = []slack.D{}
	}
	return slack.D{
		"response_type": "in_channel",
		"attachments": []slack.D{
			{
				"fallback": fmt.Sprint("<@", id, "> rolled ", fallback),
				"text":     fmt.Sprint("<@", id, "> rolled ", text),
				// TODO: Color just uses the last color chosen.
				"color":     color,
				"fields":    fields,
				"mrkdwn_in": []string{"text"},
			},
		},
	}
}

func command(args slack.Args) slack.D {
	rand.Seed(time.Now().UnixNano())
	mini := len(args.Text) > 4 && args.Text[:4] == "mini"
	result := roll.Parse(args.Text)
	for _, roll := range result {
		roll.Roll()
	}
	return formatRoll(args.UserID, mini, result)
}
<commit_msg>Now with code written after reading the code<commit_after>package dicebot

import (
	"fmt"
	"github.com/arkie/dicebot/roll"
	"github.com/arkie/dicebot/slack"
	"math/rand"
	"os"
	"strconv"
	"time"
)

func init() {
	r := os.Getenv("ROLL_COMMAND")
	if r == "" {
		r = "roll"
	}
	slack.Register(r, command)
}

func formatRoll(id string, mini bool, results []*roll.Dice) slack.D {
	var (
		color        string
		fields       []slack.D
		final        int
		text         string
		fallback     string
		runningTotal int
		rollCount    int
	)
	for i, result := range results {
		if i == 0 {
			if result.Operator == "-" {
				final -= result.Total
			} else {
				final = result.Total
			}
			text = fmt.Sprint("*", final, "*")
			fallback = fmt.Sprint(final)
			if result.For != "" {
				text += fmt.Sprint(" for *", result.For, "*")
				fallback += fmt.Sprint(" for ", result.For)
			} else {
				runningTotal += result.Total
				rollCount++
			}
		} else {
			op := result.Operator
			switch result.Operator {
			case roll.Add:
				final += result.Total
			case roll.Subtract:
				final -= result.Total
			case roll.Multiply:
				final *= result.Total
				op = "×"
			case roll.Divide:
				final /= result.Total
			case roll.Max:
				if result.Total > final {
					final = result.Total
				}
			case roll.Min:
				if result.Total < final {
					final = result.Total
				}
			}
			text += fmt.Sprint(" ", op, " *", result.Total, "*")
			fallback += fmt.Sprint(" ", result.Operator, " ", result.Total)
			if result.For != "" {
				if rollCount > 1 {
					rollCount = 0
					text += fmt.Sprint(" = *", runningTotal, "*")
					fallback += fmt.Sprint(" = ", runningTotal)
					runningTotal = 0
				}
				text += fmt.Sprint(" for *", result.For, "*")
				fallback += fmt.Sprint(" for ", result.For)
			}
			if i == len(results)-1 {
				text += fmt.Sprint(" = *", final, "*")
				fallback += fmt.Sprint(" = ", final)
			}
		}
		if result.Sides <= 1 {
			continue
		}
		rollText := fmt.Sprint(result.Rolls)
		if result.Fudge {
			if result.Total > 0 {
				color = "good"
			} else if result.Total == 0 {
				color = "warning"
			} else {
				color = "danger"
			}
		} else {
			single := result.Number * result.Sides / 3.0
			if result.Total > single*2 {
				color = "good"
			} else if result.Total > single+result.Number-1 {
				color = "warning"
			} else {
				color = "danger"
			}
		}
		dice := fmt.Sprint(result.Number, "d", result.Sides)
		if result.Fudge {
			dice = fmt.Sprint(result.Number, "df")
		}
		fields = append(fields, slack.D{
			"title": "Dice",
			"value": dice,
			"short": true,
		}, slack.D{
			"title": "Rolls",
			"value": rollText[1 : len(rollText)-1],
			"short": true,
		})
		if result.Minimum != 0 {
			count := 0
			for _, r := range result.Rolls {
				if r > result.Minimum {
					count++
				}
			}
			fields = append(fields, slack.D{
				"title": "Minimum",
				"value": strconv.Itoa(result.Minimum),
				"short": true,
			}, slack.D{
				"title": "Over",
				"value": strconv.Itoa(count),
				"short": true,
			})
		}
		if result.Maximum != 0 {
			count := 0
			for _, r := range result.Rolls {
				if r < result.Maximum {
					count++
				}
			}
			fields = append(fields, slack.D{
				"title": "Maximum",
				"value": strconv.Itoa(result.Maximum),
				"short": true,
			}, slack.D{
				"title": "Under",
				"value": strconv.Itoa(count),
				"short": true,
			})
		}
		if result.Keep != 0 {
			removed := fmt.Sprint(result.Removed)
			fields = append(fields, slack.D{
				"title": "Keep",
				"value": strconv.Itoa(result.Keep),
				"short": true,
			}, slack.D{
				"title": "Removed",
				"value": removed[1 : len(removed)-1],
				"short": true,
			})
		}
	}
	if mini {
		fields = []slack.D{}
	}
	return slack.D{
		"response_type": "in_channel",
		"attachments": []slack.D{
			{
				"fallback": fmt.Sprint("<@", id, "> rolled ", fallback),
				"text":     fmt.Sprint("<@", id, "> rolled ", text),
				// TODO: Color just uses the last color chosen.
				"color":     color,
				"fields":    fields,
				"mrkdwn_in": []string{"text"},
			},
		},
	}
}

func command(args slack.Args) slack.D {
	rand.Seed(time.Now().UnixNano())
	mini := len(args.Text) > 4 && args.Text[:4] == "mini"
	result := roll.Parse(args.Text)
	for _, roll := range result {
		roll.Roll()
	}
	return formatRoll(args.UserID, mini, result)
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     July 26, 2019
;       related version of root zone:     2019072602
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>auto-update<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     July 30, 2019
;       related version of root zone:     2019073000
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     August 17, 2021
;       related version of root zone:     2021081701
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Fri Sep 24 08:19:29 UTC 2021<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     September 23, 2021
;       related version of root zone:     2021092301
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package golangNeo4jBoltDriver

import (
	"database/sql/driver"
	"fmt"
	"io"

	"github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding"
	"github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages"
)

// Rows represents results of rows from the DB
//
// Implements sql/driver, but also includes its own more neo-friendly interface.
// Some of the features of this interface implement neo-specific features
// unavailable in the sql/driver compatible interface
//
// Row objects ARE NOT THREAD SAFE.
// If you want to use multiple go routines with these objects,
// you should use a driver to create a new conn for each routine.
type Rows interface {
	Columns() []string
	Close() error
	Next(dest []driver.Value) error
}

type boltRows struct {
	closed    bool
	metadata  map[string]interface{}
	statement *boltStmt
	consumed  bool
}

func newRows(metadata map[string]interface{}) *boltRows {
	return &boltRows{
		metadata: metadata,
	}
}

// Columns returns the columns from the result
func (r *boltRows) Columns() []string {
	fieldsInt, ok := r.metadata["fields"]
	if !ok {
		return []string{}
	}

	fields, ok := fieldsInt.([]string)
	if !ok {
		Logger.Printf("Unrecognized fields from success message: %T %#v", fieldsInt, fieldsInt)
		return []string{}
	}

	return fields
}

// Close closes the rows
func (r *boltRows) Close() error {
	if r.closed {
		return nil
	}
	r.closed = true
	r.statement.rows = nil
	return nil
}

// Next gets the next row result
func (r *boltRows) Next(dest []driver.Value) error {
	if r.closed {
		return fmt.Errorf("Rows are already closed")
	}

	pullMessage := messages.NewPullAllMessage()
	err := encoding.NewEncoder(r.statement.conn, r.statement.conn.chunkSize).Encode(pullMessage)
	if err != nil {
		Logger.Printf("An error occurred encoding pull all query: %s", err)
		return fmt.Errorf("An error occurred encoding pull all query: %s", err)
	}

	respInt, err := encoding.NewDecoder(r.statement.conn).Decode()
	if err != nil {
		Logger.Printf("An error occurred decoding pull all query response: %s", err)
		return fmt.Errorf("An error occurred decoding pull all query response: %s", err)
	}

	switch resp := respInt.(type) {
	case messages.SuccessMessage:
		Logger.Printf("Got success message: %#v", resp)
		r.consumed = true
		return io.EOF
	case messages.FailureMessage:
		Logger.Printf("Got failure message: %#v", resp)
		err := r.statement.conn.ackFailure(resp)
		if err != nil {
			Logger.Printf("An error occurred acking failure: %s", err)
		}
		return fmt.Errorf("Got failure message: %#v", resp)
	case messages.RecordMessage:
		Logger.Printf("Got record message: %#v", resp)
		dest = make([]driver.Value, len(resp.Fields))
		for i, item := range resp.Fields {
			dest[i] = item
		}
		// TODO: Implement conversion to driver.Value
		return nil
	default:
		return fmt.Errorf("Unrecognized response type: %T Value: %#v", resp, resp)
	}
}

// NextNeo gets the next row result
func (r *boltRows) NextNeo(dest interface{}) (map[string]interface{}, error) {
	if r.closed {
		return nil, fmt.Errorf("Rows are already closed")
	}

	pullMessage := messages.NewPullAllMessage()
	err := encoding.NewEncoder(r.statement.conn, r.statement.conn.chunkSize).Encode(pullMessage)
	if err != nil {
		Logger.Printf("An error occurred encoding pull all query: %s", err)
		return nil, fmt.Errorf("An error occurred encoding pull all query: %s", err)
	}

	respInt, err := encoding.NewDecoder(r.statement.conn).Decode()
	if err != nil {
		Logger.Printf("An error occurred decoding pull all query response: %s", err)
		return nil, fmt.Errorf("An error occurred decoding pull all query response: %s", err)
	}

	switch resp := respInt.(type) {
	case messages.SuccessMessage:
		Logger.Printf("Got success message: %#v", resp)
		r.consumed = true
		return resp.Metadata, io.EOF
	case messages.FailureMessage:
		Logger.Printf("Got failure message: %#v", resp)
		err := r.statement.conn.ackFailure(resp)
		if err != nil {
			Logger.Printf("An error occurred acking failure: %s", err)
		}
		return nil, fmt.Errorf("Got failure message: %#v", resp)
	case messages.RecordMessage:
		Logger.Printf("Got record message: %#v", resp)
		dest = resp.Fields
		// TODO: Implement conversion to driver.Value
		return nil, nil
	default:
		return nil, fmt.Errorf("Unrecognized response type: %T Value: %#v", resp, resp)
	}
}
<commit_msg>Adding discard all to rows if they aren't fully consumed.<commit_after>package golangNeo4jBoltDriver

import (
	"database/sql/driver"
	"fmt"
	"io"

	"github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding"
	"github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages"
)

// Rows represents results of rows from the DB
//
// Implements sql/driver, but also includes its own more neo-friendly interface.
// Some of the features of this interface implement neo-specific features
// unavailable in the sql/driver compatible interface
//
// Row objects ARE NOT THREAD SAFE.
// If you want to use multiple go routines with these objects,
// you should use a driver to create a new conn for each routine.
type Rows interface {
	Columns() []string
	Close() error
	Next(dest []driver.Value) error
}

type boltRows struct {
	closed    bool
	metadata  map[string]interface{}
	statement *boltStmt
	consumed  bool
}

func newRows(metadata map[string]interface{}) *boltRows {
	return &boltRows{
		metadata: metadata,
	}
}

// Columns returns the columns from the result
func (r *boltRows) Columns() []string {
	fieldsInt, ok := r.metadata["fields"]
	if !ok {
		return []string{}
	}

	fields, ok := fieldsInt.([]string)
	if !ok {
		Logger.Printf("Unrecognized fields from success message: %T %#v", fieldsInt, fieldsInt)
		return []string{}
	}

	return fields
}

// Close closes the rows
func (r *boltRows) Close() error {
	if r.closed {
		return nil
	}

	// Discard all messages if not consumed
	if !r.consumed {

		discardMessage := messages.NewDiscardAllMessage()
		err := encoding.NewEncoder(r.statement.conn, r.statement.conn.chunkSize).Encode(discardMessage)
		if err != nil {
			Logger.Printf("An error occurred encoding discard all query: %s", err)
			return fmt.Errorf("An error occurred encoding discard all query: %s", err)
		}

		respInt, err := encoding.NewDecoder(r.statement.conn).Decode()
		if err != nil {
			Logger.Printf("An error occurred decoding discard all query response: %s", err)
			return fmt.Errorf("An error occurred decoding discard all query response: %s", err)
		}

		switch resp := respInt.(type) {
		case messages.SuccessMessage:
			Logger.Printf("Got success message: %#v", resp)
		case messages.FailureMessage:
			Logger.Printf("Got failure message: %#v", resp)
			err := r.statement.conn.ackFailure(resp)
			if err != nil {
				Logger.Printf("An error occurred acking failure: %s", err)
			}
			return fmt.Errorf("Got failure message: %#v", resp)
		default:
			return fmt.Errorf("Unrecognized response type: %T Value: %#v", resp, resp)
		}

	}

	r.closed = true
	r.statement.rows = nil
	return nil
}

// Next gets the next row result
func (r *boltRows) Next(dest []driver.Value) error {
	if r.closed {
		return fmt.Errorf("Rows are already closed")
	}

	pullMessage := messages.NewPullAllMessage()
	err := encoding.NewEncoder(r.statement.conn, r.statement.conn.chunkSize).Encode(pullMessage)
	if err != nil {
		Logger.Printf("An error occurred encoding pull all query: %s", err)
		return fmt.Errorf("An error occurred encoding pull all query: %s", err)
	}

	respInt, err := encoding.NewDecoder(r.statement.conn).Decode()
	if err != nil {
		Logger.Printf("An error occurred decoding pull all query response: %s", err)
		return fmt.Errorf("An error occurred decoding pull all query response: %s", err)
	}

	switch resp := respInt.(type) {
	case messages.SuccessMessage:
		Logger.Printf("Got success message: %#v", resp)
		r.consumed = true
		return io.EOF
	case messages.FailureMessage:
		Logger.Printf("Got failure message: %#v", resp)
		err := r.statement.conn.ackFailure(resp)
		if err != nil {
			Logger.Printf("An error occurred acking failure: %s", err)
		}
		return fmt.Errorf("Got failure message: %#v", resp)
	case messages.RecordMessage:
		Logger.Printf("Got record message: %#v", resp)
		dest = make([]driver.Value, len(resp.Fields))
		for i, item := range resp.Fields {
			dest[i] = item
		}
		// TODO: Implement conversion to driver.Value
		return nil
	default:
		return fmt.Errorf("Unrecognized response type: %T Value: %#v", resp, resp)
	}
}

// NextNeo gets the next row result
func (r *boltRows) NextNeo(dest interface{}) (map[string]interface{}, error) {
	if r.closed {
		return nil, fmt.Errorf("Rows are already closed")
	}

	pullMessage := messages.NewPullAllMessage()
	err := encoding.NewEncoder(r.statement.conn, r.statement.conn.chunkSize).Encode(pullMessage)
	if err != nil {
		Logger.Printf("An error occurred encoding pull all query: %s", err)
		return nil, fmt.Errorf("An error occurred encoding pull all query: %s", err)
	}

	respInt, err := encoding.NewDecoder(r.statement.conn).Decode()
	if err != nil {
		Logger.Printf("An error occurred decoding pull all query response: %s", err)
		return nil, fmt.Errorf("An error occurred decoding pull all query response: %s", err)
	}

	switch resp := respInt.(type) {
	case messages.SuccessMessage:
		Logger.Printf("Got success message: %#v", resp)
		r.consumed = true
		return resp.Metadata, io.EOF
	case messages.FailureMessage:
		Logger.Printf("Got failure message: %#v", resp)
		err := r.statement.conn.ackFailure(resp)
		if err != nil {
			Logger.Printf("An error occurred acking failure: %s", err)
		}
		return nil, fmt.Errorf("Got failure message: %#v", resp)
	case messages.RecordMessage:
		Logger.Printf("Got record message: %#v", resp)
		dest = resp.Fields
		// TODO: Implement conversion to driver.Value
		return nil, nil
	default:
		return nil, fmt.Errorf("Unrecognized response type: %T Value: %#v", resp, resp)
	}
}
<|endoftext|>


<commit_before>package saml2

import (
	"sync"
	"time"

	dsig "github.com/russellhaering/goxmldsig"
)

type SAMLServiceProvider struct {
	IdentityProviderSSOURL string
	IdentityProviderIssuer string

	AssertionConsumerServiceURL string
	ServiceProviderIssuer       string

	SignAuthnRequests              bool
	SignAuthnRequestsAlgorithm     string
	SignAuthnRequestsCanonicalizer dsig.Canonicalizer

	// RequestedAuthnContext allows service providers to require that the identity
	// provider use specific authentication mechanisms. Leaving this unset will
	// permit the identity provider to choose the auth method. To maximize compatibility
	// with identity providers it is recommended to leave this unset.
	RequestedAuthnContext   *RequestedAuthnContext
	AudienceURI             string
	IDPCertificateStore     dsig.X509CertificateStore
	SPKeyStore              dsig.X509KeyStore
	NameIdFormat            string
	SkipSignatureValidation bool
	AllowMissingAttributes  bool
	Clock                   *dsig.Clock
	signingContextMu        sync.RWMutex
	signingContext          *dsig.SigningContext
}

// RequestedAuthnContext controls which authentication mechanisms are requested of
// the identity provider. It is generally sufficient to omit this and let the
// identity provider select an authentication mechansim.
type RequestedAuthnContext struct {
	// The RequestedAuthnContext comparison policy to use. See the section 3.3.2.2.1
	// of the SAML 2.0 specification for details. Constants named AuthnPolicyMatch*
	// contain standardized values.
	Comparison string

	// Contexts will be passed as AuthnContextClassRefs. For example, to force password
	// authentication on some identity providers, Contexts should have a value of
	// []string{AuthnContextPasswordProtectedTransport}, and Comparison should have a
	// value of AuthnPolicyMatchExact.
	Contexts []string
}

func (sp *SAMLServiceProvider) SigningContext() *dsig.SigningContext {
	sp.signingContextMu.RLock()
	signingContext := sp.signingContext
	sp.signingContextMu.RUnlock()

	if signingContext != nil {
		return signingContext
	}

	sp.signingContextMu.Lock()
	defer sp.signingContextMu.Unlock()

	sp.signingContext = dsig.NewDefaultSigningContext(sp.SPKeyStore)
	sp.signingContext.SetSignatureMethod(sp.SignAuthnRequestsAlgorithm)
	if sp.SignAuthnRequestsCanonicalizer != nil {
		sp.signingContext.Canonicalizer = sp.SignAuthnRequestsCanonicalizer
	}

	return sp.signingContext
}

type ProxyRestriction struct {
	Count    int
	Audience []string
}

type WarningInfo struct {
	OneTimeUse       bool
	ProxyRestriction *ProxyRestriction
	NotInAudience    bool
	InvalidTime      bool
}

type AssertionInfo struct {
	NameID              string
	Values              Values
	WarningInfo         *WarningInfo
	AuthnInstant        *time.Time
	SessionNotOnOrAfter *time.Time
}
<commit_msg>Added SAMLServiceProvider.Metadata(), SAMLServiceProvider.SPSigningKeyStore<commit_after>package saml2

import (
	"encoding/base64"
	"sync"
	"time"

	"github.com/russellhaering/gosaml2/types"
	dsig "github.com/russellhaering/goxmldsig"
	dsigtypes "github.com/russellhaering/goxmldsig/types"
)

type ErrSaml struct {
	Message string
	System  error
}

func (serr ErrSaml) Error() string {
	if serr.Message != "" {
		return serr.Message
	}
	return "SAML error"
}

type SAMLServiceProvider struct {
	IdentityProviderSSOURL string
	IdentityProviderIssuer string

	AssertionConsumerServiceURL string
	ServiceProviderIssuer       string

	SignAuthnRequests              bool
	SignAuthnRequestsAlgorithm     string
	SignAuthnRequestsCanonicalizer dsig.Canonicalizer

	// RequestedAuthnContext allows service providers to require that the identity
	// provider use specific authentication mechanisms. Leaving this unset will
	// permit the identity provider to choose the auth method. To maximize compatibility
	// with identity providers it is recommended to leave this unset.
	RequestedAuthnContext   *RequestedAuthnContext
	AudienceURI             string
	IDPCertificateStore     dsig.X509CertificateStore
	SPKeyStore              dsig.X509KeyStore // Required encryption key, default signing key
	SPSigningKeyStore       dsig.X509KeyStore // Optional signing key
	NameIdFormat            string
	SkipSignatureValidation bool
	AllowMissingAttributes  bool
	Clock                   *dsig.Clock
	signingContextMu        sync.RWMutex
	signingContext          *dsig.SigningContext
}

// RequestedAuthnContext controls which authentication mechanisms are requested of
// the identity provider. It is generally sufficient to omit this and let the
// identity provider select an authentication mechansim.
type RequestedAuthnContext struct {
	// The RequestedAuthnContext comparison policy to use. See the section 3.3.2.2.1
	// of the SAML 2.0 specification for details. Constants named AuthnPolicyMatch*
	// contain standardized values.
	Comparison string

	// Contexts will be passed as AuthnContextClassRefs. For example, to force password
	// authentication on some identity providers, Contexts should have a value of
	// []string{AuthnContextPasswordProtectedTransport}, and Comparison should have a
	// value of AuthnPolicyMatchExact.
	Contexts []string
}

func (sp *SAMLServiceProvider) Metadata() (*types.EntityDescriptor, error) {
	signingCertBytes, err := sp.GetSigningCertBytes()
	if err != nil {
		return nil, err
	}
	encryptionCertBytes, err := sp.GetEncryptionCertBytes()
	if err != nil {
		return nil, err
	}
	return &types.EntityDescriptor{
		ValidUntil: time.Now().UTC().Add(time.Hour * 24 * 7), // 7 days
		EntityID:   sp.ServiceProviderIssuer,
		SPSSODescriptor: types.SPSSODescriptor{
			AuthnRequestsSigned:        sp.SignAuthnRequests,
			WantAssertionsSigned:       !sp.SkipSignatureValidation,
			ProtocolSupportEnumeration: SAMLProtocolNamespace,
			KeyDescriptors: []types.KeyDescriptor{
				{
					Use: "signing",
					KeyInfo: dsigtypes.KeyInfo{
						X509Data: dsigtypes.X509Data{
							X509Certificate: dsigtypes.X509Certificate{
								Data: base64.StdEncoding.EncodeToString(signingCertBytes),
							},
						},
					},
				},
				{
					Use: "encryption",
					KeyInfo: dsigtypes.KeyInfo{
						X509Data: dsigtypes.X509Data{
							X509Certificate: dsigtypes.X509Certificate{
								Data: base64.StdEncoding.EncodeToString(encryptionCertBytes),
							},
						},
					},
					EncryptionMethods: []types.EncryptionMethod{
						{Algorithm: types.MethodAES128CBC},
						{Algorithm: types.MethodAES128GCM},
					},
				},
			},
			AssertionConsumerServices: []types.IndexedEndpoint{{
				Binding:  BindingHttpPost,
				Location: sp.AssertionConsumerServiceURL,
				Index:    1,
			}},
		},
	}, nil
}

func (sp *SAMLServiceProvider) GetEncryptionKey() dsig.X509KeyStore {
	return sp.SPKeyStore
}

func (sp *SAMLServiceProvider) GetSigningKey() dsig.X509KeyStore {
	if sp.SPSigningKeyStore == nil {
		return sp.GetEncryptionKey() // Default is signing key is same as encryption key
	}
	return sp.SPSigningKeyStore
}

func (sp *SAMLServiceProvider) GetEncryptionCertBytes() ([]byte, error) {
	if _, encryptionCert, err := sp.GetEncryptionKey().GetKeyPair(); err != nil {
		return nil, ErrSaml{Message: "no SP encryption certificate", System: err}
	} else if len(encryptionCert) < 1 {
		return nil, ErrSaml{Message: "empty SP encryption certificate"}
	} else {
		return encryptionCert, nil
	}
}

func (sp *SAMLServiceProvider) GetSigningCertBytes() ([]byte, error) {
	if _, signingCert, err := sp.GetSigningKey().GetKeyPair(); err != nil {
		return nil, ErrSaml{Message: "no SP signing certificate", System: err}
	} else if len(signingCert) < 1 {
		return nil, ErrSaml{Message: "empty SP signing certificate"}
	} else {
		return signingCert, nil
	}
}

func (sp *SAMLServiceProvider) SigningContext() *dsig.SigningContext {
	sp.signingContextMu.RLock()
	signingContext := sp.signingContext
	sp.signingContextMu.RUnlock()

	if signingContext != nil {
		return signingContext
	}

	sp.signingContextMu.Lock()
	defer sp.signingContextMu.Unlock()

	sp.signingContext = dsig.NewDefaultSigningContext(sp.GetSigningKey())
	sp.signingContext.SetSignatureMethod(sp.SignAuthnRequestsAlgorithm)
	if sp.SignAuthnRequestsCanonicalizer != nil {
		sp.signingContext.Canonicalizer = sp.SignAuthnRequestsCanonicalizer
	}

	return sp.signingContext
}

type ProxyRestriction struct {
	Count    int
	Audience []string
}

type WarningInfo struct {
	OneTimeUse       bool
	ProxyRestriction *ProxyRestriction
	NotInAudience    bool
	InvalidTime      bool
}

type AssertionInfo struct {
	NameID              string
	Values              Values
	WarningInfo         *WarningInfo
	AuthnInstant        *time.Time
	SessionNotOnOrAfter *time.Time
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/Tonkpils/snag/vow"
	fsn "gopkg.in/fsnotify.v1"
)

var (
	mtimes = map[string]time.Time{}
)

const (
	GoExt = ".go"
)

type Bob struct {
	w        *fsn.Watcher
	done     chan struct{}
	watching map[string]struct{}
}

func NewBuilder() (*Bob, error) {
	w, err := fsn.NewWatcher()
	if err != nil {
		return nil, err
	}

	return &Bob{
		w:        w,
		done:     make(chan struct{}),
		watching: map[string]struct{}{},
	}, nil
}

func (b *Bob) Close() {
	b.w.Close()
	close(b.done)
}

func (b *Bob) Watch() error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case ev := <-b.w.Events:
				fmt.Println(ev)
				var queueBuild bool
				switch {
				case isCreate(ev.Op):
					fmt.Println("Creating")
					queueBuild = b.watch(ev.Name)
				case isDelete(ev.Op):
					fmt.Println("Deleteing")
					if _, ok := b.watching[ev.Name]; ok {
						b.w.Remove(ev.Name)
						delete(b.watching, ev.Name)
					}
					queueBuild = true
				case isModify(ev.Op):
					queueBuild = true
				}
				if queueBuild {
					b.maybeQueue(ev.Name)
				}
			case err := <-b.w.Errors:
				log.Println("error:", err)
			}
		}
	}()
	b.watch(wd)

	<-b.done

	return nil
}

func (b *Bob) maybeQueue(path string) {
	if filepath.Ext(path) != GoExt {
		return
	}

	// clear scroll buffer
	print("\033c")

	// setup list of commands
	v := vow.To("go", "build", "./...")
	v.Then("go", "vet", "./...")
	v.Then("go", "test", "./...")

	stat, err := os.Stat(path)
	if err == nil {
		mtime := stat.ModTime()
		lasttime := mtimes[path]
		if !mtime.Equal(lasttime) {
			mtimes[path] = mtime
			v.Exec(os.Stdout)
		}
	} else {
		log.Println(err)
		delete(mtimes, path)
		v.Exec(os.Stdout)
	}
}

func (b *Bob) watch(path string) bool {
	var shouldBuild bool
	if _, ok := b.watching[path]; ok {
		return false
	}
	filepath.Walk(path, func(p string, fileInfo os.FileInfo, err error) error {
		if fileInfo == nil {
			return err
		}
		if fileInfo.IsDir() {
			if err := b.w.Add(p); err != nil {
				return err
			}
			b.watching[p] = struct{}{}
		} else {
			shouldBuild = true
		}
		return nil
	})
	return shouldBuild
}

func isCreate(op fsn.Op) bool {
	return op&fsn.Create == fsn.Create
}

func isDelete(op fsn.Op) bool {
	return op&fsn.Remove == fsn.Remove
}

func isModify(op fsn.Op) bool {
	return op&fsn.Write == fsn.Write ||
		op&fsn.Rename == fsn.Rename
}
<commit_msg>not spawning a new go routine for processing events<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/Tonkpils/snag/vow"
	fsn "gopkg.in/fsnotify.v1"
)

var (
	mtimes = map[string]time.Time{}
)

const (
	GoExt = ".go"
)

type Bob struct {
	w        *fsn.Watcher
	done     chan struct{}
	watching map[string]struct{}
}

func NewBuilder() (*Bob, error) {
	w, err := fsn.NewWatcher()
	if err != nil {
		return nil, err
	}

	return &Bob{
		w:        w,
		done:     make(chan struct{}),
		watching: map[string]struct{}{},
	}, nil
}

func (b *Bob) Close() {
	b.w.Close()
	close(b.done)
}

func (b *Bob) Watch() error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	b.watch(wd)

	for {
		select {
		case ev := <-b.w.Events:
			fmt.Println(ev)
			var queueBuild bool
			switch {
			case isCreate(ev.Op):
				fmt.Println("Creating")
				queueBuild = b.watch(ev.Name)
			case isDelete(ev.Op):
				fmt.Println("Deleteing")
				if _, ok := b.watching[ev.Name]; ok {
					b.w.Remove(ev.Name)
					delete(b.watching, ev.Name)
				}
				queueBuild = true
			case isModify(ev.Op):
				queueBuild = true
			}
			if queueBuild {
				b.maybeQueue(ev.Name)
			}
		case err := <-b.w.Errors:
			log.Println("error:", err)
		case <-b.done:
			return nil
		}
	}
}

func (b *Bob) maybeQueue(path string) {
	if filepath.Ext(path) != GoExt {
		return
	}

	// clear scroll buffer
	print("\033c")

	// setup list of commands
	v := vow.To("go", "build", "./...")
	v.Then("go", "vet", "./...")
	v.Then("go", "test", "./...")

	stat, err := os.Stat(path)
	if err == nil {
		mtime := stat.ModTime()
		lasttime := mtimes[path]
		if !mtime.Equal(lasttime) {
			mtimes[path] = mtime
			v.Exec(os.Stdout)
		}
	} else {
		log.Println(err)
		delete(mtimes, path)
		v.Exec(os.Stdout)
	}
}

func (b *Bob) watch(path string) bool {
	var shouldBuild bool
	if _, ok := b.watching[path]; ok {
		return false
	}
	filepath.Walk(path, func(p string, fileInfo os.FileInfo, err error) error {
		if fileInfo == nil {
			return err
		}
		if fileInfo.IsDir() {
			if err := b.w.Add(p); err != nil {
				return err
			}
			b.watching[p] = struct{}{}
		} else {
			shouldBuild = true
		}
		return nil
	})
	return shouldBuild
}

func isCreate(op fsn.Op) bool {
	return op&fsn.Create == fsn.Create
}

func isDelete(op fsn.Op) bool {
	return op&fsn.Remove == fsn.Remove
}

func isModify(op fsn.Op) bool {
	return op&fsn.Write == fsn.Write ||
		op&fsn.Rename == fsn.Rename
}
<|endoftext|>