<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"
)

var binaryPath string

func init() {
	var err error
	if binaryPath, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
}

func getPackages(workspace string) ([]string, error) {
	// dir := filepath.Join(workspace, "...") // does not work with '...'
	dir := filepath.Clean(workspace) + string(filepath.Separator) + "..."
	out, err := exec.Command(binaryPath, "list", dir).Output()
	if err != nil {
		return nil, err
	}
	// TODO: improve this.
	str := strings.TrimSpace(string(out))
	return strings.Split(str, "\n"), nil
}

func getDependencies(name string) (map[string]bool, error) {
	out, err := exec.Command(binaryPath, "list", "-json", name).Output()
	if err != nil {
		return nil, err
	}

	var p struct{ Deps []string }
	if err := json.Unmarshal(out, &p); err != nil {
		return nil, err
	}

	deps := make(map[string]bool)
	for _, pkg := range p.Deps {
		deps[pkg] = true
	}
	return deps, nil
}

func getUsedPackages(deps map[string]bool, packages []string) []string {
	used := make([]string, 0)
	for _, pkg := range packages {
		if !deps[pkg] {
			continue
		}
		used = append(used, pkg)
	}
	return used
}

func unique(in []string) []string {
	seen := make(map[string]bool)
	out := make([]string, 0, len(in)) // think about in-place modification
	for _, item := range in {
		if seen[item] {
			continue
		}
		seen[item] = true
		out = append(out, item)
	}
	return out
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) <= 0 {
		log.Fatal("please, specify package names")
	}

	lsPackages := unique(args)

	wsPackages, err := getPackages(".")
	if err != nil {
		log.Fatal(err)
	}

	for _, p := range wsPackages {
		deps, err := getDependencies(p)
		if err != nil {
			log.Fatal(err)
		}
		used := getUsedPackages(deps, lsPackages)
		if len(used) > 0 {
			fmt.Printf("%s: [%s]\n", p, strings.Join(used, ", "))
		}
	}
}
<commit_msg>[10.4] name changes<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"
)

var binaryPath string

func init() {
	var err error
	if binaryPath, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
}

func getPackages(workspace string) ([]string, error) {
	// dir := filepath.Join(workspace, "...") // does not work with '...'
	dir := filepath.Clean(workspace) + string(filepath.Separator) + "..."
	out, err := exec.Command(binaryPath, "list", dir).Output()
	if err != nil {
		return nil, err
	}
	// TODO: improve this.
	str := strings.TrimSpace(string(out))
	return strings.Split(str, "\n"), nil
}

func getDependencies(name string) (map[string]bool, error) {
	out, err := exec.Command(binaryPath, "list", "-json", name).Output()
	if err != nil {
		return nil, err
	}

	var p struct{ Deps []string }
	if err := json.Unmarshal(out, &p); err != nil {
		return nil, err
	}

	deps := make(map[string]bool)
	for _, pkg := range p.Deps {
		deps[pkg] = true
	}
	return deps, nil
}

func getUsedPackages(deps map[string]bool, packages []string) []string {
	used := make([]string, 0)
	for _, pkg := range packages {
		if !deps[pkg] {
			continue
		}
		used = append(used, pkg)
	}
	return used
}

func unique(in []string) []string {
	seen := make(map[string]bool)
	out := make([]string, 0, len(in)) // think about in-place modification
	for _, item := range in {
		if seen[item] {
			continue
		}
		seen[item] = true
		out = append(out, item)
	}
	return out
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) <= 0 {
		log.Fatal("please, specify package names")
	}

	argPkgs := unique(args)

	workspacePkgs, err := getPackages(".")
	if err != nil {
		log.Fatal(err)
	}

	for _, p := range workspacePkgs {
		deps, err := getDependencies(p)
		if err != nil {
			log.Fatal(err)
		}
		used := getUsedPackages(deps, argPkgs)
		if len(used) > 0 {
			fmt.Printf("%s: [%s]\n", p, strings.Join(used, ", "))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/http"

	"appengine"
	"appengine/urlfetch"
)

const runUrl = "http://golang.org/compile"

func init() {
	http.HandleFunc("/", rootHandler)
	http.HandleFunc("/compile", compileHandler)
	err := serveScripts("js", "playground.js")
	if err != nil {
		panic(err)
	}
	if err := initTour("."); err != nil {
		panic(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := renderTour(w); err != nil {
		c.Criticalf("template render: %v", err)
	}
}

func compileHandler(w http.ResponseWriter, r *http.Request) {
	if err := passThru(w, r); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "Compile server error.")
	}
}

func passThru(w io.Writer, req *http.Request) error {
	c := appengine.NewContext(req)
	client := urlfetch.Client(c)
	defer req.Body.Close()
	r, err := client.Post(runUrl, req.Header.Get("Content-type"), req.Body)
	if err != nil {
		c.Errorf("making POST request:", err)
		return err
	}
	defer r.Body.Close()
	if _, err := io.Copy(w, r.Body); err != nil {
		c.Errorf("copying response Body:", err)
		return err
	}
	return nil
}

// prepContent returns a Reader that produces the content from the given
// Reader, but strips the prefix "#appengine: " from each line. It also drops
// any non-blank like that follows a series of 1 or more lines with the prefix.
func prepContent(in io.Reader) io.Reader {
	var prefix = []byte("#appengine: ")
	out, w := io.Pipe()
	go func() {
		r := bufio.NewReader(in)
		drop := false
		for {
			b, err := r.ReadBytes('\n')
			if err != nil && err != io.EOF {
				w.CloseWithError(err)
				return
			}
			if bytes.HasPrefix(b, prefix) {
				b = b[len(prefix):]
				drop = true
			} else if drop {
				if len(b) > 1 {
					b = nil
				}
				drop = false
			}
			if len(b) > 0 {
				w.Write(b)
			}
			if err == io.EOF {
				w.Close()
				return
			}
		}
	}()
	return out
}

// socketAddr returns the WebSocket handler address.
// The App Engine version does not provide a WebSocket handler.
func socketAddr() string { return "" }
<commit_msg>go-tour: use playground package from go.talks for /compile handler<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

import (
	"bufio"
	"bytes"
	"io"
	"net/http"

	"appengine"

	_ "code.google.com/p/go.talks/pkg/playground"
)

const runUrl = "http://golang.org/compile"

func init() {
	http.HandleFunc("/", rootHandler)
	err := serveScripts("js", "playground.js")
	if err != nil {
		panic(err)
	}
	if err := initTour("."); err != nil {
		panic(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := renderTour(w); err != nil {
		c.Criticalf("template render: %v", err)
	}
}

// prepContent returns a Reader that produces the content from the given
// Reader, but strips the prefix "#appengine: " from each line. It also drops
// any non-blank like that follows a series of 1 or more lines with the prefix.
func prepContent(in io.Reader) io.Reader {
	var prefix = []byte("#appengine: ")
	out, w := io.Pipe()
	go func() {
		r := bufio.NewReader(in)
		drop := false
		for {
			b, err := r.ReadBytes('\n')
			if err != nil && err != io.EOF {
				w.CloseWithError(err)
				return
			}
			if bytes.HasPrefix(b, prefix) {
				b = b[len(prefix):]
				drop = true
			} else if drop {
				if len(b) > 1 {
					b = nil
				}
				drop = false
			}
			if len(b) > 0 {
				w.Write(b)
			}
			if err == io.EOF {
				w.Close()
				return
			}
		}
	}()
	return out
}

// socketAddr returns the WebSocket handler address.
// The App Engine version does not provide a WebSocket handler.
func socketAddr() string { return "" }
<|endoftext|>


<commit_before>package filer2

import (
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func (f *Filer) loopProcessingDeletion() {

	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
		m := make(map[string]operation.LookupResult)
		for _, vid := range vids {
			locs, _ := f.MasterClient.GetVidLocations(vid)
			var locations []operation.Location
			for _, loc := range locs {
				locations = append(locations, operation.Location{
					Url:       loc.Url,
					PublicUrl: loc.PublicUrl,
				})
			}
			m[vid] = operation.LookupResult{
				VolumeId:  vid,
				Locations: locations,
			}
		}
		return m, nil
	}

	var deletionCount int
	for {
		deletionCount = 0
		f.fileIdDeletionQueue.Consume(func(fileIds []string) {
			deletionCount = len(fileIds)
			_, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
			if err != nil {
				glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
			} else {
				glog.V(1).Infof("deleting fileIds len=%d", deletionCount)
			}
		})

		if deletionCount == 0 {
			time.Sleep(1123 * time.Millisecond)
		}
	}
}

func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
	for _, chunk := range chunks {
		f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
	}
}

// DeleteFileByFileId direct delete by file id.
// Only used when the fileId is not being managed by snapshots.
func (f *Filer) DeleteFileByFileId(fileId string) {
	f.fileIdDeletionQueue.EnQueue(fileId)
}

func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {

	if oldEntry == nil {
		return
	}
	if newEntry == nil {
		f.DeleteChunks(oldEntry.Chunks)
	}

	var toDelete []*filer_pb.FileChunk
	newChunkIds := make(map[string]bool)
	for _, newChunk := range newEntry.Chunks {
		newChunkIds[newChunk.GetFileIdString()] = true
	}

	for _, oldChunk := range oldEntry.Chunks {
		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
			toDelete = append(toDelete, oldChunk)
		}
	}
	f.DeleteChunks(toDelete)
}
<commit_msg>report deletion error in the log<commit_after>package filer2

import (
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func (f *Filer) loopProcessingDeletion() {

	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
		m := make(map[string]operation.LookupResult)
		for _, vid := range vids {
			locs, _ := f.MasterClient.GetVidLocations(vid)
			var locations []operation.Location
			for _, loc := range locs {
				locations = append(locations, operation.Location{
					Url:       loc.Url,
					PublicUrl: loc.PublicUrl,
				})
			}
			m[vid] = operation.LookupResult{
				VolumeId:  vid,
				Locations: locations,
			}
		}
		return m, nil
	}

	var deletionCount int
	for {
		deletionCount = 0
		f.fileIdDeletionQueue.Consume(func(fileIds []string) {
			deletionCount = len(fileIds)
			deleteResults, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
			if err != nil {
				glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
			} else {
				glog.V(1).Infof("deleting fileIds len=%d", deletionCount)
			}
			if len(deleteResults) != deletionCount {
				glog.V(0).Infof("delete %d fileIds actual %d", deletionCount, len(deleteResults))
			}
		})

		if deletionCount == 0 {
			time.Sleep(1123 * time.Millisecond)
		}
	}
}

func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
	for _, chunk := range chunks {
		f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
	}
}

// DeleteFileByFileId direct delete by file id.
// Only used when the fileId is not being managed by snapshots.
func (f *Filer) DeleteFileByFileId(fileId string) {
	f.fileIdDeletionQueue.EnQueue(fileId)
}

func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {

	if oldEntry == nil {
		return
	}
	if newEntry == nil {
		f.DeleteChunks(oldEntry.Chunks)
	}

	var toDelete []*filer_pb.FileChunk
	newChunkIds := make(map[string]bool)
	for _, newChunk := range newEntry.Chunks {
		newChunkIds[newChunk.GetFileIdString()] = true
	}

	for _, oldChunk := range oldEntry.Chunks {
		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
			toDelete = append(toDelete, oldChunk)
		}
	}
	f.DeleteChunks(toDelete)
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/google/uuid"
)

type DiskLocation struct {
	Directory              string
	DirectoryUuid          string
	IdxDirectory           string
	DiskType               types.DiskType
	MaxVolumeCount         int
	OriginalMaxVolumeCount int
	MinFreeSpace           util.MinFreeSpace
	volumes                map[needle.VolumeId]*Volume
	volumesLock            sync.RWMutex

	// erasure coding
	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
	ecVolumesLock sync.RWMutex

	isDiskSpaceLow bool
}

func GenerateDirUuid(dir string) (dirUuidString string, err error) {
	glog.V(1).Infof("Getting uuid of volume directory:%s", dir)
	dirUuidString = ""
	fileName := dir + "/vol_dir.uuid"
	if !util.FileExists(fileName) {
		dirUuid, _ := uuid.NewRandom()
		dirUuidString = dirUuid.String()
		writeErr := util.WriteFile(fileName, []byte(dirUuidString), 0644)
		if writeErr != nil {
			return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, writeErr)
		}
	} else {
		uuidData, readErr := os.ReadFile(fileName)
		if readErr != nil {
			return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr)
		}
		dirUuidString = string(uuidData)
	}
	return dirUuidString, nil
}

func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
	dir = util.ResolvePath(dir)
	if idxDir == "" {
		idxDir = dir
	} else {
		idxDir = util.ResolvePath(idxDir)
	}
	dirUuid, err := GenerateDirUuid(dir)
	if err != nil {
		glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
	}
	location := &DiskLocation{
		Directory:              dir,
		DirectoryUuid:          dirUuid,
		IdxDirectory:           idxDir,
		DiskType:               diskType,
		MaxVolumeCount:         maxVolumeCount,
		OriginalMaxVolumeCount: maxVolumeCount,
		MinFreeSpace:           minFreeSpace,
	}
	location.volumes = make(map[needle.VolumeId]*Volume)
	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
	go location.CheckDiskSpace()
	return location
}

func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
	if isValidVolume(filename) {
		base := filename[:len(filename)-4]
		collection, volumeId, err := parseCollectionVolumeId(base)
		return volumeId, collection, err
	}

	return 0, "", fmt.Errorf("file is not a volume: %s", filename)
}

func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
	i := strings.LastIndex(base, "_")
	if i > 0 {
		collection, base = base[0:i], base[i+1:]
	}
	vol, err := needle.NewVolumeId(base)
	return collection, vol, err
}

func isValidVolume(basename string) bool {
	return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
}

func getValidVolumeName(basename string) string {
	if isValidVolume(basename) {
		return basename[:len(basename)-4]
	}
	return ""
}

func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool) bool {
	basename := dirEntry.Name()
	if dirEntry.IsDir() {
		return false
	}
	volumeName := getValidVolumeName(basename)
	if volumeName == "" {
		return false
	}

	// skip if ec volumes exists
	if skipIfEcVolumesExists {
		if util.FileExists(l.Directory + "/" + volumeName + ".ecx") {
			return false
		}
	}

	// check for incomplete volume
	noteFile := l.Directory + "/" + volumeName + ".note"
	if util.FileExists(noteFile) {
		note, _ := os.ReadFile(noteFile)
		glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
		removeVolumeFiles(l.Directory + "/" + volumeName)
		removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
		return false
	}

	// parse out collection, volume id
	vid, collection, err := volumeIdFromFileName(basename)
	if err != nil {
		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
		return false
	}

	// avoid loading one volume more than once
	l.volumesLock.RLock()
	_, found := l.volumes[vid]
	l.volumesLock.RUnlock()
	if found {
		glog.V(1).Infof("loaded volume, %v", vid)
		return true
	}

	// load the volume
	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
	if e != nil {
		glog.V(0).Infof("new volume %s error %s", volumeName, e)
		return false
	}

	l.SetVolume(vid, v)

	size, _, _ := v.FileStat()
	glog.V(0).Infof("data file %s, replication=%s v=%d size=%d ttl=%s",
		l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
	return true
}

func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {

	task_queue := make(chan os.DirEntry, 10*concurrency)
	go func() {
		foundVolumeNames := make(map[string]bool)
		if dirEntries, err := os.ReadDir(l.Directory); err == nil {
			for _, entry := range dirEntries {
				volumeName := getValidVolumeName(entry.Name())
				if volumeName == "" {
					continue
				}
				if _, found := foundVolumeNames[volumeName]; !found {
					foundVolumeNames[volumeName] = true
					task_queue <- entry
				}
			}
		}
		close(task_queue)
	}()

	var wg sync.WaitGroup
	for workerNum := 0; workerNum < concurrency; workerNum++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for fi := range task_queue {
				_ = l.loadExistingVolume(fi, needleMapKind, true)
			}
		}()
	}
	wg.Wait()

}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {

	workerNum := runtime.NumCPU()
	if workerNum <= 10 {
		workerNum = 10
	}
	l.concurrentLoadingVolumes(needleMapKind, workerNum)
	glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)

	l.loadAllEcShards()
	glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))

}

func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {

	l.volumesLock.Lock()
	delVolsMap := l.unmountVolumeByCollection(collection)
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	delEcVolsMap := l.unmountEcVolumeByCollection(collection)
	l.ecVolumesLock.Unlock()

	errChain := make(chan error, 2)
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		for _, v := range delVolsMap {
			if err := v.Destroy(); err != nil {
				errChain <- err
			}
		}
		wg.Done()
	}()

	go func() {
		for _, v := range delEcVolsMap {
			v.Destroy()
		}
		wg.Done()
	}()

	go func() {
		wg.Wait()
		close(errChain)
	}()

	errBuilder := strings.Builder{}
	for err := range errChain {
		errBuilder.WriteString(err.Error())
		errBuilder.WriteString("; ")
	}
	if errBuilder.Len() > 0 {
		e = fmt.Errorf(errBuilder.String())
	}

	return
}

func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
	v, ok := l.volumes[vid]
	if !ok {
		return
	}
	e = v.Destroy()
	if e != nil {
		return
	}
	found = true
	delete(l.volumes, vid)
	return
}

func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
	if fileInfo, found := l.LocateVolume(vid); found {
		return l.loadExistingVolume(fileInfo, needleMapKind, false)
	}
	return false
}

var ErrVolumeNotFound = fmt.Errorf("volume not found")

func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	_, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	_, err := l.deleteVolumeById(vid)
	return err
}

func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	v, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	v.Close()
	delete(l.volumes, vid)
	return nil
}

func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
	deltaVols := make(map[needle.VolumeId]*Volume, 0)
	for k, v := range l.volumes {
		if v.Collection == collectionName && !v.isCompacting && !v.isCommitCompacting {
			deltaVols[k] = v
		}
	}

	for k := range deltaVols {
		delete(l.volumes, k)
	}
	return deltaVols
}

func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	l.volumes[vid] = volume
	volume.location = l
}

func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	v, ok := l.volumes[vid]
	return v, ok
}

func (l *DiskLocation) VolumesLen() int {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	return len(l.volumes)
}

func (l *DiskLocation) SetStopping() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.SetStopping()
	}
	l.volumesLock.Unlock()

	return
}

func (l *DiskLocation) Close() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.Close()
	}
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	for _, ecVolume := range l.ecVolumes {
		ecVolume.Close()
	}
	l.ecVolumesLock.Unlock()

	return
}

func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) {
	// println("LocateVolume", vid, "on", l.Directory)
	if dirEntries, err := os.ReadDir(l.Directory); err == nil {
		for _, entry := range dirEntries {
			// println("checking", entry.Name(), "...")
			volId, _, err := volumeIdFromFileName(entry.Name())
			// println("volId", volId, "err", err)
			if vid == volId && err == nil {
				return entry, true
			}
		}
	}

	return nil, false
}

func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {

	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	for _, vol := range l.volumes {
		if vol.IsReadOnly() {
			continue
		}
		datSize, idxSize, _ := vol.FileStat()
		unUsedSpace += volumeSizeLimit - (datSize + idxSize)
	}

	return
}

func (l *DiskLocation) CheckDiskSpace() {
	for {
		if dir, e := filepath.Abs(l.Directory); e == nil {
			s := stats.NewDiskStatus(dir)
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))

			isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
			if isLow != l.isDiskSpaceLow {
				l.isDiskSpaceLow = !l.isDiskSpaceLow
			}

			logLevel := glog.Level(4)
			if l.isDiskSpaceLow {
				logLevel = glog.Level(0)
			}

			glog.V(logLevel).Infof("dir %s %s", dir, desc)
		}
		time.Sleep(time.Minute)
	}

}
<commit_msg>optimiz concurrency user can customize number of workers via env "GOMAXPROCS"<commit_after>package storage

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/google/uuid"
)

type DiskLocation struct {
	Directory              string
	DirectoryUuid          string
	IdxDirectory           string
	DiskType               types.DiskType
	MaxVolumeCount         int
	OriginalMaxVolumeCount int
	MinFreeSpace           util.MinFreeSpace
	volumes                map[needle.VolumeId]*Volume
	volumesLock            sync.RWMutex

	// erasure coding
	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
	ecVolumesLock sync.RWMutex

	isDiskSpaceLow bool
}

func GenerateDirUuid(dir string) (dirUuidString string, err error) {
	glog.V(1).Infof("Getting uuid of volume directory:%s", dir)
	dirUuidString = ""
	fileName := dir + "/vol_dir.uuid"
	if !util.FileExists(fileName) {
		dirUuid, _ := uuid.NewRandom()
		dirUuidString = dirUuid.String()
		writeErr := util.WriteFile(fileName, []byte(dirUuidString), 0644)
		if writeErr != nil {
			return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, writeErr)
		}
	} else {
		uuidData, readErr := os.ReadFile(fileName)
		if readErr != nil {
			return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr)
		}
		dirUuidString = string(uuidData)
	}
	return dirUuidString, nil
}

func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
	dir = util.ResolvePath(dir)
	if idxDir == "" {
		idxDir = dir
	} else {
		idxDir = util.ResolvePath(idxDir)
	}
	dirUuid, err := GenerateDirUuid(dir)
	if err != nil {
		glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
	}
	location := &DiskLocation{
		Directory:              dir,
		DirectoryUuid:          dirUuid,
		IdxDirectory:           idxDir,
		DiskType:               diskType,
		MaxVolumeCount:         maxVolumeCount,
		OriginalMaxVolumeCount: maxVolumeCount,
		MinFreeSpace:           minFreeSpace,
	}
	location.volumes = make(map[needle.VolumeId]*Volume)
	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
	go location.CheckDiskSpace()
	return location
}

func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
	if isValidVolume(filename) {
		base := filename[:len(filename)-4]
		collection, volumeId, err := parseCollectionVolumeId(base)
		return volumeId, collection, err
	}

	return 0, "", fmt.Errorf("file is not a volume: %s", filename)
}

func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
	i := strings.LastIndex(base, "_")
	if i > 0 {
		collection, base = base[0:i], base[i+1:]
	}
	vol, err := needle.NewVolumeId(base)
	return collection, vol, err
}

func isValidVolume(basename string) bool {
	return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
}

func getValidVolumeName(basename string) string {
	if isValidVolume(basename) {
		return basename[:len(basename)-4]
	}
	return ""
}

func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool) bool {
	basename := dirEntry.Name()
	if dirEntry.IsDir() {
		return false
	}
	volumeName := getValidVolumeName(basename)
	if volumeName == "" {
		return false
	}

	// skip if ec volumes exists
	if skipIfEcVolumesExists {
		if util.FileExists(l.Directory + "/" + volumeName + ".ecx") {
			return false
		}
	}

	// check for incomplete volume
	noteFile := l.Directory + "/" + volumeName + ".note"
	if util.FileExists(noteFile) {
		note, _ := os.ReadFile(noteFile)
		glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
		removeVolumeFiles(l.Directory + "/" + volumeName)
		removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
		return false
	}

	// parse out collection, volume id
	vid, collection, err := volumeIdFromFileName(basename)
	if err != nil {
		glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
		return false
	}

	// avoid loading one volume more than once
	l.volumesLock.RLock()
	_, found := l.volumes[vid]
	l.volumesLock.RUnlock()
	if found {
		glog.V(1).Infof("loaded volume, %v", vid)
		return true
	}

	// load the volume
	v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
	if e != nil {
		glog.V(0).Infof("new volume %s error %s", volumeName, e)
		return false
	}

	l.SetVolume(vid, v)

	size, _, _ := v.FileStat()
	glog.V(0).Infof("data file %s, replication=%s v=%d size=%d ttl=%s",
		l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
	return true
}

func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {

	task_queue := make(chan os.DirEntry, 10*concurrency)
	go func() {
		foundVolumeNames := make(map[string]bool)
		if dirEntries, err := os.ReadDir(l.Directory); err == nil {
			for _, entry := range dirEntries {
				volumeName := getValidVolumeName(entry.Name())
				if volumeName == "" {
					continue
				}
				if _, found := foundVolumeNames[volumeName]; !found {
					foundVolumeNames[volumeName] = true
					task_queue <- entry
				}
			}
		}
		close(task_queue)
	}()

	var wg sync.WaitGroup
	for workerNum := 0; workerNum < concurrency; workerNum++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for fi := range task_queue {
				_ = l.loadExistingVolume(fi, needleMapKind, true)
			}
		}()
	}
	wg.Wait()

}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {

	workerNum := runtime.NumCPU()
	val, ok := os.LookupEnv("GOMAXPROCS")
	if ok {
		num, err := strconv.Atoi(val)
		if err != nil || num < 1 {
			num = 10
			glog.Warningf("failed to set worker number from GOMAXPROCS , set to default:10")
		}
		workerNum = num
	}
	l.concurrentLoadingVolumes(needleMapKind, workerNum)
	glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)

	l.loadAllEcShards()
	glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))

}

func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {

	l.volumesLock.Lock()
	delVolsMap := l.unmountVolumeByCollection(collection)
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	delEcVolsMap := l.unmountEcVolumeByCollection(collection)
	l.ecVolumesLock.Unlock()

	errChain := make(chan error, 2)
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		for _, v := range delVolsMap {
			if err := v.Destroy(); err != nil {
				errChain <- err
			}
		}
		wg.Done()
	}()

	go func() {
		for _, v := range delEcVolsMap {
			v.Destroy()
		}
		wg.Done()
	}()

	go func() {
		wg.Wait()
		close(errChain)
	}()

	errBuilder := strings.Builder{}
	for err := range errChain {
		errBuilder.WriteString(err.Error())
		errBuilder.WriteString("; ")
	}
	if errBuilder.Len() > 0 {
		e = fmt.Errorf(errBuilder.String())
	}

	return
}

func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
	v, ok := l.volumes[vid]
	if !ok {
		return
	}
	e = v.Destroy()
	if e != nil {
		return
	}
	found = true
	delete(l.volumes, vid)
	return
}

func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
	if fileInfo, found := l.LocateVolume(vid); found {
		return l.loadExistingVolume(fileInfo, needleMapKind, false)
	}
	return false
}

var ErrVolumeNotFound = fmt.Errorf("volume not found")

func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	_, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	_, err := l.deleteVolumeById(vid)
	return err
}

func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	v, ok := l.volumes[vid]
	if !ok {
		return ErrVolumeNotFound
	}
	v.Close()
	delete(l.volumes, vid)
	return nil
}

func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
	deltaVols := make(map[needle.VolumeId]*Volume, 0)
	for k, v := range l.volumes {
		if v.Collection == collectionName && !v.isCompacting && !v.isCommitCompacting {
			deltaVols[k] = v
		}
	}

	for k := range deltaVols {
		delete(l.volumes, k)
	}
	return deltaVols
}

func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
	l.volumesLock.Lock()
	defer l.volumesLock.Unlock()

	l.volumes[vid] = volume
	volume.location = l
}

func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	v, ok := l.volumes[vid]
	return v, ok
}

func (l *DiskLocation) VolumesLen() int {
	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	return len(l.volumes)
}

func (l *DiskLocation) SetStopping() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.SetStopping()
	}
	l.volumesLock.Unlock()

	return
}

func (l *DiskLocation) Close() {
	l.volumesLock.Lock()
	for _, v := range l.volumes {
		v.Close()
	}
	l.volumesLock.Unlock()

	l.ecVolumesLock.Lock()
	for _, ecVolume := range l.ecVolumes {
		ecVolume.Close()
	}
	l.ecVolumesLock.Unlock()

	return
}

func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) {
	// println("LocateVolume", vid, "on", l.Directory)
	if dirEntries, err := os.ReadDir(l.Directory); err == nil {
		for _, entry := range dirEntries {
			// println("checking", entry.Name(), "...")
			volId, _, err := volumeIdFromFileName(entry.Name())
			// println("volId", volId, "err", err)
			if vid == volId && err == nil {
				return entry, true
			}
		}
	}

	return nil, false
}

func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {

	l.volumesLock.RLock()
	defer l.volumesLock.RUnlock()

	for _, vol := range l.volumes {
		if vol.IsReadOnly() {
			continue
		}
		datSize, idxSize, _ := vol.FileStat()
		unUsedSpace += volumeSizeLimit - (datSize + idxSize)
	}

	return
}

func (l *DiskLocation) CheckDiskSpace() {
	for {
		if dir, e := filepath.Abs(l.Directory); e == nil {
			s := stats.NewDiskStatus(dir)
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))

			isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
			if isLow != l.isDiskSpaceLow {
				l.isDiskSpaceLow = !l.isDiskSpaceLow
			}

			logLevel := glog.Level(4)
			if l.isDiskSpaceLow {
				logLevel = glog.Level(0)
			}

			glog.V(logLevel).Infof("dir %s %s", dir, desc)
		}
		time.Sleep(time.Minute)
	}

}
<|endoftext|>


<commit_before>package tools

import (
	"math"
	"math/big"
	"sort"
	"strconv"
)

// Sum returns the sum of its argument.
func Sum(a ...int) int {
	var s int
	for _, v := range a {
		s += v
	}
	return s
}

// SqrtInt computes the square root of n and casts to an int.
func SqrtInt(n int) int {
	return int(math.Sqrt(float64(n)))
}

// Divisors returns all the proper divisors of n.
func Divisors(n int) []int {
	d := []int{1}
	for i := 2; i <= SqrtInt(n); i++ {
		if n%i == 0 {
			d = append(d, i)
			if other := n / i; other != i {
				d = append(d, other)
			}
		}
	}
	return d
}

// SumDivisors sums all the proper divisors of n.
func SumDivisors(n int) int {
	return Sum(Divisors(n)...)
}

// Perms return a channel of each permutation of a slice.
func Perms(slice []int) chan []int {
	c := make(chan []int)
	go func() {
		if len(slice) == 1 {
			c <- slice
		} else {
			for i, v := range slice {
				b := append([]int(nil), slice[:i]...)
				b = append(b, slice[i+1:]...)
				for p := range Perms(b) {
					c <- append([]int{v}, p...)
				}
			}
		}
		close(c)
	}()
	return c
}

// Divmod gives the quotient and remainder of integer division.
func Divmod(a, b int) (int, int) {
	return a / b, a % b
}

// PrimeFactors gives the prime factors of an int.
func PrimeFactors(n int) map[int]int {
	factors := make(map[int]int)
	d := 2
	for n > 1 {
		for n%d == 0 {
			n /= d
			factors[d]++
		}
		d++
	}
	return factors
}

// IsPrime determines if a given number is prime.
// https://en.wikipedia.org/wiki/Primality_test
func IsPrime(n int) bool {
	if n <= 1 {
		return false
	}
	if n <= 3 {
		return true
	}
	if n%2 == 0 || n%3 == 0 {
		return false
	}
	b := SqrtInt(n)
	for i := 5; i <= b; i += 6 {
		if n%i == 0 || n%(i+2) == 0 {
			return false
		}
	}
	return true
}

// SievePrime gives all the primes below n via the method of Eratosthenes.
func SievePrime(n int) []int {
	var primes []int
	comp := make([]bool, n+2)
	for i := 2; i <= int(math.Sqrt(float64(n))); i++ {
		for j := i * i; j < n; j += i {
			comp[j] = true
		}
	}
	for i, v := range comp {
		if i > 1 && !v && i < n {
			primes = append(primes, i)
		}
	}
	return primes
}

// Exp computes x to the power of y.
func Exp(x, y int) *big.Int {
	a := big.NewInt(1)
	b := big.NewInt(int64(x))
	for i := 0; i < y; i++ {
		a.Mul(a, b)
	}
	return a
}

// DigitSum sums the digit^p of a given number.
func DigitSum(n, p int) *big.Int {
	s := big.NewInt(0)
	for _, c := range strconv.Itoa(n) {
		i, _ := strconv.Atoi(string(c))
		s.Add(s, Exp(i, p))
	}
	return s
}

// JoinInts joins slice of ints and return it as an int.
func JoinInts(slice []int) int {
	var sum int
	len := len(slice) - 1
	for i, v := range slice {
		p := 1
		for j := 0; j < len-i; j++ {
			p *= 10
		}
		sum += p * v
	}
	return sum
}

// ProdInts computes the product of the slice of ints.
func ProdInts(slice []int) int {
	p := 1
	for _, v := range slice {
		p *= v
	}
	return p
}

// MinInt returns the smaller number of a and b.
func MinInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// MaxInt returns the larger number of a and b.
func MaxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// SimplifyFraction simplifies a/b and return the (numerator, denominator).
func SimplifyFraction(a, b int) (int, int) {
	pa := PrimeFactors(a)
	pb := PrimeFactors(b)
	for k, v := range pa {
		v2, ok := pb[k]
		if !ok {
			continue
		}
		m := MinInt(v, v2)
		pa[k] = v - m
		pb[k] = v2 - m
	}
	sa := 1
	for k, v := range pa {
		for i := 0; i < v; i++ {
			sa *= k
		}
	}
	sb := 1
	for k, v := range pb {
		for i := 0; i < v; i++ {
			sb *= k
		}
	}
	return sa, sb
}

// Digits returns the individual digits as a slice of int.
func Digits(n int) []int {
	var d []int
	for n > 0 {
		d = append([]int{n % 10}, d...)
		n /= 10
	}
	return d
}

// Factorial returns the fractorial of a number.
func Factorial(n int) *big.Int {
	a := big.NewInt(1)
	for i := 2; i <= n; i++ {
		a.Mul(a, big.NewInt(int64(i)))
	}
	return a
}

// IsPalindromeInt tells if a number is a palindrome.
func IsPalindromeInt(n int) bool {
	return IsPalindromeString(strconv.Itoa(n))
}

// IsPandigital determines if a number is 1 to 9 pandigital.
func IsPandigital(n int) bool {
	d := Digits(n)
	sort.Ints(d)
	a := JoinInts(d)
	return a == 123456789
}
<commit_msg>+ DigitsIth<commit_after>package tools

import (
	"math"
	"math/big"
	"sort"
	"strconv"
)

// Sum returns the sum of its argument.
func Sum(a ...int) int {
	var s int
	for _, v := range a {
		s += v
	}
	return s
}

// SqrtInt computes the square root of n and casts to an int.
func SqrtInt(n int) int {
	return int(math.Sqrt(float64(n)))
}

// Divisors returns all the proper divisors of n.
func Divisors(n int) []int {
	d := []int{1}
	for i := 2; i <= SqrtInt(n); i++ {
		if n%i == 0 {
			d = append(d, i)
			if other := n / i; other != i {
				d = append(d, other)
			}
		}
	}
	return d
}

// SumDivisors sums all the proper divisors of n.
func SumDivisors(n int) int {
	return Sum(Divisors(n)...)
}

// Perms return a channel of each permutation of a slice.
func Perms(slice []int) chan []int {
	c := make(chan []int)
	go func() {
		if len(slice) == 1 {
			c <- slice
		} else {
			for i, v := range slice {
				b := append([]int(nil), slice[:i]...)
				b = append(b, slice[i+1:]...)
				for p := range Perms(b) {
					c <- append([]int{v}, p...)
				}
			}
		}
		close(c)
	}()
	return c
}

// Divmod gives the quotient and remainder of integer division.
func Divmod(a, b int) (int, int) {
	return a / b, a % b
}

// PrimeFactors gives the prime factors of an int.
func PrimeFactors(n int) map[int]int {
	factors := make(map[int]int)
	d := 2
	for n > 1 {
		for n%d == 0 {
			n /= d
			factors[d]++
		}
		d++
	}
	return factors
}

// IsPrime determines if a given number is prime.
// https://en.wikipedia.org/wiki/Primality_test
func IsPrime(n int) bool {
	if n <= 1 {
		return false
	}
	if n <= 3 {
		return true
	}
	if n%2 == 0 || n%3 == 0 {
		return false
	}
	b := SqrtInt(n)
	for i := 5; i <= b; i += 6 {
		if n%i == 0 || n%(i+2) == 0 {
			return false
		}
	}
	return true
}

// SievePrime gives all the primes below n via the method of Eratosthenes.
func SievePrime(n int) []int {
	var primes []int
	comp := make([]bool, n+2)
	for i := 2; i <= int(math.Sqrt(float64(n))); i++ {
		for j := i * i; j < n; j += i {
			comp[j] = true
		}
	}
	for i, v := range comp {
		if i > 1 && !v && i < n {
			primes = append(primes, i)
		}
	}
	return primes
}

// Exp computes x to the power of y.
func Exp(x, y int) *big.Int {
	a := big.NewInt(1)
	b := big.NewInt(int64(x))
	for i := 0; i < y; i++ {
		a.Mul(a, b)
	}
	return a
}

// DigitSum sums the digit^p of a given number.
func DigitSum(n, p int) *big.Int {
	s := big.NewInt(0)
	for _, c := range strconv.Itoa(n) {
		i, _ := strconv.Atoi(string(c))
		s.Add(s, Exp(i, p))
	}
	return s
}

// JoinInts joins slice of ints and return it as an int.
func JoinInts(slice []int) int {
	var sum int
	len := len(slice) - 1
	for i, v := range slice {
		p := 1
		for j := 0; j < len-i; j++ {
			p *= 10
		}
		sum += p * v
	}
	return sum
}

// ProdInts computes the product of the slice of ints.
func ProdInts(slice []int) int {
	p := 1
	for _, v := range slice {
		p *= v
	}
	return p
}

// MinInt returns the smaller number of a and b.
func MinInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// MaxInt returns the larger number of a and b.
func MaxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// SimplifyFraction simplifies a/b and return the (numerator, denominator).
func SimplifyFraction(a, b int) (int, int) {
	pa := PrimeFactors(a)
	pb := PrimeFactors(b)
	for k, v := range pa {
		v2, ok := pb[k]
		if !ok {
			continue
		}
		m := MinInt(v, v2)
		pa[k] = v - m
		pb[k] = v2 - m
	}
	sa := 1
	for k, v := range pa {
		for i := 0; i < v; i++ {
			sa *= k
		}
	}
	sb := 1
	for k, v := range pb {
		for i := 0; i < v; i++ {
			sb *= k
		}
	}
	return sa, sb
}

// Digits returns the individual digits as a slice of int.
func Digits(n int) []int {
	var d []int
	for n > 0 {
		d = append([]int{n % 10}, d...)
		n /= 10
	}
	return d
}

// DigitsIth returns the i-th digit of a number n.
func DigitsIth(n, i int) int {
	d := Digits(n)
	if i < 0 || i >= len(d) {
		return -1
	}
	return Digits(n)[i]
}

// Factorial returns the fractorial of a number.
func Factorial(n int) *big.Int {
	a := big.NewInt(1)
	for i := 2; i <= n; i++ {
		a.Mul(a, big.NewInt(int64(i)))
	}
	return a
}

// IsPalindromeInt tells if a number is a palindrome.
func IsPalindromeInt(n int) bool {
	return IsPalindromeString(strconv.Itoa(n))
}

// IsPandigital determines if a number is 1 to 9 pandigital.
func IsPandigital(n int) bool {
	d := Digits(n)
	sort.Ints(d)
	a := JoinInts(d)
	return a == 123456789
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"net/http"
	"time"

	"appengine"

	"github.com/gorilla/feeds"
	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

var baseUrl = "https://writing.natwelch.com"

func buildFeed(c appengine.Context, entries *[]models.Entry) *feeds.Feed {
	now := time.Now()
	me := &feeds.Author{"Nat Welch", "nat@natwelch.com"}
	feed := &feeds.Feed{
		Title:       "Nat? Nat. Nat!",
		Link:        &feeds.Link{Href: baseUrl},
		Description: "Thoughts from Nat about stuff",
		Author:      me,
		Created:     now,
	}

	feed.Items = []*feeds.Item{}
	for _, v := range *entries {
		if v.Public {
			feed.Items = append(feed.Items, &feeds.Item{
				Title:       v.Title,
				Link:        &feeds.Link{Href: baseUrl + v.Url()},
				Description: string(v.Html()),
				Author:      me,
				Created:     v.Datetime,
				Updated:     v.Modified,
			})
		}
	}

	return feed
}

func buildSummary(c appengine.Context, entries *[]models.Entry) *feeds.Feed {
	now := time.Now()
	me := &feeds.Author{"Nat Welch", "nat@natwelch.com"}
	feed := &feeds.Feed{
		Title:       "Nat? Nat. Nat! - Summaries",
		Link:        &feeds.Link{Href: baseUrl},
		Description: "Thoughts from Nat about stuff",
		Author:      me,
		Created:     now,
	}

	feed.Items = []*feeds.Item{}
	for _, v := range *entries {
		if v.Public {
			feed.Items = append(feed.Items, &feeds.Item{
				Title:       fmt.Sprintf("Nat? Nat. Nat! #%d", v.Id),
				Link:        &feeds.Link{Href: baseUrl + v.Url()},
				Description: "At some point, I'll put something here.",
				Author:      me,
				Created:     v.Datetime,
				Updated:     v.Modified,
			})
		}
	}

	return feed
}

func FeedAtomHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.RecentPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildFeed(c, entries)
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/atom+xml; charset=utf-8")
	w.WriteText("%s", atom)
	return
}

func FeedRssHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.RecentPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildFeed(c, entries)
	rss, err := feed.ToRss()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
	w.WriteText("%s", rss)
	return
}

func SummaryAtomHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildSummary(c, entries)
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/atom+xml; charset=utf-8")
	w.WriteText("%s", atom)
	return
}

func SummaryRssHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildSummary(c, entries)
	rss, err := feed.ToRss()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
	w.WriteText("%s", rss)
	return
}
<commit_msg>add title to tweets<commit_after>package handlers

import (
	"fmt"
	"net/http"
	"time"

	"appengine"

	"github.com/gorilla/feeds"
	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

var baseUrl = "https://writing.natwelch.com"

func buildFeed(c appengine.Context, entries *[]models.Entry) *feeds.Feed {
	now := time.Now()
	me := &feeds.Author{"Nat Welch", "nat@natwelch.com"}
	feed := &feeds.Feed{
		Title:       "Nat? Nat. Nat!",
		Link:        &feeds.Link{Href: baseUrl},
		Description: "Thoughts from Nat about stuff",
		Author:      me,
		Created:     now,
	}

	feed.Items = []*feeds.Item{}
	for _, v := range *entries {
		if v.Public {
			feed.Items = append(feed.Items, &feeds.Item{
				Title:       v.Title,
				Link:        &feeds.Link{Href: baseUrl + v.Url()},
				Description: string(v.Html()),
				Author:      me,
				Created:     v.Datetime,
				Updated:     v.Modified,
			})
		}
	}

	return feed
}

func buildSummary(c appengine.Context, entries *[]models.Entry) *feeds.Feed {
	now := time.Now()
	me := &feeds.Author{"Nat Welch", "nat@natwelch.com"}
	feed := &feeds.Feed{
		Title:       "Nat? Nat. Nat! - Summaries",
		Link:        &feeds.Link{Href: baseUrl},
		Description: "Thoughts from Nat about stuff",
		Author:      me,
		Created:     now,
	}

	feed.Items = []*feeds.Item{}
	for _, v := range *entries {
		if v.Public {
			title := fmt.Sprintf("Nat? Nat. Nat! #%d", v.Id)
			if v.Title != "" {
				title = v.Title
			}

			feed.Items = append(feed.Items, &feeds.Item{
				Title:       title,
				Link:        &feeds.Link{Href: baseUrl + v.Url()},
				Description: "At some point, I'll put something here.",
				Author:      me,
				Created:     v.Datetime,
				Updated:     v.Modified,
			})
		}
	}

	return feed
}

func FeedAtomHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.RecentPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildFeed(c, entries)
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/atom+xml; charset=utf-8")
	w.WriteText("%s", atom)
	return
}

func FeedRssHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.RecentPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildFeed(c, entries)
	rss, err := feed.ToRss()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
	w.WriteText("%s", rss)
	return
}

func SummaryAtomHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildSummary(c, entries)
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/atom+xml; charset=utf-8")
	w.WriteText("%s", atom)
	return
}

func SummaryRssHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	feed := buildSummary(c, entries)
	rss, err := feed.ToRss()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
	w.WriteText("%s", rss)
	return
}
<|endoftext|>


<commit_before>package handlers

import (
	"appengine"
	"appengine/user"
	"errors"
	"net/http"
	"strconv"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ResponseData struct {
	Entry   *models.Entry
	IsAdmin bool
	Next    string
	Prev    string
}

func PostHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	id, err := strconv.ParseInt(r.Param("id"), 10, 64)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	entry, err := models.GetEntry(c, id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if !entry.Public {
		http.Error(w, errors.New("Post is not public").Error(), 403)
		return
	} else {
		responseData := &ResponseData{
			Entry:   entry,
			IsAdmin: user.IsAdmin(c),
			Next:    entry.NextPost(c),
			Prev:    entry.PrevPost(c)}
		w.Render("post", responseData)
		return
	}
}
<commit_msg>show if user is admin<commit_after>package handlers

import (
	"appengine"
	"appengine/user"
	"errors"
	"net/http"
	"strconv"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ResponseData struct {
	Entry   *models.Entry
	IsAdmin bool
	Next    string
	Prev    string
}

func PostHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	id, err := strconv.ParseInt(r.Param("id"), 10, 64)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	entry, err := models.GetEntry(c, id)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if !entry.Public && !user.IsAdmin(c) {
		http.Error(w, errors.New("Post is not public").Error(), 403)
		return
	} else {
		responseData := &ResponseData{
			Entry:   entry,
			IsAdmin: user.IsAdmin(c),
			Next:    entry.NextPost(c),
			Prev:    entry.PrevPost(c)}
		w.Render("post", responseData)
		return
	}
}
<|endoftext|>


<commit_before>package packer

import (
	"fmt"
	"log"
	"strings"
	"sync"

	"github.com/armon/circbuf"
)

// OutputCallback is the type that is called when there is a matching
// machine-readable output line for the UI.
type OutputCallback func(*Output)

// Output is a single line of machine-readable output from Packer.
type Output struct {
	Timestamp string
	Target    string
	Type      string
	Data      []string
}

// packerUi is an implementation of ui.Ui that we pass to helper/exec
// that parses the machine-readable output of Packer and calls callbacks
// for the various entries.
type packerUi struct {
	Callbacks map[string]OutputCallback

	buf  *circbuf.Buffer
	once sync.Once
}

// Finish should be called when you're done to force the final newline
// to parse the final event.
func (u *packerUi) Finish() {
	// We only need to do anything if we have stuff in our buffer
	if u.buf != nil && u.buf.TotalWritten() > 0 {
		u.Raw("\n")
	}
}

// Ignore Header and Message because we don't use them from helper/exec
func (u *packerUi) Header(string)  {}
func (u *packerUi) Message(string) {}

func (u *packerUi) Raw(msg string) {
	if msg == "" {
		// Not sure how this would happen, but there is nothing to do here.
		return
	}

	u.once.Do(u.init)

	// We loop while we have a newline in the message
	for {
		// Get the index for the newline if there is one
		idx := strings.IndexRune(msg, '\n')
		if idx == -1 {
			// The newline isn't there, write it to the circular buffer
			// and wait longer.
			u.buf.Write([]byte(msg))
			break
		}

		// We got a newline! Grab the contents from the circular buffer and
		// copy it so we can clear the buffer.
		bufRaw := u.buf.Bytes()
		buf := string(bufRaw) + msg[:idx]
		bufRaw = nil
		u.buf.Reset()

		// Write anything past the index to the circular buffer for the
		// next event.
		if idx < len(msg) {
			msg = msg[idx+1:]
		} else {
			msg = ""
		}

		// Combine the data from the buffer up to the newline so we
		// have the full line, and split that by the commas.
		parts := strings.Split(buf, ",")
		if len(parts) < 3 {
			// Uh, invalid event?
			log.Printf("[ERROR] Invalid Packer event line: %s", buf)
			return
		}

		// Look for the callback
		cb, ok := u.Callbacks[parts[2]]
		if !ok {
			// No callback registered for this type, drop it
			return
		}

		// We have a callback, construct the output!
		var data []string
		if len(parts) > 3 {
			data = make([]string, len(parts)-3)
			for i, raw := range parts[3:] {
				data[i] = strings.Replace(
					strings.Replace(
						strings.Replace(raw, "%!(PACKER_COMMA)", ",", -1),
						"\\n", "\n", -1),
					"\\r", "\r", -1)
			}
		}

		// Callback
		cb(&Output{
			Timestamp: parts[0],
			Target:    parts[1],
			Type:      parts[2],
			Data:      data,
		})
	}
}

func (u *packerUi) init() {
	// Allocating the circular buffer. The circular buffer should only
	// keep track up to the point that there is a \n found so it doesn't
	// need to be huge, but it also limits the max length of an event.
	var err error
	u.buf, err = circbuf.NewBuffer(4096)
	if err != nil {
		panic(err)
	}
}

// For testing
func (o *Output) GoString() string {
	return fmt.Sprintf("*%#v", *o)
}
<commit_msg>helper/packer: update comment<commit_after>package packer

import (
	"fmt"
	"log"
	"strings"
	"sync"

	"github.com/armon/circbuf"
)

// OutputCallback is the type that is called when there is a matching
// machine-readable output line for the UI.
type OutputCallback func(*Output)

// Output is a single line of machine-readable output from Packer.
type Output struct {
	Timestamp string
	Target    string
	Type      string
	Data      []string
}

// packerUi is an implementation of ui.Ui that we pass to helper/exec
// that parses the machine-readable output of Packer and calls callbacks
// for the various entries.
type packerUi struct {
	Callbacks map[string]OutputCallback

	buf  *circbuf.Buffer
	once sync.Once
}

// Finish should be called when you're done to force the final newline
// to parse the final event.
func (u *packerUi) Finish() {
	// We only need to do anything if we have stuff in our buffer
	if u.buf != nil && u.buf.TotalWritten() > 0 {
		u.Raw("\n")
	}
}

// Ignore Header and Message because we don't use them from helper/exec
func (u *packerUi) Header(string)  {}
func (u *packerUi) Message(string) {}

func (u *packerUi) Raw(msg string) {
	if msg == "" {
		// Not sure how this would happen, but there is nothing to do here.
		return
	}

	u.once.Do(u.init)

	// We loop while we have a newline in the message
	for {
		// Get the index for the newline if there is one
		idx := strings.IndexRune(msg, '\n')
		if idx == -1 {
			// The newline isn't there, write it to the circular buffer
			// and wait longer.
			u.buf.Write([]byte(msg))
			break
		}

		// We got a newline! Grab the contents from the circular buffer and
		// copy it so we can clear the buffer.
		bufRaw := u.buf.Bytes()
		buf := string(bufRaw) + msg[:idx]
		bufRaw = nil
		u.buf.Reset()

		// If we have more data, clear msg to that point. If we have
		// no more data, then just set message to empty and we'll break
		// on the next loop iteration
		if idx < len(msg) {
			msg = msg[idx+1:]
		} else {
			msg = ""
		}

		// Combine the data from the buffer up to the newline so we
		// have the full line, and split that by the commas.
		parts := strings.Split(buf, ",")
		if len(parts) < 3 {
			// Uh, invalid event?
			log.Printf("[ERROR] Invalid Packer event line: %s", buf)
			return
		}

		// Look for the callback
		cb, ok := u.Callbacks[parts[2]]
		if !ok {
			// No callback registered for this type, drop it
			return
		}

		// We have a callback, construct the output!
		var data []string
		if len(parts) > 3 {
			data = make([]string, len(parts)-3)
			for i, raw := range parts[3:] {
				data[i] = strings.Replace(
					strings.Replace(
						strings.Replace(raw, "%!(PACKER_COMMA)", ",", -1),
						"\\n", "\n", -1),
					"\\r", "\r", -1)
			}
		}

		// Callback
		cb(&Output{
			Timestamp: parts[0],
			Target:    parts[1],
			Type:      parts[2],
			Data:      data,
		})
	}
}

func (u *packerUi) init() {
	// Allocating the circular buffer. The circular buffer should only
	// keep track up to the point that there is a \n found so it doesn't
	// need to be huge, but it also limits the max length of an event.
	var err error
	u.buf, err = circbuf.NewBuffer(4096)
	if err != nil {
		panic(err)
	}
}

// For testing
func (o *Output) GoString() string {
	return fmt.Sprintf("*%#v", *o)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"strconv"
	"time"
)

type HourObservation struct {
	Year_rtm              int
	Day_rtm               int
	Hourminute_rtm        int
	Air_temp107_avg       sql.NullFloat64
	Relative_humidity_avg sql.NullFloat64
	Solar_radiation_avg   sql.NullFloat64
	Soil_temp_q_avg       sql.NullFloat64
	Soil_moisture_5_cm    sql.NullFloat64
	Soil_moisture_20_cm   sql.NullFloat64
	Wind_direction_d1_wvt sql.NullFloat64
	Wind_speed_wvt        sql.NullFloat64
	Rain_mm               sql.NullFloat64
	Battery_voltage_min   sql.NullFloat64
	Datetime              time.Time
}

func (d *HourObservation) toMawn() []string {
	values := []string{
		"60",
		strconv.Itoa(d.Year_rtm),
		strconv.Itoa(d.Day_rtm),
		strconv.Itoa(d.Hourminute_rtm),
		floatToString(d.Air_temp107_avg),
		"nil",
		floatToString(d.Relative_humidity_avg),
		// floatToString(d.Solar_radiation_avg),
		floatToString(d.Soil_temp_q_avg),
		"nil",
		floatToString(d.Soil_moisture_5_cm),
		floatToString(d.Soil_moisture_20_cm),
		floatToString(d.Wind_direction_d1_wvt),
		floatToString(d.Wind_speed_wvt),
		"nil", "nil",
		floatToString(d.Rain_mm),
		"nil", "nil",
		floatToString(d.Battery_voltage_min),
		d.Datetime.Format(time.RFC3339),
	}
	return values
}

func (d *HourObservation) mawnHeader() []string {
	values := []string{
		"#code",
		"year",
		"day of year",
		"report hour minute",
		"air temperature",
		"relative humidity",
		"solar radiation",
		"soil temperature at 5 cm",
		"soil temperature at 10 cm",
		"soil  moisture at 5 cm",
		"soil moisture at 20 cm",
		"wind direction",
		"wind speed",
		"maximum hourly wind speed",
		"time of maximum wind speed",
		"precipitation",
		"leaf0",
		"leaf1",
		"battery voltage minimum",
		"timestamp",
	}
	return values
}

func (d *HourObservation) mawnUnit() []string {
	values := []string{
		"#",
		"",
		"",
		"",
		"C",
		"%",
		"kJ/m^2",
		"C",
		"C",
		"%",
		"%",
		"degrees",
		"m/s",
		"m/s",
		"",
		"mm",
		"",
		"",
		"",
		"V",
	}
	return values
}

func hour_observations(db *sqlx.DB, c *gin.Context) {
	rows, err := db.Queryx("select * from (select Air_temp107_avg,Relative_humidity_avg,Solar_radiation_avg,Soil_temp_q_avg,Soil_moisture_5_cm,Soil_moisture_20_cm,Wind_direction_d1_wvt,Wind_speed_wvt,Rain_mm,Battery_voltage_min,Datetime from weather.lter_hour_d where datetime < now() - interval '1 hour' order by datetime desc limit $1) t1 order by datetime", limit(c, 97))
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	i := 0
	writer := csv.NewWriter(c.Writer)

	obs := HourObservation{}
	writer.Write(obs.mawnHeader())
	writer.Write(obs.mawnUnit())
	for rows.Next() {
		if err := rows.StructScan(&obs); err != nil {
			log.Fatal(err)
		}

		obs.Year_rtm, obs.Day_rtm, obs.Hourminute_rtm = CampbellTime(obs.Datetime.Local())

		obs.Solar_radiation_avg.Float64 = obs.Solar_radiation_avg.Float64 * 0.6977 * 3600

		writer.Write(obs.toMawn())

		if i%500 == 0 {
			writer.Flush()
		}
		i = i + 1

	}
	writer.Flush()

}
<commit_msg>updated query to use data from the raingauge directly<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"strconv"
	"time"
)

type HourObservation struct {
	Year_rtm              int
	Day_rtm               int
	Hourminute_rtm        int
	Air_temp107_avg       sql.NullFloat64
	Relative_humidity_avg sql.NullFloat64
	Solar_radiation_avg   sql.NullFloat64
	Soil_temp_q_avg       sql.NullFloat64
	Soil_moisture_5_cm    sql.NullFloat64
	Soil_moisture_20_cm   sql.NullFloat64
	Wind_direction_d1_wvt sql.NullFloat64
	Wind_speed_wvt        sql.NullFloat64
	Rain_mm               sql.NullFloat64
	Battery_voltage_min   sql.NullFloat64
	Datetime              time.Time
}

func (d *HourObservation) toMawn() []string {
	values := []string{
		"60",
		strconv.Itoa(d.Year_rtm),
		strconv.Itoa(d.Day_rtm),
		strconv.Itoa(d.Hourminute_rtm),
		floatToString(d.Air_temp107_avg),
		"nil",
		floatToString(d.Relative_humidity_avg),
		// floatToString(d.Solar_radiation_avg),
		floatToString(d.Soil_temp_q_avg),
		"nil",
		floatToString(d.Soil_moisture_5_cm),
		floatToString(d.Soil_moisture_20_cm),
		floatToString(d.Wind_direction_d1_wvt),
		floatToString(d.Wind_speed_wvt),
		"nil", "nil",
		floatToString(d.Rain_mm),
		"nil", "nil",
		floatToString(d.Battery_voltage_min),
		d.Datetime.Format(time.RFC3339),
	}
	return values
}

func (d *HourObservation) mawnHeader() []string {
	values := []string{
		"#code",
		"year",
		"day of year",
		"report hour minute",
		"air temperature",
		"relative humidity",
		"solar radiation",
		"soil temperature at 5 cm",
		"soil temperature at 10 cm",
		"soil  moisture at 5 cm",
		"soil moisture at 20 cm",
		"wind direction",
		"wind speed",
		"maximum hourly wind speed",
		"time of maximum wind speed",
		"precipitation",
		"leaf0",
		"leaf1",
		"battery voltage minimum",
		"timestamp",
	}
	return values
}

func (d *HourObservation) mawnUnit() []string {
	values := []string{
		"#",
		"",
		"",
		"",
		"C",
		"%",
		"kJ/m^2",
		"C",
		"C",
		"%",
		"%",
		"degrees",
		"m/s",
		"m/s",
		"",
		"mm",
		"",
		"",
		"",
		"V",
	}
	return values
}

func hour_observations(db *sqlx.DB, c *gin.Context) {
	rows, err := db.Queryx("select * from ( select Air_temp107_avg,Relative_humidity_avg,Solar_radiation_avg, Soil_temp_q_avg,Soil_moisture_5_cm,Soil_moisture_20_cm,Wind_direction_d1_wvt, Wind_speed_wvt,hourly_rainfall.rain_mm,Battery_voltage_min,Datetime from weather.lter_hour_d join weather.hourly_rainfall on hourly_rainfall.hours = lter_hour_d.datetime where datetime < now() - interval '1 hour' order by datetime desc limit $1) t1 order by datetime", limit(c, 97))
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	i := 0
	writer := csv.NewWriter(c.Writer)

	obs := HourObservation{}
	writer.Write(obs.mawnHeader())
	writer.Write(obs.mawnUnit())
	for rows.Next() {
		if err := rows.StructScan(&obs); err != nil {
			log.Fatal(err)
		}

		obs.Year_rtm, obs.Day_rtm, obs.Hourminute_rtm = CampbellTime(obs.Datetime.Local())

		obs.Solar_radiation_avg.Float64 = obs.Solar_radiation_avg.Float64 * 0.6977 * 3600

		writer.Write(obs.toMawn())

		if i%500 == 0 {
			writer.Flush()
		}
		i = i + 1

	}
	writer.Flush()

}
<|endoftext|>


<commit_before>package candidate

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/gigawattio/concurrency"
	zkutil "github.com/gigawattio/zklib/util"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

const (
	LeaderChanSize = 6
)

var (
	AlreadyRegisteredError  = errors.New("already registered")
	NotRegisteredError      = errors.New("not registered")
	NotEnrolledError        = errors.New("candidate is not presently enrolled in the election")
	InvalidChildrenLenError = errors.New("0 children listed afer created protected ephemeral sequential - this should never happen")
)

var (
	worldAllAcl = zk.WorldACL(zk.PermAll)
)

type Candidate struct {
	ElectionPath     string
	Node             *Node
	zNode            string
	zNodeLock        sync.RWMutex
	registered       bool
	registrationLock sync.Mutex
	stopChan         chan chan struct{}
	Debug            bool
}

func New(electionPath string, node *Node) *Candidate {
	candidate := &Candidate{
		ElectionPath: electionPath,
		Node:         node,
		stopChan:     make(chan chan struct{}, 1),
	}
	return candidate
}

func (c *Candidate) Registered() bool {
	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()
	return c.registered
}

func (c *Candidate) Register(conn *zk.Conn) (<-chan *Node, error) {
	log.Infof("[uuid=%v] Candidate registering..", c.Node.Uuid)

	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()

	if c.registered {
		return nil, AlreadyRegisteredError
	}
	c.registered = true

	enroll := func() (leader *Node, watch <-chan zk.Event, err error) {
		log.Infof("[uuid=%v] Candidate enrolling..", c.Node.Uuid)

		var (
			zNode    string
			data     []byte
			children []string
		)

		if zNode, err = c.validZNode(conn); err != nil {
			return
		}
		if zNode == "" {
			if err = c.ensureElectionPathExists(conn); err != nil {
				return
			}
			if data, err = json.Marshal(c.Node); err != nil {
				err = fmt.Errorf("serializing node data: %s", err)
				return
			}
			// log.Debugf("CPES for node=%v", string(data))
			if zNode, err = conn.CreateProtectedEphemeralSequential(c.ElectionPath+"/n_", data, worldAllAcl); err != nil {
				err = fmt.Errorf("creating protected ephemeral sequential %q: %s", c.ElectionPath, err)
				return
			}
			if idx := strings.LastIndex(zNode, "/"); idx > 0 {
				zNode = zNode[idx+1:]
			} else {
				err = fmt.Errorf("received invalid zxid=%v", zNode)
				return
			}
			if c.Debug {
				log.Debugf("[uuid=%v] Candidate has new zNode=%v", c.Node.Uuid, zNode)
			}
			c.zNodeLock.Lock()
			c.zNode = zNode
			c.zNodeLock.Unlock()
		}

		if children, _, watch, err = conn.ChildrenW(c.ElectionPath); err != nil {
			return
		}
		sort.Sort(ZNodes(children))
		if c.Debug {
			log.Debugf("[uuid=%v] Candidate children=%v", c.Node.Uuid, children)
		}

		if children[0] == zNode {
			leader = c.Node
		} else if leader, err = c.getNode(conn, children[0]); err != nil {
			return
		}

		log.Infof("[uuid=%v] Candidate enrolled OK", c.Node.Uuid)
		return
	}

	type asyncEnrollResult struct {
		leader *Node
		watch  <-chan zk.Event
		err    error
	}

	asyncEnroll := func() chan *asyncEnrollResult {
		resultChan := make(chan *asyncEnrollResult)
		go func() {
			leader, watch, err := enroll()
			resultChan <- &asyncEnrollResult{
				leader: leader,
				watch:  watch,
				err:    err,
			}
		}()
		return resultChan
	}

	leaderChan := make(chan *Node, LeaderChanSize)

	// Watcher.
	go func() {
		var (
			leader *Node
			watch  <-chan zk.Event
		)

		for {
			if watch == nil {
				select {
				case result := <-asyncEnroll():
					if result.err != nil {
						log.Errorf("[uuid=%v] Candidate watcher enrollment err=%s", c.Node.Uuid, result.err)
						time.Sleep(1 * time.Second) // TODO: Definitely use backoff here.
						continue
					} else if result.leader != leader {
						leader = result.leader

						select {
						case leaderChan <- leader:
							// pass
						default:
						}
					}
					watch = result.watch

				case ackCh := <-c.stopChan:
					close(leaderChan)
					ackCh <- struct{}{}
					return
				}
			}

			select {
			case event, ok := <-watch:
				if !ok {
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher detected watch chan closed, triggering enroll", c.Node.Uuid)
					}
					watch = nil
					continue
				}

				switch event.Type {
				case zk.EventNodeDeleted, zk.EventNotWatching, zk.EventNodeChildrenChanged:
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher received event=%+v, triggering enroll", c.Node.Uuid, event)
					}
					watch = nil

				default:
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher received misc event=%+v", c.Node.Uuid, event)
					}
				}

			case ackCh := <-c.stopChan:
				close(leaderChan)
				c.wipeZNode(conn)
				ackCh <- struct{}{}
				return
			}
		}
	}()

	log.Infof("[uuid=%v] Candidate registered OK", c.Node.Uuid)
	return leaderChan, nil
}

func (c *Candidate) wipeZNode(conn *zk.Conn) {
	c.zNodeLock.Lock()
	defer c.zNodeLock.Unlock()

	path := c.ElectionPath + "/" + c.zNode
	exists, stat, err := conn.Exists(path)
	if err != nil {
		log.Warnf("[uuid=%v] Candidate zNode=%v deletion skipped due to existence check err=%s", c.Node.Uuid, c.zNode, err)
	} else if exists {
		if err = conn.Delete(path, stat.Version); err != nil {
			log.Warnf("[uuid=%v] Candidate zNode=%v deletion failed due to delete err=%s", c.Node.Uuid, c.zNode, err)
		} else {
			log.Infof("[uuid=%v] Candidate successfully deleted zNode path=%v", c.Node.Uuid, path)
		}
	} else {
		log.Infof("[uuid=%v] Candidate zNode=%v deletion skipped due to !exists", c.Node.Uuid, c.zNode)
	}
}

func (c *Candidate) Unregister() error {
	log.Infof("[uuid=%v] Candidate unregistering..", c.Node.Uuid)

	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()

	if !c.registered {
		return NotRegisteredError
	}
	c.registered = false

	ackCh := make(chan struct{})
	c.stopChan <- ackCh
	<-ackCh

	log.Infof("[uuid=%v] Candidate unregistered", c.Node.Uuid)
	return nil
}

func (c *Candidate) validZNode(conn *zk.Conn) (zNode string, err error) {
	c.zNodeLock.RLock()
	zNode = c.zNode
	c.zNodeLock.RUnlock()

	if zNode == "" {
		return
	}

	var (
		path   = c.ElectionPath + "/" + zNode
		exists bool
	)

	if exists, _, err = conn.Exists(path); err != nil {
		err = fmt.Errorf("checking if zNode=%v exists: %s", path, err)
	} else if !exists {
		zNode = ""
	}
	return
}

func (c *Candidate) ensureElectionPathExists(conn *zk.Conn) error {
	exists, _, err := conn.Exists(c.ElectionPath)
	if err != nil {
		return fmt.Errorf("checking if electionPath=%v exists: %s", c.ElectionPath, err)
	}
	if !exists {
		if _, err = zkutil.CreateP(conn, c.ElectionPath, []byte{}, 0, worldAllAcl); err != nil && err != zk.ErrNodeExists {
			return fmt.Errorf("creating electionPath=%v", c.ElectionPath, err)
		}
	}
	return nil
}

func (c *Candidate) getNode(conn *zk.Conn, zNode string) (node *Node, err error) {
	var data []byte
	if data, _, err = conn.Get(c.ElectionPath + "/" + zNode); err != nil {
		err = fmt.Errorf("getting node data for zNode=%v: %s", zNode, err)
		return
	}
	node = &Node{}
	if err = json.Unmarshal(data, node); err != nil {
		err = fmt.Errorf("deserializing node data for zNode=%v: %s", zNode, err)
		return
	}
	return
}

func (c *Candidate) Participants(conn *zk.Conn) (participants []Node, err error) {
	var (
		children []string
	)
	if children, err = c.children(conn); err != nil {
		return
	}

	var (
		numChildren      = len(children)
		collectorFuncs   = make([]func() error, 0, numChildren)
		participantsLock sync.Mutex
	)
	participants = make([]Node, 0, numChildren)

	for _, zNode := range children {
		func(zNode string) {
			collectorFunc := func() (err error) {
				var node *Node
				if node, err = c.getNode(conn, zNode); err != nil {
					return
				}
				participantsLock.Lock()
				participants = append(participants, *node)
				participantsLock.Unlock()
				return
			}
			collectorFuncs = append(collectorFuncs, collectorFunc)
		}(zNode)
	}

	if err = concurrency.MultiGo(collectorFuncs...); err != nil {
		return
	}
	return
}

// func (c *Candidate) ZNode() string {
// 	c.zNodeLock.RLock()
// 	defer c.zNodeLock.RUnlock()
// 	return c.zNode
// }

func (c *Candidate) children(conn *zk.Conn) (children []string, err error) {
	if children, _, err = conn.Children(c.ElectionPath); err != nil {
		err = fmt.Errorf("listing children: %s", err)
		return
	}
	if len(children) == 0 {
		err = InvalidChildrenLenError
		return
	}
	sort.Strings(children)
	return
}
<commit_msg>ZK connection health monitoring.<commit_after>package candidate

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/gigawattio/concurrency"
	zkutil "github.com/gigawattio/zklib/util"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

const (
	LeaderChanSize = 6
)

var (
	AlreadyRegisteredError  = errors.New("already registered")
	NotRegisteredError      = errors.New("not registered")
	NotEnrolledError        = errors.New("candidate is not presently enrolled in the election")
	InvalidChildrenLenError = errors.New("0 children listed afer created protected ephemeral sequential - this should never happen")

	DefaultConnCheckFrequency = 5 * time.Second // Default frequency of connection checks.
)

var (
	worldAllAcl = zk.WorldACL(zk.PermAll)
)

type Candidate struct {
	ElectionPath       string
	Node               *Node
	ConnCheckFrequency time.Duration // Frequency of connection checks.
	zNode              string
	zNodeLock          sync.RWMutex
	registered         bool
	registrationLock   sync.Mutex
	stopChan           chan chan struct{}
	Debug              bool
}

func New(electionPath string, node *Node) *Candidate {
	candidate := &Candidate{
		ElectionPath:       electionPath,
		Node:               node,
		ConnCheckFrequency: DefaultConnCheckFrequency,
		stopChan:           make(chan chan struct{}, 1),
	}
	return candidate
}

func (c *Candidate) Registered() bool {
	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()
	return c.registered
}

func (c *Candidate) Register(conn *zk.Conn) (<-chan *Node, error) {
	log.Infof("[uuid=%v] Candidate registering..", c.Node.Uuid)

	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()

	if c.registered {
		return nil, AlreadyRegisteredError
	}
	c.registered = true

	enroll := func() (leader *Node, watch <-chan zk.Event, err error) {
		log.Infof("[uuid=%v] Candidate enrolling..", c.Node.Uuid)

		var (
			zNode    string
			data     []byte
			children []string
		)

		if zNode, err = c.validZNode(conn); err != nil {
			return
		}
		if zNode == "" {
			if err = c.ensureElectionPathExists(conn); err != nil {
				return
			}
			if data, err = json.Marshal(c.Node); err != nil {
				err = fmt.Errorf("serializing node data: %s", err)
				return
			}
			// log.Debugf("CPES for node=%v", string(data))
			if zNode, err = conn.CreateProtectedEphemeralSequential(c.ElectionPath+"/n_", data, worldAllAcl); err != nil {
				err = fmt.Errorf("creating protected ephemeral sequential %q: %s", c.ElectionPath, err)
				return
			}
			if idx := strings.LastIndex(zNode, "/"); idx > 0 {
				zNode = zNode[idx+1:]
			} else {
				err = fmt.Errorf("received invalid zxid=%v", zNode)
				return
			}
			if c.Debug {
				log.Debugf("[uuid=%v] Candidate has new zNode=%v", c.Node.Uuid, zNode)
			}
			c.zNodeLock.Lock()
			c.zNode = zNode
			c.zNodeLock.Unlock()
		}

		if children, _, watch, err = conn.ChildrenW(c.ElectionPath); err != nil {
			return
		}
		sort.Sort(ZNodes(children))
		if c.Debug {
			log.Debugf("[uuid=%v] Candidate children=%v", c.Node.Uuid, children)
		}

		if children[0] == zNode {
			leader = c.Node
		} else if leader, err = c.getNode(conn, children[0]); err != nil {
			return
		}

		log.Infof("[uuid=%v] Candidate enrolled OK", c.Node.Uuid)
		return
	}

	type asyncEnrollResult struct {
		leader *Node
		watch  <-chan zk.Event
		err    error
	}

	asyncEnroll := func() chan *asyncEnrollResult {
		resultChan := make(chan *asyncEnrollResult)
		go func() {
			leader, watch, err := enroll()
			resultChan <- &asyncEnrollResult{
				leader: leader,
				watch:  watch,
				err:    err,
			}
		}()
		return resultChan
	}

	leaderChan := make(chan *Node, LeaderChanSize)

	// Watcher.
	go func() {
		var (
			leader *Node
			watch  <-chan zk.Event
		)

		for {
			if watch == nil {
				select {
				case result := <-asyncEnroll():
					if result.err != nil {
						log.Errorf("[uuid=%v] Candidate watcher enrollment err=%s", c.Node.Uuid, result.err)
						time.Sleep(1 * time.Second) // TODO: Definitely use backoff here.
						continue
					} else if result.leader != leader {
						leader = result.leader

						if c.Debug && leader != nil {
							log.Debugf("[uuid=%v] Sending updated leader=%+v", c.Node.Uuid, *leader)
						}

						select {
						case leaderChan <- leader:
							// pass
						default:
						}
					}
					watch = result.watch

				case ackCh := <-c.stopChan:
					close(leaderChan)
					ackCh <- struct{}{}
					return
				}
			}

			select {
			case event, ok := <-watch:
				if !ok {
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher detected watch chan closed", c.Node.Uuid)
					}
					watch = nil
					continue
				}

				switch event.Type {
				case zk.EventNodeDeleted, zk.EventNotWatching, zk.EventNodeChildrenChanged:
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher received event=%+v, triggering enroll", c.Node.Uuid, event)
					}
					watch = nil

				default:
					if c.Debug {
						log.Debugf("[uuid=%v] Candidate watcher received misc event=%+v", c.Node.Uuid, event)
					}
				}

			case <-time.After(c.ConnCheckFrequency):
				if leader != nil {
					state := conn.State()
					if state == zk.StateConnecting || state == zk.StateDisconnected || state == zk.StateExpired || state == zk.StateAuthFailed || state == zk.StateUnknown {
						// When connection state is deemed unhealthy, reset the values for watch
						// and leader.
						watch = nil
						leader = nil

						if c.Debug {
							log.Debugf("[uuid=%v] Unhealthy connection state deteted, values for leader and watch have been reset", c.Node.Uuid)
						}

						select {
						case leaderChan <- leader:
							// pass
						default:
						}
					}
				}

			case ackCh := <-c.stopChan:
				close(leaderChan)
				c.wipeZNode(conn)
				ackCh <- struct{}{}
				return
			}
		}
	}()

	log.Infof("[uuid=%v] Candidate registered OK", c.Node.Uuid)
	return leaderChan, nil
}

func (c *Candidate) wipeZNode(conn *zk.Conn) {
	c.zNodeLock.Lock()
	defer c.zNodeLock.Unlock()

	path := c.ElectionPath + "/" + c.zNode
	exists, stat, err := conn.Exists(path)
	if err != nil {
		log.Warnf("[uuid=%v] Candidate zNode=%v deletion skipped due to existence check err=%s", c.Node.Uuid, c.zNode, err)
	} else if exists {
		if err = conn.Delete(path, stat.Version); err != nil {
			log.Warnf("[uuid=%v] Candidate zNode=%v deletion failed due to delete err=%s", c.Node.Uuid, c.zNode, err)
		} else {
			log.Infof("[uuid=%v] Candidate successfully deleted zNode path=%v", c.Node.Uuid, path)
		}
	} else {
		log.Infof("[uuid=%v] Candidate zNode=%v deletion skipped due to !exists", c.Node.Uuid, c.zNode)
	}
}

func (c *Candidate) Unregister() error {
	log.Infof("[uuid=%v] Candidate unregistering..", c.Node.Uuid)

	c.registrationLock.Lock()
	defer c.registrationLock.Unlock()

	if !c.registered {
		return NotRegisteredError
	}
	c.registered = false

	ackCh := make(chan struct{})
	c.stopChan <- ackCh
	<-ackCh

	log.Infof("[uuid=%v] Candidate unregistered", c.Node.Uuid)
	return nil
}

func (c *Candidate) validZNode(conn *zk.Conn) (zNode string, err error) {
	c.zNodeLock.RLock()
	zNode = c.zNode
	c.zNodeLock.RUnlock()

	if zNode == "" {
		return
	}

	var (
		path   = c.ElectionPath + "/" + zNode
		exists bool
	)

	if exists, _, err = conn.Exists(path); err != nil {
		err = fmt.Errorf("checking if zNode=%v exists: %s", path, err)
	} else if !exists {
		zNode = ""
	}
	return
}

func (c *Candidate) ensureElectionPathExists(conn *zk.Conn) error {
	exists, _, err := conn.Exists(c.ElectionPath)
	if err != nil {
		return fmt.Errorf("checking if electionPath=%v exists: %s", c.ElectionPath, err)
	}
	if !exists {
		if _, err = zkutil.CreateP(conn, c.ElectionPath, []byte{}, 0, worldAllAcl); err != nil && err != zk.ErrNodeExists {
			return fmt.Errorf("creating electionPath=%v", c.ElectionPath, err)
		}
	}
	return nil
}

func (c *Candidate) getNode(conn *zk.Conn, zNode string) (node *Node, err error) {
	var data []byte
	if data, _, err = conn.Get(c.ElectionPath + "/" + zNode); err != nil {
		err = fmt.Errorf("getting node data for zNode=%v: %s", zNode, err)
		return
	}
	node = &Node{}
	if err = json.Unmarshal(data, node); err != nil {
		err = fmt.Errorf("deserializing node data for zNode=%v: %s", zNode, err)
		return
	}
	return
}

func (c *Candidate) Participants(conn *zk.Conn) (participants []Node, err error) {
	var (
		children []string
	)
	if children, err = c.children(conn); err != nil {
		return
	}

	var (
		numChildren      = len(children)
		collectorFuncs   = make([]func() error, 0, numChildren)
		participantsLock sync.Mutex
	)
	participants = make([]Node, 0, numChildren)

	for _, zNode := range children {
		func(zNode string) {
			collectorFunc := func() (err error) {
				var node *Node
				if node, err = c.getNode(conn, zNode); err != nil {
					return
				}
				participantsLock.Lock()
				participants = append(participants, *node)
				participantsLock.Unlock()
				return
			}
			collectorFuncs = append(collectorFuncs, collectorFunc)
		}(zNode)
	}

	if err = concurrency.MultiGo(collectorFuncs...); err != nil {
		return
	}
	return
}

// func (c *Candidate) ZNode() string {
// 	c.zNodeLock.RLock()
// 	defer c.zNodeLock.RUnlock()
// 	return c.zNode
// }

func (c *Candidate) children(conn *zk.Conn) (children []string, err error) {
	if children, _, err = conn.Children(c.ElectionPath); err != nil {
		err = fmt.Errorf("listing children: %s", err)
		return
	}
	if len(children) == 0 {
		err = InvalidChildrenLenError
		return
	}
	sort.Strings(children)
	return
}
<|endoftext|>


<commit_before>package httptrace

import (
	"log"
	"net/http"
	"time"

	"sourcegraph.com/sourcegraph/appdash"
)

func init() { appdash.RegisterEvent(ServerEvent{}) }

// NewServerEvent returns an event which records various aspects of an
// HTTP response. It takes an HTTP request, not response, as input
// because the information it records is derived from the request, and
// HTTP handlers don't have access to the response struct (only
// http.ResponseWriter, which requires wrapping or buffering to
// introspect).
//
// The returned value is incomplete and should have its Response and
// ServerRecv/ServerSend values set before being logged.
func NewServerEvent(r *http.Request) *ServerEvent {
	return &ServerEvent{Request: requestInfo(r)}
}

// ResponseInfo describes an HTTP response.
type ResponseInfo struct {
	Headers       map[string]string
	ContentLength int64
	StatusCode    int
}

func responseInfo(r *http.Response) ResponseInfo {
	return ResponseInfo{
		Headers:       redactHeaders(r.Header, r.Trailer),
		ContentLength: r.ContentLength,
		StatusCode:    r.StatusCode,
	}
}

// ServerEvent records an HTTP server request handling event.
type ServerEvent struct {
	Request    RequestInfo  `trace:"Server.Request"`
	Response   ResponseInfo `trace:"Server.Response"`
	Route      string       `trace:"Server.Route"`
	User       string       `trace:"Server.User"`
	ServerRecv time.Time    `trace:"Server.Recv"`
	ServerSend time.Time    `trace:"Server.Send"`
}

// Schema returns the constant "HTTPServer".
func (ServerEvent) Schema() string { return "HTTPServer" }

// Important implements the appdash ImportantEvent.
func (ServerEvent) Important() []string {
	return []string{"Server.Response.StatusCode"}
}

// Start implements the appdash TimespanEvent interface.
func (e ServerEvent) Start() time.Time { return e.ServerRecv }

// End implements the appdash TimespanEvent interface.
func (e ServerEvent) End() time.Time { return e.ServerSend }

// Middleware creates a new http.Handler middleware
// (negroni-compliant) that records incoming HTTP requests to the
// collector c as "HTTPServer"-schema events.
func Middleware(c appdash.Collector, conf *MiddlewareConfig) func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	return func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
		spanID, spanFromHeader, err := getSpanID(r.Header)
		if err != nil {
			log.Printf("Warning: invalid %s header: %s. (Continuing with request handling.)", spanFromHeader, err)
		}
		usingProvidedSpanID := (spanFromHeader == HeaderSpanID)

		if conf.SetContextSpan != nil {
			conf.SetContextSpan(r, *spanID)
		}

		e := NewServerEvent(r)
		e.ServerRecv = time.Now()
		if conf.RouteName != nil {
			e.Route = conf.RouteName(r)
		}
		if conf.CurrentUser != nil {
			e.User = conf.CurrentUser(r)
		}

		rr := &responseInfoRecorder{ResponseWriter: rw}
		next(rr, r)
		SetSpanIDHeader(rr.Header(), *spanID)

		if !usingProvidedSpanID {
			e.Request = requestInfo(r)
		}
		e.Response = responseInfo(rr.partialResponse())
		e.ServerSend = time.Now()

		rec := appdash.NewRecorder(*spanID, c)
		if e.Route != "" {
			rec.Name(e.Route)
		} else {
			rec.Name(e.Request.Host)
		}
		rec.Event(e)
	}
}

// MiddlewareConfig configures the HTTP tracing middleware.
type MiddlewareConfig struct {
	// RouteName, if non-nil, is called to get the current route's
	// name. This name is used as the span's name.
	RouteName func(*http.Request) string

	// CurrentUser, if non-nil, is called to get the current user ID
	// (which may be a login or a numeric ID).
	CurrentUser func(*http.Request) string

	// SetContextSpan, if non-nil, is called to set the span (which is
	// either taken from the client request header or created anew) in
	// the HTTP request context, so it may be used by other parts of
	// the handling process.
	SetContextSpan func(*http.Request, appdash.SpanID)
}

// responseInfoRecorder is an http.ResponseWriter that records a
// response's HTTP status code and body length and forwards all
// operations onto an underlying http.ResponseWriter, without
// buffering the response body.
type responseInfoRecorder struct {
	statusCode    int   // HTTP response status code
	ContentLength int64 // number of bytes written using the Write method

	http.ResponseWriter // underlying ResponseWriter to pass-thru to
}

// Write always succeeds and writes to r.Body, if not nil.
func (r *responseInfoRecorder) Write(b []byte) (int, error) {
	r.ContentLength += int64(len(b))
	if r.statusCode == 0 {
		r.statusCode = http.StatusOK
	}
	return r.ResponseWriter.Write(b)
}

func (r *responseInfoRecorder) StatusCode() int {
	if r.statusCode == 0 {
		return http.StatusOK
	}
	return r.statusCode
}

// WriteHeader sets r.Code.
func (r *responseInfoRecorder) WriteHeader(code int) {
	r.statusCode = code
	r.ResponseWriter.WriteHeader(code)
}

// partialResponse constructs a partial response object based on the
// information it is able to determine about the response.
func (r *responseInfoRecorder) partialResponse() *http.Response {
	return &http.Response{
		StatusCode:    r.StatusCode(),
		ContentLength: r.ContentLength,
		Header:        r.Header(),
	}
}
<commit_msg>http: responseInfoRecoder now implements http.Flusher<commit_after>package httptrace

import (
	"log"
	"net/http"
	"time"

	"sourcegraph.com/sourcegraph/appdash"
)

func init() { appdash.RegisterEvent(ServerEvent{}) }

// NewServerEvent returns an event which records various aspects of an
// HTTP response. It takes an HTTP request, not response, as input
// because the information it records is derived from the request, and
// HTTP handlers don't have access to the response struct (only
// http.ResponseWriter, which requires wrapping or buffering to
// introspect).
//
// The returned value is incomplete and should have its Response and
// ServerRecv/ServerSend values set before being logged.
func NewServerEvent(r *http.Request) *ServerEvent {
	return &ServerEvent{Request: requestInfo(r)}
}

// ResponseInfo describes an HTTP response.
type ResponseInfo struct {
	Headers       map[string]string
	ContentLength int64
	StatusCode    int
}

func responseInfo(r *http.Response) ResponseInfo {
	return ResponseInfo{
		Headers:       redactHeaders(r.Header, r.Trailer),
		ContentLength: r.ContentLength,
		StatusCode:    r.StatusCode,
	}
}

// ServerEvent records an HTTP server request handling event.
type ServerEvent struct {
	Request    RequestInfo  `trace:"Server.Request"`
	Response   ResponseInfo `trace:"Server.Response"`
	Route      string       `trace:"Server.Route"`
	User       string       `trace:"Server.User"`
	ServerRecv time.Time    `trace:"Server.Recv"`
	ServerSend time.Time    `trace:"Server.Send"`
}

// Schema returns the constant "HTTPServer".
func (ServerEvent) Schema() string { return "HTTPServer" }

// Important implements the appdash ImportantEvent.
func (ServerEvent) Important() []string {
	return []string{"Server.Response.StatusCode"}
}

// Start implements the appdash TimespanEvent interface.
func (e ServerEvent) Start() time.Time { return e.ServerRecv }

// End implements the appdash TimespanEvent interface.
func (e ServerEvent) End() time.Time { return e.ServerSend }

// Middleware creates a new http.Handler middleware
// (negroni-compliant) that records incoming HTTP requests to the
// collector c as "HTTPServer"-schema events.
func Middleware(c appdash.Collector, conf *MiddlewareConfig) func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	return func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
		spanID, spanFromHeader, err := getSpanID(r.Header)
		if err != nil {
			log.Printf("Warning: invalid %s header: %s. (Continuing with request handling.)", spanFromHeader, err)
		}
		usingProvidedSpanID := (spanFromHeader == HeaderSpanID)

		if conf.SetContextSpan != nil {
			conf.SetContextSpan(r, *spanID)
		}

		e := NewServerEvent(r)
		e.ServerRecv = time.Now()
		if conf.RouteName != nil {
			e.Route = conf.RouteName(r)
		}
		if conf.CurrentUser != nil {
			e.User = conf.CurrentUser(r)
		}

		rr := &responseInfoRecorder{ResponseWriter: rw}
		next(rr, r)
		SetSpanIDHeader(rr.Header(), *spanID)

		if !usingProvidedSpanID {
			e.Request = requestInfo(r)
		}
		e.Response = responseInfo(rr.partialResponse())
		e.ServerSend = time.Now()

		rec := appdash.NewRecorder(*spanID, c)
		if e.Route != "" {
			rec.Name(e.Route)
		} else {
			rec.Name(e.Request.Host)
		}
		rec.Event(e)
	}
}

// MiddlewareConfig configures the HTTP tracing middleware.
type MiddlewareConfig struct {
	// RouteName, if non-nil, is called to get the current route's
	// name. This name is used as the span's name.
	RouteName func(*http.Request) string

	// CurrentUser, if non-nil, is called to get the current user ID
	// (which may be a login or a numeric ID).
	CurrentUser func(*http.Request) string

	// SetContextSpan, if non-nil, is called to set the span (which is
	// either taken from the client request header or created anew) in
	// the HTTP request context, so it may be used by other parts of
	// the handling process.
	SetContextSpan func(*http.Request, appdash.SpanID)
}

// responseInfoRecorder is an http.ResponseWriter that records a
// response's HTTP status code and body length and forwards all
// operations onto an underlying http.ResponseWriter, without
// buffering the response body.
type responseInfoRecorder struct {
	statusCode    int   // HTTP response status code
	ContentLength int64 // number of bytes written using the Write method

	http.ResponseWriter // underlying ResponseWriter to pass-thru to
}

// Write always succeeds and writes to r.Body, if not nil.
func (r *responseInfoRecorder) Write(b []byte) (int, error) {
	r.ContentLength += int64(len(b))
	if r.statusCode == 0 {
		r.statusCode = http.StatusOK
	}
	return r.ResponseWriter.Write(b)
}

func (r *responseInfoRecorder) StatusCode() int {
	if r.statusCode == 0 {
		return http.StatusOK
	}
	return r.statusCode
}

// WriteHeader sets r.Code.
func (r *responseInfoRecorder) WriteHeader(code int) {
	r.statusCode = code
	r.ResponseWriter.WriteHeader(code)
}

// partialResponse constructs a partial response object based on the
// information it is able to determine about the response.
func (r *responseInfoRecorder) partialResponse() *http.Response {
	return &http.Response{
		StatusCode:    r.StatusCode(),
		ContentLength: r.ContentLength,
		Header:        r.Header(),
	}
}

func (r *responseInfoRecorder) Flush() {
	if f, ok := r.ResponseWriter.(http.Flusher); ok {
		f.Flush()
	}
}
<|endoftext|>