<commit_before>package namesys

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"time"

	dag "github.com/ipfs/go-ipfs/merkledag"
	pb "github.com/ipfs/go-ipfs/namesys/pb"
	path "github.com/ipfs/go-ipfs/path"
	pin "github.com/ipfs/go-ipfs/pin"
	dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
	ft "github.com/ipfs/go-ipfs/unixfs"

	ci "gx/ipfs/QmPGxZ1DP2w45WcogpW1h43BvseXbfke9N91qotpoQcUeS/go-libp2p-crypto"
	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
	routing "gx/ipfs/QmUc6twRJRE9MNrUGd8eo9WjHHxebGppdZfptGCASkR7fF/go-libp2p-routing"
	peer "gx/ipfs/QmWUswjn261LSyVxWAEpMVtPdy8zmKBJJfBpG3Qdpa8ZsE/go-libp2p-peer"
	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
	record "gx/ipfs/QmcTnycWsBgvNYFYgWdWi8SRDCeevG8HBUQHkvg4KLXUsW/go-libp2p-record"
	dhtpb "gx/ipfs/QmcTnycWsBgvNYFYgWdWi8SRDCeevG8HBUQHkvg4KLXUsW/go-libp2p-record/pb"
)

// ErrExpiredRecord should be returned when an ipns record is
// invalid due to being too old
var ErrExpiredRecord = errors.New("expired record")

// ErrUnrecognizedValidity is returned when an IpnsRecord has an
// unknown validity type.
var ErrUnrecognizedValidity = errors.New("unrecognized validity type")

const PublishPutValTimeout = time.Minute
const DefaultRecordTTL = 24 * time.Hour

var DefaultPublishLifetime = time.Hour * 24 * 7

// ipnsPublisher is capable of publishing and resolving names to the IPFS
// routing system.
type ipnsPublisher struct {
	routing routing.ValueStore
	ds      ds.Datastore
}

// NewRoutingPublisher constructs a publisher for the IPFS Routing name system.
func NewRoutingPublisher(route routing.ValueStore, ds ds.Datastore) *ipnsPublisher {
	if ds == nil {
		panic("nil datastore")
	}
	return &ipnsPublisher{routing: route, ds: ds}
}

// Publish implements Publisher. Accepts a keypair and a value,
// and publishes it out to the routing system
func (p *ipnsPublisher) Publish(ctx context.Context, k ci.PrivKey, value path.Path) error {
	log.Debugf("Publish %s", value)
	return p.PublishWithEOL(ctx, k, value, time.Now().Add(DefaultPublishLifetime))
}

// PublishWithEOL is a temporary stand in for the ipns records implementation
// see here for more details: https://github.com/ipfs/specs/tree/master/records
func (p *ipnsPublisher) PublishWithEOL(ctx context.Context, k ci.PrivKey, value path.Path, eol time.Time) error {

	id, err := peer.IDFromPrivateKey(k)
	if err != nil {
		return err
	}

	_, ipnskey := IpnsKeysForID(id)

	// get previous records sequence number
	seqnum, err := p.getPreviousSeqNo(ctx, ipnskey)
	if err != nil {
		return err
	}

	// increment it
	seqnum++

	return PutRecordToRouting(ctx, k, value, seqnum, eol, p.routing, id)
}

func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey string) (uint64, error) {
	prevrec, err := p.ds.Get(dshelp.NewKeyFromBinary([]byte(ipnskey)))
	if err != nil && err != ds.ErrNotFound {
		// None found, lets start at zero!
		return 0, err
	}
	var val []byte
	if err == nil {
		prbytes, ok := prevrec.([]byte)
		if !ok {
			return 0, fmt.Errorf("unexpected type returned from datastore: %#v", prevrec)
		}
		dhtrec := new(dhtpb.Record)
		err := proto.Unmarshal(prbytes, dhtrec)
		if err != nil {
			return 0, err
		}

		val = dhtrec.GetValue()
	} else {
		// try and check the dht for a record
		ctx, cancel := context.WithTimeout(ctx, time.Second*30)
		defer cancel()

		rv, err := p.routing.GetValue(ctx, ipnskey)
		if err != nil {
			// no such record found, start at zero!
			return 0, nil
		}

		val = rv
	}

	e := new(pb.IpnsEntry)
	err = proto.Unmarshal(val, e)
	if err != nil {
		return 0, err
	}

	return e.GetSequence(), nil
}

// setting the TTL on published records is an experimental feature.
// as such, i'm using the context to wire it through to avoid changing too
// much code along the way.
func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
	v := ctx.Value("ipns-publish-ttl")
	if v == nil {
		return 0, false
	}

	d, ok := v.(time.Duration)
	return d, ok
}

func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.ValueStore, id peer.ID) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	namekey, ipnskey := IpnsKeysForID(id)
	entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
	if err != nil {
		return err
	}

	ttl, ok := checkCtxTTL(ctx)
	if ok {
		entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
	}

	errs := make(chan error, 2)

	go func() {
		errs <- PublishEntry(ctx, r, ipnskey, entry)
	}()

	go func() {
		errs <- PublishPublicKey(ctx, r, namekey, k.GetPublic())
	}()

	err = waitOnErrChan(ctx, errs)
	if err != nil {
		return err
	}

	err = waitOnErrChan(ctx, errs)
	if err != nil {
		return err
	}

	return nil
}

func waitOnErrChan(ctx context.Context, errs chan error) error {
	select {
	case err := <-errs:
		return err
	case <-ctx.Done():
		return ctx.Err()
	}
}

func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk ci.PubKey) error {
	log.Debugf("Storing pubkey at: %s", k)
	pkbytes, err := pubk.Bytes()
	if err != nil {
		return err
	}

	// Store associated public key
	timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
	defer cancel()
	err = r.PutValue(timectx, k, pkbytes)
	if err != nil {
		return err
	}

	return nil
}

func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *pb.IpnsEntry) error {
	timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
	defer cancel()

	data, err := proto.Marshal(rec)
	if err != nil {
		return err
	}

	log.Debugf("Storing ipns entry at: %s", ipnskey)
	// Store ipns entry at "/ipns/"+b58(h(pubkey))
	if err := r.PutValue(timectx, ipnskey, data); err != nil {
		return err
	}

	return nil
}

func CreateRoutingEntryData(pk ci.PrivKey, val path.Path, seq uint64, eol time.Time) (*pb.IpnsEntry, error) {
	entry := new(pb.IpnsEntry)

	entry.Value = []byte(val)
	typ := pb.IpnsEntry_EOL
	entry.ValidityType = &typ
	entry.Sequence = proto.Uint64(seq)
	entry.Validity = []byte(u.FormatRFC3339(eol))

	sig, err := pk.Sign(ipnsEntryDataForSig(entry))
	if err != nil {
		return nil, err
	}
	entry.Signature = sig
	return entry, nil
}

func ipnsEntryDataForSig(e *pb.IpnsEntry) []byte {
	return bytes.Join([][]byte{
		e.Value,
		e.Validity,
		[]byte(fmt.Sprint(e.GetValidityType())),
	},
		[]byte{})
}

var IpnsRecordValidator = &record.ValidChecker{
	Func: ValidateIpnsRecord,
	Sign: true,
}

func IpnsSelectorFunc(k string, vals [][]byte) (int, error) {
	var recs []*pb.IpnsEntry
	for _, v := range vals {
		e := new(pb.IpnsEntry)
		err := proto.Unmarshal(v, e)
		if err == nil {
			recs = append(recs, e)
		} else {
			recs = append(recs, nil)
		}
	}

	return selectRecord(recs, vals)
}

func selectRecord(recs []*pb.IpnsEntry, vals [][]byte) (int, error) {
	var best_seq uint64
	best_i := -1

	for i, r := range recs {
		if r == nil || r.GetSequence() < best_seq {
			continue
		}

		if best_i == -1 || r.GetSequence() > best_seq {
			best_seq = r.GetSequence()
			best_i = i
		} else if r.GetSequence() == best_seq {
			rt, err := u.ParseRFC3339(string(r.GetValidity()))
			if err != nil {
				continue
			}

			bestt, err := u.ParseRFC3339(string(recs[best_i].GetValidity()))
			if err != nil {
				continue
			}

			if rt.After(bestt) {
				best_i = i
			} else if rt == bestt {
				if bytes.Compare(vals[i], vals[best_i]) > 0 {
					best_i = i
				}
			}
		}
	}
	if best_i == -1 {
		return 0, errors.New("no usable records in given set")
	}

	return best_i, nil
}

// ValidateIpnsRecord implements ValidatorFunc and verifies that the
// given 'val' is an IpnsEntry and that that entry is valid.
func ValidateIpnsRecord(k string, val []byte) error {
	entry := new(pb.IpnsEntry)
	err := proto.Unmarshal(val, entry)
	if err != nil {
		return err
	}
	switch entry.GetValidityType() {
	case pb.IpnsEntry_EOL:
		t, err := u.ParseRFC3339(string(entry.GetValidity()))
		if err != nil {
			log.Debug("failed parsing time for ipns record EOL")
			return err
		}
		if time.Now().After(t) {
			return ErrExpiredRecord
		}
	default:
		return ErrUnrecognizedValidity
	}
	return nil
}

// InitializeKeyspace sets the ipns record for the given key to
// point to an empty directory.
// TODO: this doesnt feel like it belongs here
func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error {
	emptyDir := ft.EmptyDirNode()
	nodek, err := ds.Add(emptyDir)
	if err != nil {
		return err
	}

	// pin recursively because this might already be pinned
	// and doing a direct pin would throw an error in that case
	err = pins.Pin(ctx, emptyDir, true)
	if err != nil {
		return err
	}

	err = pins.Flush()
	if err != nil {
		return err
	}

	err = pub.Publish(ctx, key, path.FromCid(nodek))
	if err != nil {
		return err
	}

	return nil
}

func IpnsKeysForID(id peer.ID) (name, ipns string) {
	namekey := "/pk/" + string(id)
	ipnskey := "/ipns/" + string(id)

	return namekey, ipnskey
}
<commit_msg>Update default record ttl<commit_after>package namesys

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"time"

	dag "github.com/ipfs/go-ipfs/merkledag"
	pb "github.com/ipfs/go-ipfs/namesys/pb"
	path "github.com/ipfs/go-ipfs/path"
	pin "github.com/ipfs/go-ipfs/pin"
	dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
	ft "github.com/ipfs/go-ipfs/unixfs"

	ci "gx/ipfs/QmPGxZ1DP2w45WcogpW1h43BvseXbfke9N91qotpoQcUeS/go-libp2p-crypto"
	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
	routing "gx/ipfs/QmUc6twRJRE9MNrUGd8eo9WjHHxebGppdZfptGCASkR7fF/go-libp2p-routing"
	peer "gx/ipfs/QmWUswjn261LSyVxWAEpMVtPdy8zmKBJJfBpG3Qdpa8ZsE/go-libp2p-peer"
	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
	record "gx/ipfs/QmcTnycWsBgvNYFYgWdWi8SRDCeevG8HBUQHkvg4KLXUsW/go-libp2p-record"
	dhtpb "gx/ipfs/QmcTnycWsBgvNYFYgWdWi8SRDCeevG8HBUQHkvg4KLXUsW/go-libp2p-record/pb"
)

// ErrExpiredRecord should be returned when an ipns record is
// invalid due to being too old
var ErrExpiredRecord = errors.New("expired record")

// ErrUnrecognizedValidity is returned when an IpnsRecord has an
// unknown validity type.
var ErrUnrecognizedValidity = errors.New("unrecognized validity type")

const PublishPutValTimeout = time.Minute
const DefaultRecordTTL = time.Hour * 24 * 7

var DefaultPublishLifetime = time.Hour * 24 * 7

// ipnsPublisher is capable of publishing and resolving names to the IPFS
// routing system.
type ipnsPublisher struct {
	routing routing.ValueStore
	ds      ds.Datastore
}

// NewRoutingPublisher constructs a publisher for the IPFS Routing name system.
func NewRoutingPublisher(route routing.ValueStore, ds ds.Datastore) *ipnsPublisher {
	if ds == nil {
		panic("nil datastore")
	}
	return &ipnsPublisher{routing: route, ds: ds}
}

// Publish implements Publisher. Accepts a keypair and a value,
// and publishes it out to the routing system
func (p *ipnsPublisher) Publish(ctx context.Context, k ci.PrivKey, value path.Path) error {
	log.Debugf("Publish %s", value)
	return p.PublishWithEOL(ctx, k, value, time.Now().Add(DefaultPublishLifetime))
}

// PublishWithEOL is a temporary stand in for the ipns records implementation
// see here for more details: https://github.com/ipfs/specs/tree/master/records
func (p *ipnsPublisher) PublishWithEOL(ctx context.Context, k ci.PrivKey, value path.Path, eol time.Time) error {

	id, err := peer.IDFromPrivateKey(k)
	if err != nil {
		return err
	}

	_, ipnskey := IpnsKeysForID(id)

	// get previous records sequence number
	seqnum, err := p.getPreviousSeqNo(ctx, ipnskey)
	if err != nil {
		return err
	}

	// increment it
	seqnum++

	return PutRecordToRouting(ctx, k, value, seqnum, eol, p.routing, id)
}

func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey string) (uint64, error) {
	prevrec, err := p.ds.Get(dshelp.NewKeyFromBinary([]byte(ipnskey)))
	if err != nil && err != ds.ErrNotFound {
		// None found, lets start at zero!
		return 0, err
	}
	var val []byte
	if err == nil {
		prbytes, ok := prevrec.([]byte)
		if !ok {
			return 0, fmt.Errorf("unexpected type returned from datastore: %#v", prevrec)
		}
		dhtrec := new(dhtpb.Record)
		err := proto.Unmarshal(prbytes, dhtrec)
		if err != nil {
			return 0, err
		}

		val = dhtrec.GetValue()
	} else {
		// try and check the dht for a record
		ctx, cancel := context.WithTimeout(ctx, time.Second*30)
		defer cancel()

		rv, err := p.routing.GetValue(ctx, ipnskey)
		if err != nil {
			// no such record found, start at zero!
			return 0, nil
		}

		val = rv
	}

	e := new(pb.IpnsEntry)
	err = proto.Unmarshal(val, e)
	if err != nil {
		return 0, err
	}

	return e.GetSequence(), nil
}

// setting the TTL on published records is an experimental feature.
// as such, i'm using the context to wire it through to avoid changing too
// much code along the way.
func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
	v := ctx.Value("ipns-publish-ttl")
	if v == nil {
		return 0, false
	}

	d, ok := v.(time.Duration)
	return d, ok
}

func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.ValueStore, id peer.ID) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	namekey, ipnskey := IpnsKeysForID(id)
	entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
	if err != nil {
		return err
	}

	ttl, ok := checkCtxTTL(ctx)
	if ok {
		entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
	}

	errs := make(chan error, 2)

	go func() {
		errs <- PublishEntry(ctx, r, ipnskey, entry)
	}()

	go func() {
		errs <- PublishPublicKey(ctx, r, namekey, k.GetPublic())
	}()

	err = waitOnErrChan(ctx, errs)
	if err != nil {
		return err
	}

	err = waitOnErrChan(ctx, errs)
	if err != nil {
		return err
	}

	return nil
}

func waitOnErrChan(ctx context.Context, errs chan error) error {
	select {
	case err := <-errs:
		return err
	case <-ctx.Done():
		return ctx.Err()
	}
}

func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk ci.PubKey) error {
	log.Debugf("Storing pubkey at: %s", k)
	pkbytes, err := pubk.Bytes()
	if err != nil {
		return err
	}

	// Store associated public key
	timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
	defer cancel()
	err = r.PutValue(timectx, k, pkbytes)
	if err != nil {
		return err
	}

	return nil
}

func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *pb.IpnsEntry) error {
	timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
	defer cancel()

	data, err := proto.Marshal(rec)
	if err != nil {
		return err
	}

	log.Debugf("Storing ipns entry at: %s", ipnskey)
	// Store ipns entry at "/ipns/"+b58(h(pubkey))
	if err := r.PutValue(timectx, ipnskey, data); err != nil {
		return err
	}

	return nil
}

func CreateRoutingEntryData(pk ci.PrivKey, val path.Path, seq uint64, eol time.Time) (*pb.IpnsEntry, error) {
	entry := new(pb.IpnsEntry)

	entry.Value = []byte(val)
	typ := pb.IpnsEntry_EOL
	entry.ValidityType = &typ
	entry.Sequence = proto.Uint64(seq)
	entry.Validity = []byte(u.FormatRFC3339(eol))

	sig, err := pk.Sign(ipnsEntryDataForSig(entry))
	if err != nil {
		return nil, err
	}
	entry.Signature = sig
	return entry, nil
}

func ipnsEntryDataForSig(e *pb.IpnsEntry) []byte {
	return bytes.Join([][]byte{
		e.Value,
		e.Validity,
		[]byte(fmt.Sprint(e.GetValidityType())),
	},
		[]byte{})
}

var IpnsRecordValidator = &record.ValidChecker{
	Func: ValidateIpnsRecord,
	Sign: true,
}

func IpnsSelectorFunc(k string, vals [][]byte) (int, error) {
	var recs []*pb.IpnsEntry
	for _, v := range vals {
		e := new(pb.IpnsEntry)
		err := proto.Unmarshal(v, e)
		if err == nil {
			recs = append(recs, e)
		} else {
			recs = append(recs, nil)
		}
	}

	return selectRecord(recs, vals)
}

func selectRecord(recs []*pb.IpnsEntry, vals [][]byte) (int, error) {
	var best_seq uint64
	best_i := -1

	for i, r := range recs {
		if r == nil || r.GetSequence() < best_seq {
			continue
		}

		if best_i == -1 || r.GetSequence() > best_seq {
			best_seq = r.GetSequence()
			best_i = i
		} else if r.GetSequence() == best_seq {
			rt, err := u.ParseRFC3339(string(r.GetValidity()))
			if err != nil {
				continue
			}

			bestt, err := u.ParseRFC3339(string(recs[best_i].GetValidity()))
			if err != nil {
				continue
			}

			if rt.After(bestt) {
				best_i = i
			} else if rt == bestt {
				if bytes.Compare(vals[i], vals[best_i]) > 0 {
					best_i = i
				}
			}
		}
	}
	if best_i == -1 {
		return 0, errors.New("no usable records in given set")
	}

	return best_i, nil
}

// ValidateIpnsRecord implements ValidatorFunc and verifies that the
// given 'val' is an IpnsEntry and that that entry is valid.
func ValidateIpnsRecord(k string, val []byte) error {
	entry := new(pb.IpnsEntry)
	err := proto.Unmarshal(val, entry)
	if err != nil {
		return err
	}
	switch entry.GetValidityType() {
	case pb.IpnsEntry_EOL:
		t, err := u.ParseRFC3339(string(entry.GetValidity()))
		if err != nil {
			log.Debug("failed parsing time for ipns record EOL")
			return err
		}
		if time.Now().After(t) {
			return ErrExpiredRecord
		}
	default:
		return ErrUnrecognizedValidity
	}
	return nil
}

// InitializeKeyspace sets the ipns record for the given key to
// point to an empty directory.
// TODO: this doesnt feel like it belongs here
func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error {
	emptyDir := ft.EmptyDirNode()
	nodek, err := ds.Add(emptyDir)
	if err != nil {
		return err
	}

	// pin recursively because this might already be pinned
	// and doing a direct pin would throw an error in that case
	err = pins.Pin(ctx, emptyDir, true)
	if err != nil {
		return err
	}

	err = pins.Flush()
	if err != nil {
		return err
	}

	err = pub.Publish(ctx, key, path.FromCid(nodek))
	if err != nil {
		return err
	}

	return nil
}

func IpnsKeysForID(id peer.ID) (name, ipns string) {
	namekey := "/pk/" + string(id)
	ipnskey := "/ipns/" + string(id)

	return namekey, ipnskey
}
<|endoftext|>


<commit_before>// A framework that allows you to create your own web robots!
package gobot

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
)

var (
	tagRe  = regexp.MustCompilePOSIX("<a[^>]*>")
	attrRe = regexp.MustCompilePOSIX("href=\"[^\"]*\"")
	urlRe  = regexp.MustCompilePOSIX("http://[^\"]*")
)

type VisitAction func(*http.Response)

type ErrorAction func(*http.Request)

type VisitDecision func(string) bool

type GoBot struct {
	http.Client
	OnVisit     VisitAction
	OnError     ErrorAction
	ShouldVisit VisitDecision
	visited     map[string]bool
}

func NewGoBot() *GoBot {
	bot := new(GoBot)
	bot.OnVisit = defaultVisitAction
	bot.OnError = defaultErrorAction
	bot.ShouldVisit = defaultVisitDecision
	bot.Jar = NewBotCookieJar()
	bot.visited = make(map[string]bool)
	return bot
}

func (bot *GoBot) StartCrawl(seed string) {
	resp, err := bot.Get(seed)
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	if err != nil {
		log.Printf("StartCrawl: ERROR\n")
	}
	urls := ExtractLinks(resp)
	for i, u := range urls {
		if i == (ncpu - 1) {
			break
		}
		go bot.Crawl(u)
	}
	bot.Crawl(seed)
}

func (bot *GoBot) Crawl(seed string) {
	queue := make([]string, 0)
	currUrl := seed

	for {
		resp, err := bot.Get(currUrl)
		for err != nil {
			if resp != nil {
				go bot.OnError(resp.Request)
			}
			if len(queue) > 0 {
				currUrl = queue[0]
				queue = queue[1:]
				resp, err = bot.Get(currUrl)
			} else {
				os.Exit(1) // TODO: Should find a better way to exit.
			}
		}

		go bot.OnVisit(resp)

		urls := ExtractLinks(resp)
		for _, url := range urls {
			_, present := bot.visited[url]
			if !present {
				queue = append(queue, url)
				bot.visited[url] = true
			}
		}

		if len(queue) > 0 {
			currUrl = queue[0]
			queue = queue[1:]
		} else {
			break
		}

	}
}

// Extracts all the links from the body of an http response.
func ExtractLinks(resp *http.Response) []string {
	urls := make([]string, 0)
	body := ResponseBodyToString(resp)
	tags := tagRe.FindAllString(body, -1)

	for _, tag := range tags {
		url := urlRe.FindString(attrRe.FindString(tag))
		if url != "" {
			urls = append(urls, url)
		}
	}
	return urls
}

// Returns a string of the body of an http response.
func ResponseBodyToString(resp *http.Response) string {
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}
	return string(body)
}

func defaultVisitAction(resp *http.Response) {
	request := resp.Request
	log.Printf("%s\n", request.URL.String())
}

func defaultErrorAction(resp *http.Request) {
}

func defaultVisitDecision(url string) bool {
	return true
}

type BotCookieJar struct {
	cookies map[string][]*http.Cookie
}

func NewBotCookieJar() *BotCookieJar {
	jar := new(BotCookieJar)
	jar.cookies = make(map[string][]*http.Cookie)
	return jar
}

func (jar *BotCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	jar.cookies[u.Host] = cookies
}

func (jar *BotCookieJar) Cookies(u *url.URL) []*http.Cookie {
	return jar.cookies[u.Host]
}
<commit_msg>Add callback function that is called when a link is found.<commit_after>// A framework that allows you to create your own web robots!
package gobot

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"runtime"
)

var (
	tagRe  = regexp.MustCompilePOSIX("<a[^>]*>")
	attrRe = regexp.MustCompilePOSIX("href=\"[^\"]*\"")
	urlRe  = regexp.MustCompilePOSIX("http://[^\"]*")
)

type VisitAction func(*http.Response)

type FoundAction func(string)

type ErrorAction func(*http.Request)

type VisitDecision func(string) bool

type GoBot struct {
	http.Client
	OnVisit     VisitAction
    OnFind      FoundAction
	OnError     ErrorAction
	ShouldVisit VisitDecision
	visited     map[string]bool
}

func NewGoBot() *GoBot {
	bot := new(GoBot)
	bot.OnVisit = defaultVisitAction
    bot.OnFind = defaultFoundAction
	bot.OnError = defaultErrorAction
	bot.ShouldVisit = defaultVisitDecision
	bot.Jar = NewBotCookieJar()
	bot.visited = make(map[string]bool)
	return bot
}

func (bot *GoBot) StartCrawl(seed string) {
	resp, err := bot.Get(seed)
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	if err != nil {
		log.Printf("StartCrawl: ERROR\n")
	}
	urls := ExtractLinks(resp)
	for i, u := range urls {
		if i == (ncpu - 1) {
			break
		}
		go bot.Crawl(u)
	}
	bot.Crawl(seed)
}

func (bot *GoBot) Crawl(seed string) {
	queue := make([]string, 0)
	currUrl := seed

	for {
		resp, err := bot.Get(currUrl)
		for err != nil {
			if resp != nil {
				go bot.OnError(resp.Request)
			}
			if len(queue) > 0 {
				currUrl = queue[0]
				queue = queue[1:]
				resp, err = bot.Get(currUrl)
			} else {
				os.Exit(1) // TODO: Should find a better way to exit.
			}
		}

		go bot.OnVisit(resp)

		urls := ExtractLinks(resp)
		for _, url := range urls {
			_, present := bot.visited[url]
			if !present {
                bot.OnFind(url)
				queue = append(queue, url)
				bot.visited[url] = true
			}
		}

		if len(queue) > 0 {
			currUrl = queue[0]
			queue = queue[1:]
		} else {
			break
		}

	}
}

// Extracts all the links from the body of an http response.
func ExtractLinks(resp *http.Response) []string {
	urls := make([]string, 0)
	body := ResponseBodyToString(resp)
	tags := tagRe.FindAllString(body, -1)

	for _, tag := range tags {
		url := urlRe.FindString(attrRe.FindString(tag))
		if url != "" {
			urls = append(urls, url)
		}
	}
	return urls
}

// Returns a string of the body of an http response.
func ResponseBodyToString(resp *http.Response) string {
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return ""
	}
	return string(body)
}

func defaultVisitAction(resp *http.Response) {
	//request := resp.Request
	//log.Printf("%s\n", request.URL.String())
}

func defaultFoundAction(u string) {
    log.Printf("%s\n", u)
}

func defaultErrorAction(resp *http.Request) {
}

func defaultVisitDecision(url string) bool {
	return true
}

type BotCookieJar struct {
	cookies map[string][]*http.Cookie
}

func NewBotCookieJar() *BotCookieJar {
	jar := new(BotCookieJar)
	jar.cookies = make(map[string][]*http.Cookie)
	return jar
}

func (jar *BotCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
	jar.cookies[u.Host] = cookies
}

func (jar *BotCookieJar) Cookies(u *url.URL) []*http.Cookie {
	return jar.cookies[u.Host]
}
<|endoftext|>


<commit_before>// Package weekly provides functionality for handling events that are periodic
// over the course of a week.
package weekly

import (
	"errors"
	"fmt"
	"time"
)

// A Ticker holds a channel that delivers ticks of a clock at intervals.
// It starts & stops ticking at the same time each week.
type Ticker struct {
	C    <-chan time.Time
	done chan struct{}
}

// Stop closes the ticker and releases any resources it has acquired.
func (t *Ticker) Stop() {
	close(t.done)
}

// TickSpecification is used with NewTicker. It specifies a period each week
// when ticks occur, and how frequently ticks occur during that period.
type TickSpecification struct {
	Start, End Time          // when to start and stop ticking each week
	Frequency  time.Duration // how often to tick while ticking
}

// NewTicker returns a ticker that starts and stops ticking at the same time each week.
func NewTicker(tickSpecs []TickSpecification) (*Ticker, error) {
	for _, ts := range tickSpecs {
		if ts.End.Before(ts.Start) {
			return nil, errors.New("end is before start")
		}
		if ts.Frequency <= 0 {
			return nil, errors.New("freq is nonpositive")
		}
	}

	ch := make(chan time.Time)
	done := make(chan struct{})
	for _, ts := range tickSpecs {
		go tick(ch, done, ts.Start, ts.End, ts.Frequency)
	}
	return &Ticker{
		C:    ch,
		done: done,
	}, nil
}

func tick(ch chan<- time.Time, done chan struct{}, start, end Time, freq time.Duration) {
	tck := time.Now()
	for {
		// Go to sleep until we reach the next tick.
		nxt := nextTick(tck, start, end, freq)
		tmr := time.NewTimer(time.Until(nxt))
		select {
		case <-tmr.C:
			// Send the tick from the  in case it was delayed.
			// Drop the tick if it is not ready to be received.
			select {
			case ch <- nxt:
			default:
			}

		case <-done:
			if !tmr.Stop() {
				<-tmr.C
			}
			return
		}
		tck = nxt
	}
}

func nextTick(tck time.Time, start, end Time, freq time.Duration) (nxt time.Time) {
	s, e := start.InWeek(tck), end.InWeek(tck)
	switch {
	case tck.Before(s):
		// We haven't started ticking yet this week.
		return s

	case tck.Before(e):
		// We are currently ticking. Figure out the next tick from when we are.
		nxt := s.Add(freq * (1 + (tck.Sub(s) / freq)))
		if nxt.Before(e) {
			return nxt
		}
		// The next tick is after the end of the ticking interval. We're done ticking this week.
		fallthrough

	default:
		// We are done ticking this week. Wait until we start ticking next week.
		return s.AddDate(0, 0, 7)
	}
}

// Time represents a specific time during a week; weeks start on Sunday and go
// through the following Saturday. A weekly.Time value represents an instant in
// time in every week, and may be converted to a specific instant in a specific
// week.
type Time struct {
	day       time.Weekday
	hour, min int
}

// Parse parses a string value into a time during the week. The expected format
// is like: "Thu 7:30PM". The local location is used.
func Parse(val string) (Time, error) {
	if len(val) < 4 {
		return Time{}, errors.New("bad weekday")
	}
	day, ok := strToDay[val[:4]]
	if !ok {
		return Time{}, errors.New("bad weekday")
	}
	t, err := time.Parse(time.Kitchen, val[4:])
	if err != nil {
		return Time{}, fmt.Errorf("bad time: %v", err)
	}
	return Time{day: day, hour: t.Hour(), min: t.Minute()}, nil
}

func MustParse(val string) Time {
	t, err := Parse(val)
	if err != nil {
		panic(fmt.Sprintf("Parse(%q): %v", val, err))
	}
	return t
}

// InWeek converts a given weekly.Time to a time.Time in the same week as the
// given time.Time.
func (wt Time) InWeek(tt time.Time) time.Time {
	return time.Date(tt.Year(), tt.Month(), tt.Day()+int(wt.day)-int(tt.Weekday()), wt.hour, wt.min, 0, 0, tt.Location())
}

func (wt Time) Before(owt Time) bool {
	return wt.day < owt.day ||
		(wt.day == owt.day && wt.hour < owt.hour) ||
		(wt.day == owt.day && wt.hour == owt.hour && wt.min < owt.min)
}

func (wt Time) String() string {
	ampm := "AM"
	if wt.hour >= 12 {
		ampm = "PM"
	}
	mhr := wt.hour % 12
	if mhr == 0 {
		mhr = 12
	}
	return fmt.Sprintf("%s %d:%02d%s", dayToStr[wt.day], mhr, wt.min, ampm)
}

var (
	// Used by Parse.
	strToDay = map[string]time.Weekday{
		"Sun ": time.Sunday,
		"Mon ": time.Monday,
		"Tue ": time.Tuesday,
		"Wed ": time.Wednesday,
		"Thu ": time.Thursday,
		"Fri ": time.Friday,
		"Sat ": time.Saturday,
	}

	// Used by String.
	dayToStr = map[time.Weekday]string{
		time.Sunday:    "Sun",
		time.Monday:    "Mon",
		time.Tuesday:   "Tue",
		time.Wednesday: "Wed",
		time.Thursday:  "Thu",
		time.Friday:    "Fri",
		time.Saturday:  "Sat",
	}
)
<commit_msg>Add randomness to tick times.<commit_after>// Package weekly provides functionality for handling events that are periodic
// over the course of a week.
package weekly

import (
	crand "crypto/rand"
	"encoding/binary"
	"errors"
	"fmt"
	"math/rand"
	"time"
)

// A Ticker holds a channel that delivers ticks of a clock at intervals.
// It starts & stops ticking at the same time each week.
type Ticker struct {
	C    <-chan time.Time
	done chan struct{}
}

// Stop closes the ticker and releases any resources it has acquired.
func (t *Ticker) Stop() {
	close(t.done)
}

// TickSpecification is used with NewTicker. It specifies a period each week
// when ticks occur, and how frequently ticks occur during that period.
type TickSpecification struct {
	Start, End Time          // when to start and stop ticking each week
	Frequency  time.Duration // how often to tick while ticking
}

// NewTicker returns a ticker that starts and stops ticking at the same time each week.
func NewTicker(tickSpecs []TickSpecification) (*Ticker, error) {
	for _, ts := range tickSpecs {
		if ts.End.Before(ts.Start) {
			return nil, errors.New("end is before start")
		}
		if ts.Frequency <= 0 {
			return nil, errors.New("freq is nonpositive")
		}
	}

	var buf [8]byte
	if _, err := crand.Read(buf[:]); err != nil {
		return nil, fmt.Errorf("could not seed RNG: %v", err)
	}
	seed := int64(binary.LittleEndian.Uint64(buf[:]))
	rnd := rand.New(rand.NewSource(seed))

	ch := make(chan time.Time)
	done := make(chan struct{})
	for _, ts := range tickSpecs {
		go tick(ch, rnd, done, ts.Start, ts.End, ts.Frequency)
	}
	return &Ticker{
		C:    ch,
		done: done,
	}, nil
}

func tick(ch chan<- time.Time, rnd *rand.Rand, done chan struct{}, start, end Time, freq time.Duration) {
	tck := time.Now()
	for {
		// Go to sleep until we reach the next tick.
		nxt := nextTick(tck, start, end, freq)
		fuzzyNxt := nxt.Add(time.Duration(rnd.Float64() * float64(freq)))
		tmr := time.NewTimer(time.Until(fuzzyNxt))
		select {
		case t := <-tmr.C:
			// Drop the tick if it is not ready to be received.
			select {
			case ch <- t:
			default:
			}

		case <-done:
			if !tmr.Stop() {
				<-tmr.C
			}
			return
		}
		tck = nxt
	}
}

func nextTick(tck time.Time, start, end Time, freq time.Duration) (nxt time.Time) {
	s, e := start.InWeek(tck), end.InWeek(tck)
	switch {
	case tck.Before(s):
		// We haven't started ticking yet this week.
		return s

	case tck.Before(e):
		// We are currently ticking. Figure out the next tick from when we are.
		nxt := s.Add(freq * (1 + (tck.Sub(s) / freq)))
		if nxt.Before(e) {
			return nxt
		}
		// The next tick is after the end of the ticking interval. We're done ticking this week.
		fallthrough

	default:
		// We are done ticking this week. Wait until we start ticking next week.
		return s.AddDate(0, 0, 7)
	}
}

// Time represents a specific time during a week; weeks start on Sunday and go
// through the following Saturday. A weekly.Time value represents an instant in
// time in every week, and may be converted to a specific instant in a specific
// week.
type Time struct {
	day       time.Weekday
	hour, min int
}

// Parse parses a string value into a time during the week. The expected format
// is like: "Thu 7:30PM". The local location is used.
func Parse(val string) (Time, error) {
	if len(val) < 4 {
		return Time{}, errors.New("bad weekday")
	}
	day, ok := strToDay[val[:4]]
	if !ok {
		return Time{}, errors.New("bad weekday")
	}
	t, err := time.Parse(time.Kitchen, val[4:])
	if err != nil {
		return Time{}, fmt.Errorf("bad time: %v", err)
	}
	return Time{day: day, hour: t.Hour(), min: t.Minute()}, nil
}

func MustParse(val string) Time {
	t, err := Parse(val)
	if err != nil {
		panic(fmt.Sprintf("Parse(%q): %v", val, err))
	}
	return t
}

// InWeek converts a given weekly.Time to a time.Time in the same week as the
// given time.Time.
func (wt Time) InWeek(tt time.Time) time.Time {
	return time.Date(tt.Year(), tt.Month(), tt.Day()+int(wt.day)-int(tt.Weekday()), wt.hour, wt.min, 0, 0, tt.Location())
}

func (wt Time) Before(owt Time) bool {
	return wt.day < owt.day ||
		(wt.day == owt.day && wt.hour < owt.hour) ||
		(wt.day == owt.day && wt.hour == owt.hour && wt.min < owt.min)
}

func (wt Time) String() string {
	ampm := "AM"
	if wt.hour >= 12 {
		ampm = "PM"
	}
	mhr := wt.hour % 12
	if mhr == 0 {
		mhr = 12
	}
	return fmt.Sprintf("%s %d:%02d%s", dayToStr[wt.day], mhr, wt.min, ampm)
}

var (
	// Used by Parse.
	strToDay = map[string]time.Weekday{
		"Sun ": time.Sunday,
		"Mon ": time.Monday,
		"Tue ": time.Tuesday,
		"Wed ": time.Wednesday,
		"Thu ": time.Thursday,
		"Fri ": time.Friday,
		"Sat ": time.Saturday,
	}

	// Used by String.
	dayToStr = map[time.Weekday]string{
		time.Sunday:    "Sun",
		time.Monday:    "Mon",
		time.Tuesday:   "Tue",
		time.Wednesday: "Wed",
		time.Thursday:  "Thu",
		time.Friday:    "Fri",
		time.Saturday:  "Sat",
	}
)
<|endoftext|>


<commit_before>package fs

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/dotcloud/docker/pkg/cgroups"
	"github.com/dotcloud/docker/pkg/system"
)

var (
	cpuCount   = float64(runtime.NumCPU())
	clockTicks = float64(system.GetClockTicks())
)

type cpuacctGroup struct {
}

func (s *cpuacctGroup) Set(d *data) error {
	// we just want to join this group even though we don't set anything
	if _, err := d.join("cpuacct"); err != nil && err != cgroups.ErrNotFound {
		return err
	}
	return nil
}

func (s *cpuacctGroup) Remove(d *data) error {
	return removePath(d.path("cpuacct"))
}

func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) {
	var (
		startCpu, lastCpu, startSystem, lastSystem float64
		percentage                                 float64
		paramData                                  = make(map[string]float64)
	)
	path, err := d.path("cpuacct")
	if startCpu, err = s.getCpuUsage(d, path); err != nil {
		return nil, err
	}
	if startSystem, err = s.getSystemCpuUsage(d); err != nil {
		return nil, err
	}
	// sample for 100ms
	time.Sleep(100 * time.Millisecond)
	if lastCpu, err = s.getCpuUsage(d, path); err != nil {
		return nil, err
	}
	if lastSystem, err = s.getSystemCpuUsage(d); err != nil {
		return nil, err
	}

	var (
		deltaProc   = lastCpu - startCpu
		deltaSystem = lastSystem - startSystem
	)
	if deltaSystem > 0.0 {
		percentage = ((deltaProc / deltaSystem) * clockTicks) * cpuCount
	}
	// NOTE: a percentage over 100% is valid for POSIX because that means the
	// processes is using multiple cores
	paramData["percentage"] = percentage
	return paramData, nil
}

func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) {
	rawStart, err := system.GetProcessStartTime(d.pid)
	if err != nil {
		return 0, err
	}
	return strconv.ParseFloat(rawStart, 64)
}

func (s *cpuacctGroup) getSystemCpuUsage(d *data) (float64, error) {

	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, err
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		parts := strings.Fields(sc.Text())
		switch parts[0] {
		case "cpu":
			if len(parts) < 8 {
				return 0, fmt.Errorf("invalid number of cpu fields")
			}

			var total float64
			for _, i := range parts[1:8] {
				v, err := strconv.ParseFloat(i, 64)
				if err != nil {
					return 0.0, fmt.Errorf("Unable to convert value %s to float: %s", i, err)
				}
				total += v
			}
			return total, nil
		default:
			continue
		}
	}
	return 0, fmt.Errorf("invalid stat format")
}

func (s *cpuacctGroup) getCpuUsage(d *data, path string) (float64, error) {
	cpuTotal := 0.0
	f, err := os.Open(filepath.Join(path, "cpuacct.stat"))
	if err != nil {
		return 0.0, err
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		_, v, err := getCgroupParamKeyValue(sc.Text())
		if err != nil {
			return 0.0, err
		}
		// set the raw data in map
		cpuTotal += v
	}
	return cpuTotal, nil
}
<commit_msg>Export cpuacct CPU usage in total cores over the sampled period.<commit_after>package fs

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/dotcloud/docker/pkg/cgroups"
	"github.com/dotcloud/docker/pkg/system"
)

var (
	cpuCount   = float64(runtime.NumCPU())
	clockTicks = float64(system.GetClockTicks())
)

type cpuacctGroup struct {
}

func (s *cpuacctGroup) Set(d *data) error {
	// we just want to join this group even though we don't set anything
	if _, err := d.join("cpuacct"); err != nil && err != cgroups.ErrNotFound {
		return err
	}
	return nil
}

func (s *cpuacctGroup) Remove(d *data) error {
	return removePath(d.path("cpuacct"))
}

func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) {
	var (
		startCpu, lastCpu, startSystem, lastSystem, startUsage, lastUsage float64
		percentage                                                        float64
		paramData                                                         = make(map[string]float64)
	)
	path, err := d.path("cpuacct")
	if startCpu, err = s.getCpuUsage(d, path); err != nil {
		return nil, err
	}
	if startSystem, err = s.getSystemCpuUsage(d); err != nil {
		return nil, err
	}
	startUsageTime := time.Now()
	if startUsage, err = getCgroupParamFloat64(path, "cpuacct.usage"); err != nil {
		return nil, err
	}
	// sample for 100ms
	time.Sleep(100 * time.Millisecond)
	if lastCpu, err = s.getCpuUsage(d, path); err != nil {
		return nil, err
	}
	if lastSystem, err = s.getSystemCpuUsage(d); err != nil {
		return nil, err
	}
	usageSampleDuration := time.Since(startUsageTime)
	if lastUsage, err = getCgroupParamFloat64(path, "cpuacct.usage"); err != nil {
		return nil, err
	}

	var (
		deltaProc   = lastCpu - startCpu
		deltaSystem = lastSystem - startSystem
		deltaUsage  = lastUsage - startUsage
	)
	if deltaSystem > 0.0 {
		percentage = ((deltaProc / deltaSystem) * clockTicks) * cpuCount
	}
	// NOTE: a percentage over 100% is valid for POSIX because that means the
	// processes is using multiple cores
	paramData["percentage"] = percentage

	// Delta usage is in nanoseconds of CPU time so get the usage (in cores) over the sample time.
	paramData["usage"] = deltaUsage / float64(usageSampleDuration.Nanoseconds())
	return paramData, nil
}

func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) {
	rawStart, err := system.GetProcessStartTime(d.pid)
	if err != nil {
		return 0, err
	}
	return strconv.ParseFloat(rawStart, 64)
}

func (s *cpuacctGroup) getSystemCpuUsage(d *data) (float64, error) {

	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, err
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		parts := strings.Fields(sc.Text())
		switch parts[0] {
		case "cpu":
			if len(parts) < 8 {
				return 0, fmt.Errorf("invalid number of cpu fields")
			}

			var total float64
			for _, i := range parts[1:8] {
				v, err := strconv.ParseFloat(i, 64)
				if err != nil {
					return 0.0, fmt.Errorf("Unable to convert value %s to float: %s", i, err)
				}
				total += v
			}
			return total, nil
		default:
			continue
		}
	}
	return 0, fmt.Errorf("invalid stat format")
}

func (s *cpuacctGroup) getCpuUsage(d *data, path string) (float64, error) {
	cpuTotal := 0.0
	f, err := os.Open(filepath.Join(path, "cpuacct.stat"))
	if err != nil {
		return 0.0, err
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		_, v, err := getCgroupParamKeyValue(sc.Text())
		if err != nil {
			return 0.0, err
		}
		// set the raw data in map
		cpuTotal += v
	}
	return cpuTotal, nil
}
<|endoftext|>


<commit_before>package zego

import (
	"encoding/json"
)

type Comments struct {
	Id          int           `json:"id"`
	Type        string        `json:"type"`
	Body        string        `json:"body"`
	Public      bool          `json:"public"`
	CreatedAt   string        `json:"created_at"`
	AuthorId    int           `json:"author_id"`
	Attachments []Attachments `json:"attachments"`
	Metadata    interface{}   `json:"metadata"`
	Via         interface{}   `json:"via"`
}

type Attachments struct {
	Id          int           `json:"id"`
	Name        string        `json:"name"`
	ContentURL  string        `json:"content_url"`
	ContentType string        `json:"content_type"`
	Size        int           `json:"size"`
	Thumbnails  []interface{} `json:"thumbnails"`
}
<commit_msg>Added comment array<commit_after>package zego

import (
	"encoding/json"
)

type CommentArray struct {
	Comments []Comments `json:"comments"`
}

type Comments struct {
	Id          int           `json:"id"`
	Type        string        `json:"type"`
	Body        string        `json:"body"`
	Public      bool          `json:"public"`
	CreatedAt   string        `json:"created_at"`
	AuthorId    int           `json:"author_id"`
	Attachments []Attachments `json:"attachments"`
	Metadata    interface{}   `json:"metadata"`
	Via         interface{}   `json:"via"`
}

type Attachments struct {
	Id          int           `json:"id"`
	Name        string        `json:"name"`
	ContentURL  string        `json:"content_url"`
	ContentType string        `json:"content_type"`
	Size        int           `json:"size"`
	Thumbnails  []interface{} `json:"thumbnails"`
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

var (
	plugin_long = templates.LongDesc(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.`)

	plugin_list_long = templates.LongDesc(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`)
)

func NewCmdPlugin(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use: "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short: i18n.T("Provides utilities for interacting with plugins."),
		Long:  plugin_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(f, streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:   "list",
		Short: "list all visible plugin executables on a user's PATH",
		Long:  plugin_list_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string, 0),
	}
	return nil
}

func (o *PluginListOptions) Run() error {
	path := "PATH"
	if runtime.GOOS == "windows" {
		path = "path"
	}

	pluginsFound := false
	isFirstFile := true
	pluginWarnings := 0
	for _, dir := range filepath.SplitList(os.Getenv(path)) {
		files, err := ioutil.ReadDir(dir)
		if err != nil {
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !strings.HasPrefix(f.Name(), "kubectl-") {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.ErrOut, "The following kubectl-compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		return fmt.Errorf("error: unable to find any kubectl plugins in your PATH")
	}

	if pluginWarnings > 0 {
		fmt.Fprintln(o.ErrOut)
		if pluginWarnings == 1 {
			return fmt.Errorf("one plugin warning was found")
		}
		return fmt.Errorf("%v plugin warnings were found", pluginWarnings)
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		if strings.HasSuffix(info.Name(), ".exe") {
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}
<commit_msg>Deduplicate list of directories<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

var (
	plugin_long = templates.LongDesc(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.`)

	plugin_list_long = templates.LongDesc(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`)
)

func NewCmdPlugin(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use: "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short: i18n.T("Provides utilities for interacting with plugins."),
		Long:  plugin_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(f, streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:   "list",
		Short: "list all visible plugin executables on a user's PATH",
		Long:  plugin_list_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string, 0),
	}
	return nil
}

func (o *PluginListOptions) Run() error {
	path := "PATH"
	if runtime.GOOS == "windows" {
		path = "path"
	}

	pluginsFound := false
	isFirstFile := true
	pluginWarnings := 0
	paths := sets.NewString(filepath.SplitList(os.Getenv(path))...)
	for _, dir := range paths.List() {
		files, err := ioutil.ReadDir(dir)
		if err != nil {
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !strings.HasPrefix(f.Name(), "kubectl-") {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.ErrOut, "The following kubectl-compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		return fmt.Errorf("error: unable to find any kubectl plugins in your PATH")
	}

	if pluginWarnings > 0 {
		fmt.Fprintln(o.ErrOut)
		if pluginWarnings == 1 {
			return fmt.Errorf("one plugin warning was found")
		}
		return fmt.Errorf("%v plugin warnings were found", pluginWarnings)
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		if strings.HasSuffix(info.Name(), ".exe") {
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// execprog executes a single program or a set of programs
// and optinally prints information about execution.
package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/google/syzkaller/cover"
	"github.com/google/syzkaller/ipc"
	"github.com/google/syzkaller/prog"
)

var (
	flagExecutor  = flag.String("executor", "./syz-executor", "path to executor binary")
	flagCoverFile = flag.String("coverfile", "", "write coverage to the file")
	flagRepeat    = flag.Int("repeat", 1, "repeat execution that many times (0 for infinite loop)")
	flagProcs     = flag.Int("procs", 1, "number of parallel processes to execute programs")
)

func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		fmt.Fprintf(os.Stderr, "usage: execprog [flags] file-with-programs+\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var progs []*prog.Prog
	for _, fn := range flag.Args() {
		data, err := ioutil.ReadFile(fn)
		if err != nil {
			log.Fatalf("failed to read log file: %v", err)
		}
		entries := prog.ParseLog(data)
		for _, ent := range entries {
			progs = append(progs, ent.P)
		}
	}
	log.Printf("parsed %v programs", len(progs))
	if len(progs) == 0 {
		return
	}

	flags, timeout, err := ipc.DefaultFlags()
	if err != nil {
		log.Fatalf("%v", err)
	}
	if *flagCoverFile != "" {
		flags |= ipc.FlagCover
		flags &= ^ipc.FlagDedupCover
	}

	var wg sync.WaitGroup
	wg.Add(*flagProcs)
	var posMu sync.Mutex
	var pos int
	var lastPrint time.Time
	var shutdown uint32
	for p := 0; p < *flagProcs; p++ {
		go func() {
			defer wg.Done()
			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags)
			if err != nil {
				log.Fatalf("failed to create ipc env: %v", err)
			}
			defer env.Close()
			for {
				posMu.Lock()
				idx := pos
				pos++
				if idx%len(progs) == 0 && time.Since(lastPrint) > 5*time.Second {
					log.Printf("executed programs: %v", idx)
					lastPrint = time.Now()
				}
				posMu.Unlock()
				if *flagRepeat > 0 && idx >= len(progs)**flagRepeat {
					return
				}
				p := progs[idx%len(progs)]
				output, cov, _, failed, hanged, err := env.Exec(p)
				if atomic.LoadUint32(&shutdown) != 0 {
					return
				}
				if failed {
					fmt.Printf("BUG: executor-detected bug:\n%s", output)
				}
				if flags&ipc.FlagDebug != 0 || err != nil {
					fmt.Printf("result: failed=%v hanged=%v err=%v\n\n%s", failed, hanged, err, output)
				}
				if *flagCoverFile != "" {
					// Coverage is dumped in sanitizer format.
					// github.com/google/sanitizers/tools/sancov command can be used to dump PCs,
					// then they can be piped via addr2line to symbolize.
					for i, c := range cov {
						fmt.Printf("call #%v: coverage %v\n", i, len(c))
						if len(c) == 0 {
							continue
						}
						buf := new(bytes.Buffer)
						binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64))
						for _, pc := range c {
							binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc, 0xffffffff))
						}
						err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCoverFile, i), buf.Bytes(), 0660)
						if err != nil {
							log.Fatalf("failed to write coverage file: %v", err)
						}
					}
				}
			}
		}()
	}

	go func() {
		c := make(chan os.Signal, 2)
		signal.Notify(c, syscall.SIGINT)
		<-c
		log.Printf("shutting down...")
		atomic.StoreUint32(&shutdown, 1)
		<-c
		log.Fatalf("terminating")
	}()

	wg.Wait()
}
<commit_msg>execprog: make syz-execprog work more like syz-fuzzer<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// execprog executes a single program or a set of programs
// and optinally prints information about execution.
package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/google/syzkaller/cover"
	"github.com/google/syzkaller/ipc"
	"github.com/google/syzkaller/prog"
)

var (
	flagExecutor  = flag.String("executor", "./syz-executor", "path to executor binary")
	flagCoverFile = flag.String("coverfile", "", "write coverage to the file")
	flagRepeat    = flag.Int("repeat", 1, "repeat execution that many times (0 for infinite loop)")
	flagProcs     = flag.Int("procs", 1, "number of parallel processes to execute programs")
	flagOutput    = flag.String("output", "none", "write programs to none/stdout")
)

func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		fmt.Fprintf(os.Stderr, "usage: execprog [flags] file-with-programs+\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var progs []*prog.Prog
	for _, fn := range flag.Args() {
		data, err := ioutil.ReadFile(fn)
		if err != nil {
			log.Fatalf("failed to read log file: %v", err)
		}
		entries := prog.ParseLog(data)
		for _, ent := range entries {
			progs = append(progs, ent.P)
		}
	}
	log.Printf("parsed %v programs", len(progs))
	if len(progs) == 0 {
		return
	}

	flags, timeout, err := ipc.DefaultFlags()
	if err != nil {
		log.Fatalf("%v", err)
	}
	if *flagCoverFile != "" {
		flags |= ipc.FlagCover
		flags &= ^ipc.FlagDedupCover
	}

	var wg sync.WaitGroup
	wg.Add(*flagProcs)
	var posMu, logMu sync.Mutex
	gate := ipc.NewGate(2**flagProcs, nil)
	var pos int
	var lastPrint time.Time
	var shutdown uint32
	for p := 0; p < *flagProcs; p++ {
		pid := p
		go func() {
			defer wg.Done()
			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags)
			if err != nil {
				log.Fatalf("failed to create ipc env: %v", err)
			}
			defer env.Close()
			for {
				if !func() bool {
					// Limit concurrency window.
					ticket := gate.Enter()
					defer gate.Leave(ticket)

					posMu.Lock()
					idx := pos
					pos++
					if /*idx%len(progs) == 0 && time.Since(lastPrint) > 5*time.Second*/ (idx % 10) == 0 {
						log.Printf("executed programs: %v", idx)
						lastPrint = time.Now()
					}
					posMu.Unlock()
					if *flagRepeat > 0 && idx >= len(progs)**flagRepeat {
						return false
					}
					p := progs[idx%len(progs)]
					switch *flagOutput {
					case "stdout":
						data := p.Serialize()
						logMu.Lock()
						log.Printf("executing program %v:\n%s", pid, data)
						logMu.Unlock()
					}
					output, cov, _, failed, hanged, err := env.Exec(p)
					if atomic.LoadUint32(&shutdown) != 0 {
						return false
					}
					if failed {
						fmt.Printf("BUG: executor-detected bug:\n%s", output)
					}
					if flags&ipc.FlagDebug != 0 || err != nil {
						fmt.Printf("result: failed=%v hanged=%v err=%v\n\n%s", failed, hanged, err, output)
					}
					if *flagCoverFile != "" {
						// Coverage is dumped in sanitizer format.
						// github.com/google/sanitizers/tools/sancov command can be used to dump PCs,
						// then they can be piped via addr2line to symbolize.
						for i, c := range cov {
							fmt.Printf("call #%v: coverage %v\n", i, len(c))
							if len(c) == 0 {
								continue
							}
							buf := new(bytes.Buffer)
							binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64))
							for _, pc := range c {
								binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc, 0xffffffff))
							}
							err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCoverFile, i), buf.Bytes(), 0660)
							if err != nil {
								log.Fatalf("failed to write coverage file: %v", err)
							}
						}
					}
					return true
				}() {
					return
				}
			}
		}()
	}

	go func() {
		c := make(chan os.Signal, 2)
		signal.Notify(c, syscall.SIGINT)
		<-c
		log.Printf("shutting down...")
		atomic.StoreUint32(&shutdown, 1)
		<-c
		log.Fatalf("terminating")
	}()

	wg.Wait()
}
<|endoftext|>


<commit_before>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2018] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package events

import (
	"context"
	"github.com/lastbackend/lastbackend/pkg/api/types/v1"
	"github.com/lastbackend/lastbackend/pkg/api/types/v1/request"
	"github.com/lastbackend/lastbackend/pkg/distribution/types"
	"github.com/lastbackend/lastbackend/pkg/log"
	"github.com/lastbackend/lastbackend/pkg/node/envs"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"io/ioutil"
)

const (
	logLevel  = 3
	logPrefix = "node:events"
)

// NewConnectEventt - send node info event after
// node is successful accepted and each hour
func NewConnectEvent(ctx context.Context) error {

	var (
		c = envs.Get().GetNodeClient()
	)

	opts := v1.Request().Node().NodeConnectOptions()
	opts.Info = envs.Get().GetState().Node().Info
	opts.Status = envs.Get().GetState().Node().Status
	opts.Network = *envs.Get().GetCNI().Info(ctx)

	if viper.IsSet("node.tls") {
		opts.TLS = !viper.GetBool("node.tls.insecure")

		if opts.TLS {
			caData, err := ioutil.ReadFile(viper.GetString("node.tls.ca"))
			if err != nil {
				log.Errorf("%s:connect_event:> read ca cert file err: %v", logPrefix, err)
				return err
			}

			certData, err := ioutil.ReadFile(viper.GetString("node.tls.client_cert"))
			if err != nil {
				log.Errorf("%s:connect_event:> read client cert file err: %v", logPrefix, err)
				return err
			}

			keyData, err := ioutil.ReadFile(viper.GetString("node.tls.client_key"))
			if err != nil {
				log.Errorf("%s:connect_event:> read client key file err: %v", logPrefix, err)
				return err
			}

			opts.SSL = new(request.SSL)
			opts.SSL.CA = caData
			opts.SSL.Key = keyData
			opts.SSL.Cert = certData
		}
	}

	opts.Status.Mode.Ingress = envs.Get().GetModeIngress()
	return c.Connect(ctx, opts)

}

// NewStatusEvent - send node state event after
// node is successful accepted and each hour
func NewStatusEvent(ctx context.Context) error {
	var (
		e = envs.Get().GetExporter()
	)

	e.Resources(envs.Get().GetState().Node().Status)
	return nil
}

// NewPodStatusEvent - send pod state event after
// node is successful accepted and each hour
func NewPodStatusEvent(ctx context.Context, pod string) error {

	var (
		p = envs.Get().GetState().Pods().GetPod(pod)
		e = envs.Get().GetExporter()
	)

	if pod == types.EmptyString {
		log.Errorf("%s:pod_status_event:> pod state event: pod is empty", logPrefix)
		return errors.New("pod state event: pod is empty")
	}

	if p == nil {
		return nil
	}

	if p.Local {
		return nil
	}

	e.PodStatus(pod, p)

	return nil
}

// NewRouteStatusEvent - send pod state event after
// node is successful accepted and each hour
func NewVolumeStatusEvent(ctx context.Context, volume string) error {

	var (
		c = envs.Get().GetNodeClient()
	)

	if volume == "" {
		log.Errorf("%s:volume_status_event:> volume state event: volume is empty", logPrefix)
		return errors.New("Event: volume state event: volume is empty")
	}

	log.V(logLevel).Debugf("%s:volume_status_event:> volume state event state: %s", logPrefix, volume)

	opts := v1.Request().Node().NodeVolumeStatusOptions()
	return c.SetVolumeStatus(ctx, volume, opts)
}
<commit_msg>skip seend events for local pods<commit_after>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2018] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package events

import (
	"context"
	"github.com/lastbackend/lastbackend/pkg/api/types/v1"
	"github.com/lastbackend/lastbackend/pkg/api/types/v1/request"
	"github.com/lastbackend/lastbackend/pkg/distribution/types"
	"github.com/lastbackend/lastbackend/pkg/log"
	"github.com/lastbackend/lastbackend/pkg/node/envs"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"io/ioutil"
)

const (
	logLevel  = 3
	logPrefix = "node:events"
)

// NewConnectEventt - send node info event after
// node is successful accepted and each hour
func NewConnectEvent(ctx context.Context) error {

	var (
		c = envs.Get().GetNodeClient()
	)

	opts := v1.Request().Node().NodeConnectOptions()
	opts.Info = envs.Get().GetState().Node().Info
	opts.Status = envs.Get().GetState().Node().Status
	opts.Network = *envs.Get().GetCNI().Info(ctx)

	if viper.IsSet("node.tls") {
		opts.TLS = !viper.GetBool("node.tls.insecure")

		if opts.TLS {
			caData, err := ioutil.ReadFile(viper.GetString("node.tls.ca"))
			if err != nil {
				log.Errorf("%s:connect_event:> read ca cert file err: %v", logPrefix, err)
				return err
			}

			certData, err := ioutil.ReadFile(viper.GetString("node.tls.client_cert"))
			if err != nil {
				log.Errorf("%s:connect_event:> read client cert file err: %v", logPrefix, err)
				return err
			}

			keyData, err := ioutil.ReadFile(viper.GetString("node.tls.client_key"))
			if err != nil {
				log.Errorf("%s:connect_event:> read client key file err: %v", logPrefix, err)
				return err
			}

			opts.SSL = new(request.SSL)
			opts.SSL.CA = caData
			opts.SSL.Key = keyData
			opts.SSL.Cert = certData
		}
	}

	opts.Status.Mode.Ingress = envs.Get().GetModeIngress()
	return c.Connect(ctx, opts)

}

// NewStatusEvent - send node state event after
// node is successful accepted and each hour
func NewStatusEvent(ctx context.Context) error {
	var (
		e = envs.Get().GetExporter()
	)

	e.Resources(envs.Get().GetState().Node().Status)
	return nil
}

// NewPodStatusEvent - send pod state event after
// node is successful accepted and each hour
func NewPodStatusEvent(ctx context.Context, pod string) error {

	var (
		p = envs.Get().GetState().Pods().GetPod(pod)
		e = envs.Get().GetExporter()
	)

	if pod == types.EmptyString {
		log.Errorf("%s:pod_status_event:> pod state event: pod is empty", logPrefix)
		return errors.New("pod state event: pod is empty")
	}

	if p == nil {
		return nil
	}

	if envs.Get().GetState().Pods().IsLocal(pod) {
		return nil
	}

	e.PodStatus(pod, p)

	return nil
}

// NewRouteStatusEvent - send pod state event after
// node is successful accepted and each hour
func NewVolumeStatusEvent(ctx context.Context, volume string) error {

	var (
		c = envs.Get().GetNodeClient()
	)

	if volume == "" {
		log.Errorf("%s:volume_status_event:> volume state event: volume is empty", logPrefix)
		return errors.New("Event: volume state event: volume is empty")
	}

	log.V(logLevel).Debugf("%s:volume_status_event:> volume state event state: %s", logPrefix, volume)

	opts := v1.Request().Node().NodeVolumeStatusOptions()
	return c.SetVolumeStatus(ctx, volume, opts)
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"strings"
	"time"

	"fmt"

	"github.com/zalando-incubator/postgres-operator/pkg/spec"
)

// CRD describes CustomResourceDefinition specific configuration parameters
type CRD struct {
	ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"`
	ReadyWaitTimeout  time.Duration `name:"ready_wait_timeout" default:"30s"`
	ResyncPeriod      time.Duration `name:"resync_period" default:"5m"`
}

// Resources describes kubernetes resource specific configuration parameters
type Resources struct {
	ResourceCheckInterval   time.Duration     `name:"resource_check_interval" default:"3s"`
	ResourceCheckTimeout    time.Duration     `name:"resource_check_timeout" default:"10m"`
	PodLabelWaitTimeout     time.Duration     `name:"pod_label_wait_timeout" default:"10m"`
	PodDeletionWaitTimeout  time.Duration     `name:"pod_deletion_wait_timeout" default:"10m"`
	PodTerminateGracePeriod time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
	ClusterLabels           map[string]string `name:"cluster_labels" default:"application:spilo"`
	ClusterNameLabel        string            `name:"cluster_name_label" default:"cluster-name"`
	PodRoleLabel            string            `name:"pod_role_label" default:"spilo-role"`
	PodToleration           map[string]string `name:"toleration" default:""`
	DefaultCPURequest       string            `name:"default_cpu_request" default:"100m"`
	DefaultMemoryRequest    string            `name:"default_memory_request" default:"100Mi"`
	DefaultCPULimit         string            `name:"default_cpu_limit" default:"3"`
	DefaultMemoryLimit      string            `name:"default_memory_limit" default:"1Gi"`
	PodEnvironmentConfigMap string            `name:"pod_environment_configmap" default:""`
	NodeReadinessLabel      map[string]string `name:"node_readiness_label" default:""`
	MaxInstances            int32             `name:"max_instances" default:"-1"`
	MinInstances            int32             `name:"min_instances" default:"-1"`
}

// Auth describes authentication specific configuration parameters
type Auth struct {
	SecretNameTemplate            stringTemplate      `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"`
	PamRoleName                   string              `name:"pam_role_name" default:"zalandos"`
	PamConfiguration              string              `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"`
	TeamsAPIUrl                   string              `name:"teams_api_url" default:"https://teams.example.com/api/"`
	OAuthTokenSecretName          spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"`
	InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"`
	SuperUsername                 string              `name:"super_username" default:"postgres"`
	ReplicationUsername           string              `name:"replication_username" default:"standby"`
}

// Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:
type Scalyr struct {
	ScalyrAPIKey        string `name:"scalyr_api_key" default:""`
	ScalyrImage         string `name:"scalyr_image" default:""`
	ScalyrServerURL     string `name:"scalyr_server_url" default:"https://upload.eu.scalyr.com"`
	ScalyrCPURequest    string `name:"scalyr_cpu_request" default:"100m"`
	ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"`
	ScalyrCPULimit      string `name:"scalyr_cpu_limit" default:"1"`
	ScalyrMemoryLimit   string `name:"scalyr_memory_limit" default:"1Gi"`
}

// Config describes operator config
type Config struct {
	CRD
	Resources
	Auth
	Scalyr
	WatchedNamespace         string            `name:"watched_namespace"` // may be "*", meaning watch all namespaces
	EtcdHost                 string            `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
	DockerImage              string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`
	ServiceAccountName       string            `name:"service_account_name" default:"operator"`
	DbHostedZone             string            `name:"db_hosted_zone" default:"db.example.com"`
	EtcdScope                string            `name:"etcd_scope" default:"service"`
	WALES3Bucket             string            `name:"wal_s3_bucket"`
	KubeIAMRole              string            `name:"kube_iam_role"`
	DebugLogging             bool              `name:"debug_logging" default:"true"`
	EnableDBAccess           bool              `name:"enable_database_access" default:"true"`
	EnableTeamsAPI           bool              `name:"enable_teams_api" default:"true"`
	EnableTeamSuperuser      bool              `name:"enable_team_superuser" default:"false"`
	TeamAdminRole            string            `name:"team_admin_role" default:"admin"`
	EnableLoadBalancer       bool              `name:"enable_load_balancer" default:"true"`
	MasterDNSNameFormat      stringTemplate    `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
	ReplicaDNSNameFormat     stringTemplate    `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
	PDBNameFormat            stringTemplate    `name:"pdb_name_format" default:"postgres-{cluster}-pdb"`
	Workers                  uint32            `name:"workers" default:"4"`
	APIPort                  int               `name:"api_port" default:"8080"`
	RingLogLines             int               `name:"ring_log_lines" default:"100"`
	ClusterHistoryEntries    int               `name:"cluster_history_entries" default:"1000"`
	TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"`
	PodTerminateGracePeriod  time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
	ProtectedRoles           []string          `name:"protected_role_names" default:"admin"`
}

// MustMarshal marshals the config or panics
func (c Config) MustMarshal() string {
	b, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		panic(err)
	}

	return string(b)
}

// NewFromMap creates Config from the map
func NewFromMap(m map[string]string) *Config {
	cfg := Config{}
	fields, _ := structFields(&cfg)

	for _, structField := range fields {
		key := strings.ToLower(structField.Name)
		value, ok := m[key]
		if !ok && structField.Default != "" {
			value = structField.Default
		}

		if value == "" {
			continue
		}
		err := processField(value, structField.Field)
		if err != nil {
			panic(err)
		}
	}
	if err := validate(&cfg); err != nil {
		panic(err)
	}

	return &cfg
}

// Copy creates a copy of the config
func Copy(c *Config) Config {
	cfg := *c

	cfg.ClusterLabels = make(map[string]string, len(c.ClusterLabels))
	for k, v := range c.ClusterLabels {
		cfg.ClusterLabels[k] = v
	}

	return cfg
}

func validate(cfg *Config) (err error) {
	if cfg.MinInstances > 0 && cfg.MaxInstances > 0 && cfg.MinInstances > cfg.MaxInstances {
		err = fmt.Errorf("minimum number of instances %d is set higher than the maximum number %d",
			cfg.MinInstances, cfg.MaxInstances)
	}
	if cfg.Workers == 0 {
		err = fmt.Errorf("number of workers should be higher than 0")
	}
	return
}
<commit_msg>Comment on special values for watched namespace<commit_after>package config

import (
	"encoding/json"
	"strings"
	"time"

	"fmt"

	"github.com/zalando-incubator/postgres-operator/pkg/spec"
)

// CRD describes CustomResourceDefinition specific configuration parameters
type CRD struct {
	ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"`
	ReadyWaitTimeout  time.Duration `name:"ready_wait_timeout" default:"30s"`
	ResyncPeriod      time.Duration `name:"resync_period" default:"5m"`
}

// Resources describes kubernetes resource specific configuration parameters
type Resources struct {
	ResourceCheckInterval   time.Duration     `name:"resource_check_interval" default:"3s"`
	ResourceCheckTimeout    time.Duration     `name:"resource_check_timeout" default:"10m"`
	PodLabelWaitTimeout     time.Duration     `name:"pod_label_wait_timeout" default:"10m"`
	PodDeletionWaitTimeout  time.Duration     `name:"pod_deletion_wait_timeout" default:"10m"`
	PodTerminateGracePeriod time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
	ClusterLabels           map[string]string `name:"cluster_labels" default:"application:spilo"`
	ClusterNameLabel        string            `name:"cluster_name_label" default:"cluster-name"`
	PodRoleLabel            string            `name:"pod_role_label" default:"spilo-role"`
	PodToleration           map[string]string `name:"toleration" default:""`
	DefaultCPURequest       string            `name:"default_cpu_request" default:"100m"`
	DefaultMemoryRequest    string            `name:"default_memory_request" default:"100Mi"`
	DefaultCPULimit         string            `name:"default_cpu_limit" default:"3"`
	DefaultMemoryLimit      string            `name:"default_memory_limit" default:"1Gi"`
	PodEnvironmentConfigMap string            `name:"pod_environment_configmap" default:""`
	NodeReadinessLabel      map[string]string `name:"node_readiness_label" default:""`
	MaxInstances            int32             `name:"max_instances" default:"-1"`
	MinInstances            int32             `name:"min_instances" default:"-1"`
}

// Auth describes authentication specific configuration parameters
type Auth struct {
	SecretNameTemplate            stringTemplate      `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"`
	PamRoleName                   string              `name:"pam_role_name" default:"zalandos"`
	PamConfiguration              string              `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"`
	TeamsAPIUrl                   string              `name:"teams_api_url" default:"https://teams.example.com/api/"`
	OAuthTokenSecretName          spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"`
	InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"`
	SuperUsername                 string              `name:"super_username" default:"postgres"`
	ReplicationUsername           string              `name:"replication_username" default:"standby"`
}

// Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:
type Scalyr struct {
	ScalyrAPIKey        string `name:"scalyr_api_key" default:""`
	ScalyrImage         string `name:"scalyr_image" default:""`
	ScalyrServerURL     string `name:"scalyr_server_url" default:"https://upload.eu.scalyr.com"`
	ScalyrCPURequest    string `name:"scalyr_cpu_request" default:"100m"`
	ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"`
	ScalyrCPULimit      string `name:"scalyr_cpu_limit" default:"1"`
	ScalyrMemoryLimit   string `name:"scalyr_memory_limit" default:"1Gi"`
}

// Config describes operator config
type Config struct {
	CRD
	Resources
	Auth
	Scalyr
	WatchedNamespace         string            `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
	EtcdHost                 string            `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
	DockerImage              string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`
	ServiceAccountName       string            `name:"service_account_name" default:"operator"`
	DbHostedZone             string            `name:"db_hosted_zone" default:"db.example.com"`
	EtcdScope                string            `name:"etcd_scope" default:"service"`
	WALES3Bucket             string            `name:"wal_s3_bucket"`
	KubeIAMRole              string            `name:"kube_iam_role"`
	DebugLogging             bool              `name:"debug_logging" default:"true"`
	EnableDBAccess           bool              `name:"enable_database_access" default:"true"`
	EnableTeamsAPI           bool              `name:"enable_teams_api" default:"true"`
	EnableTeamSuperuser      bool              `name:"enable_team_superuser" default:"false"`
	TeamAdminRole            string            `name:"team_admin_role" default:"admin"`
	EnableLoadBalancer       bool              `name:"enable_load_balancer" default:"true"`
	MasterDNSNameFormat      stringTemplate    `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"`
	ReplicaDNSNameFormat     stringTemplate    `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"`
	PDBNameFormat            stringTemplate    `name:"pdb_name_format" default:"postgres-{cluster}-pdb"`
	Workers                  uint32            `name:"workers" default:"4"`
	APIPort                  int               `name:"api_port" default:"8080"`
	RingLogLines             int               `name:"ring_log_lines" default:"100"`
	ClusterHistoryEntries    int               `name:"cluster_history_entries" default:"1000"`
	TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"`
	PodTerminateGracePeriod  time.Duration     `name:"pod_terminate_grace_period" default:"5m"`
	ProtectedRoles           []string          `name:"protected_role_names" default:"admin"`
}

// MustMarshal marshals the config or panics
func (c Config) MustMarshal() string {
	b, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		panic(err)
	}

	return string(b)
}

// NewFromMap creates Config from the map
func NewFromMap(m map[string]string) *Config {
	cfg := Config{}
	fields, _ := structFields(&cfg)

	for _, structField := range fields {
		key := strings.ToLower(structField.Name)
		value, ok := m[key]
		if !ok && structField.Default != "" {
			value = structField.Default
		}

		if value == "" {
			continue
		}
		err := processField(value, structField.Field)
		if err != nil {
			panic(err)
		}
	}
	if err := validate(&cfg); err != nil {
		panic(err)
	}

	return &cfg
}

// Copy creates a copy of the config
func Copy(c *Config) Config {
	cfg := *c

	cfg.ClusterLabels = make(map[string]string, len(c.ClusterLabels))
	for k, v := range c.ClusterLabels {
		cfg.ClusterLabels[k] = v
	}

	return cfg
}

func validate(cfg *Config) (err error) {
	if cfg.MinInstances > 0 && cfg.MaxInstances > 0 && cfg.MinInstances > cfg.MaxInstances {
		err = fmt.Errorf("minimum number of instances %d is set higher than the maximum number %d",
			cfg.MinInstances, cfg.MaxInstances)
	}
	if cfg.Workers == 0 {
		err = fmt.Errorf("number of workers should be higher than 0")
	}
	return
}
<|endoftext|>


<commit_before>package vfs

import (
	"encoding/hex"
	"time"

	"github.com/cozy/cozy-stack/pkg/crypto"
)

// A DownloadStore is essentially an object to store Archives & Files by keys
type DownloadStore interface {
	AddFile(f string) (string, error)
	AddArchive(a *Archive) (string, error)
	GetFile(k string) (string, error)
	GetArchive(k string) (*Archive, error)
}

type fileRef struct {
	Path      string
	ExpiresAt time.Time
}

// downloadStoreTTL is the time an Archive stay alive
const downloadStoreTTL = 1 * time.Hour
const downloadStoreCleanInterval = 1 * time.Hour

var storeStore map[string]*memStore

func init() {
	go cleanDownloadStoreInterval()
}

func cleanDownloadStoreInterval() {
	for range time.Tick(downloadStoreCleanInterval) {
		cleanDownloadStore()
	}
}

func cleanStore(s *memStore) {
	now := time.Now()
	for k, f := range s.Files {
		if now.After(f.ExpiresAt) {
			delete(s.Files, k)
		}
	}
	for k, a := range s.Archives {
		if now.After(a.ExpiresAt) {
			delete(s.Archives, k)
		}
	}
}

func cleanDownloadStore() {
	for i, s := range storeStore {
		cleanStore(s)
		if len(s.Files) == 0 && len(s.Archives) == 0 {
			delete(storeStore, i)
		}
	}
}

// GetStore returns the DownloadStore for the given Instance
func GetStore(domain string) DownloadStore {
	if storeStore == nil {
		storeStore = make(map[string]*memStore)
	}
	store, ok := storeStore[domain]
	if ok {
		return store
	}
	storeStore[domain] = &memStore{
		Archives: make(map[string]*Archive),
		Files:    make(map[string]*fileRef),
	}
	return storeStore[domain]
}

type memStore struct {
	Archives map[string]*Archive
	Files    map[string]*fileRef
}

func (s *memStore) makeSecret() string {
	return hex.EncodeToString(crypto.GenerateRandomBytes(16))
}

func (s *memStore) AddFile(f string) (string, error) {
	fref := &fileRef{
		Path:      f,
		ExpiresAt: time.Now().Add(downloadStoreTTL),
	}
	cleanStore(s)
	key := s.makeSecret()
	s.Files[key] = fref
	return key, nil
}

func (s *memStore) AddArchive(a *Archive) (string, error) {
	a.ExpiresAt = time.Now().Add(downloadStoreTTL)
	cleanStore(s)
	key := s.makeSecret()
	s.Archives[key] = a
	return key, nil
}

func (s *memStore) GetFile(k string) (string, error) {
	f, ok := s.Files[k]
	if !ok {
		return "", nil
	}
	if time.Now().After(f.ExpiresAt) {
		delete(s.Files, k)
		return "", nil
	}
	return f.Path, nil
}

func (s *memStore) GetArchive(k string) (*Archive, error) {
	a, ok := s.Archives[k]
	if !ok {
		return nil, nil
	}
	if time.Now().After(a.ExpiresAt) {
		delete(s.Files, k)
		return nil, nil
	}
	return a, nil
}
<commit_msg>add mutex on download store<commit_after>package vfs

import (
	"encoding/hex"
	"sync"
	"time"

	"github.com/cozy/cozy-stack/pkg/crypto"
)

// A DownloadStore is essentially an object to store Archives & Files by keys
type DownloadStore interface {
	AddFile(f string) (string, error)
	AddArchive(a *Archive) (string, error)
	GetFile(k string) (string, error)
	GetArchive(k string) (*Archive, error)
}

type fileRef struct {
	Path      string
	ExpiresAt time.Time
}

// downloadStoreTTL is the time an Archive stay alive
const downloadStoreTTL = 1 * time.Hour
const downloadStoreCleanInterval = 1 * time.Hour

var storeStoreMutex = &sync.Mutex{}
var storeStore map[string]*memStore

func init() {
	go cleanDownloadStoreInterval()
}

func cleanDownloadStoreInterval() {
	for range time.Tick(downloadStoreCleanInterval) {
		cleanDownloadStore()
	}
}

func cleanStore(s *memStore) {
	now := time.Now()
	for k, f := range s.Files {
		if now.After(f.ExpiresAt) {
			delete(s.Files, k)
		}
	}
	for k, a := range s.Archives {
		if now.After(a.ExpiresAt) {
			delete(s.Archives, k)
		}
	}
}

func cleanDownloadStore() {
	storeStoreMutex.Lock()
	defer storeStoreMutex.Unlock()
	for i, s := range storeStore {
		cleanStore(s)
		if len(s.Files) == 0 && len(s.Archives) == 0 {
			delete(storeStore, i)
		}
	}
}

// GetStore returns the DownloadStore for the given Instance
func GetStore(domain string) DownloadStore {
	storeStoreMutex.Lock()
	defer storeStoreMutex.Unlock()
	if storeStore == nil {
		storeStore = make(map[string]*memStore)
	}
	store, exists := storeStore[domain]
	if !exists {
		store = &memStore{
			Archives: make(map[string]*Archive),
			Files:    make(map[string]*fileRef),
		}
		storeStore[domain] = store
	}
	return store
}

type memStore struct {
	Mutex    sync.Mutex
	Archives map[string]*Archive
	Files    map[string]*fileRef
}

func (s *memStore) makeSecret() string {
	return hex.EncodeToString(crypto.GenerateRandomBytes(16))
}

func (s *memStore) AddFile(f string) (string, error) {
	fref := &fileRef{
		Path:      f,
		ExpiresAt: time.Now().Add(downloadStoreTTL),
	}
	s.Mutex.Lock()
	defer s.Mutex.Unlock()
	cleanStore(s)
	key := s.makeSecret()
	s.Files[key] = fref
	return key, nil
}

func (s *memStore) AddArchive(a *Archive) (string, error) {
	a.ExpiresAt = time.Now().Add(downloadStoreTTL)
	s.Mutex.Lock()
	defer s.Mutex.Unlock()
	cleanStore(s)
	key := s.makeSecret()
	s.Archives[key] = a
	return key, nil
}

func (s *memStore) GetFile(k string) (string, error) {
	s.Mutex.Lock()
	defer s.Mutex.Unlock()
	f, ok := s.Files[k]
	if !ok {
		return "", nil
	}
	if time.Now().After(f.ExpiresAt) {
		delete(s.Files, k)
		return "", nil
	}
	return f.Path, nil
}

func (s *memStore) GetArchive(k string) (*Archive, error) {
	s.Mutex.Lock()
	defer s.Mutex.Unlock()
	a, ok := s.Archives[k]
	if !ok {
		return nil, nil
	}
	if time.Now().After(a.ExpiresAt) {
		delete(s.Files, k)
		return nil, nil
	}
	return a, nil
}
<|endoftext|>