<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"net"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apiserver/pkg/server"
	utilfeature "k8s.io/apiserver/pkg/util/feature"

	// add the generic feature gates
	_ "k8s.io/apiserver/pkg/features"

	"github.com/spf13/pflag"
)

// ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct {
	AdvertiseAddress net.IP

	CorsAllowedOriginList       []string
	ExternalHost                string
	MaxRequestsInFlight         int
	MaxMutatingRequestsInFlight int
	RequestTimeout              time.Duration
	MinRequestTimeout           int
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	JSONPatchMaxCopyBytes int64
	// The limit on the request body size that would be accepted and
	// decoded in a write request. 0 means no limit.
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	MaxRequestBodyBytes int64
	TargetRAMMB         int
}

func NewServerRunOptions() *ServerRunOptions {
	defaults := server.NewConfig(serializer.CodecFactory{})
	return &ServerRunOptions{
		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
		RequestTimeout:              defaults.RequestTimeout,
		MinRequestTimeout:           defaults.MinRequestTimeout,
		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
	}
}

// ApplyOptions applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
	c.CorsAllowedOriginList = s.CorsAllowedOriginList
	c.ExternalAddress = s.ExternalHost
	c.MaxRequestsInFlight = s.MaxRequestsInFlight
	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
	c.RequestTimeout = s.RequestTimeout
	c.MinRequestTimeout = s.MinRequestTimeout
	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
	c.PublicAddress = s.AdvertiseAddress

	return nil
}

// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
	if secure == nil {
		return nil
	}

	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
		hostIP, err := secure.DefaultExternalAddress()
		if err != nil {
			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
		}
		s.AdvertiseAddress = hostIP
	}

	return nil
}

// Validate checks validation of ServerRunOptions
func (s *ServerRunOptions) Validate() []error {
	errors := []error{}
	if s.TargetRAMMB < 0 {
		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
	}
	if s.MaxRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
	}
	if s.MaxMutatingRequestsInFlight < 0 {
		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
	}

	if s.RequestTimeout.Nanoseconds() < 0 {
		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
	}

	if s.MinRequestTimeout < 0 {
		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
	}

	if s.JSONPatchMaxCopyBytes < 0 {
		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
	}

	if s.MaxRequestBodyBytes < 0 {
		errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
	}

	return errors
}

// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
	// arrange these text blocks sensibly. Grrr.

	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
		"The IP address on which to advertise the apiserver to members of the cluster. This "+
		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
		"will be used. If --bind-address is unspecified, the host's default interface will "+
		"be used.")

	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")

	fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
		"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")

	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).")

	deprecatedMasterServiceNamespace := metav1.NamespaceDefault
	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
		"DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.")

	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
		"The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
		"The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
		"An optional field indicating the duration a handler must keep a request open before timing "+
		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
		"--min-request-timeout for specific types of requests.")

	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
		"An optional field indicating the minimum number of seconds a handler must keep "+
		"a request open before timing it out. Currently only honored by the watch request "+
		"handler, which picks a randomized value above this number as the connection timeout, "+
		"to spread out load.")

	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
<commit_msg>add new flag for enabling requests mgmt handler<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"k8s.io/apiserver/pkg/features"
	"net"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apiserver/pkg/server"
	utilfeature "k8s.io/apiserver/pkg/util/feature"

	// add the generic feature gates
	_ "k8s.io/apiserver/pkg/features"

	"github.com/spf13/pflag"
)

// ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct {
	AdvertiseAddress net.IP

	CorsAllowedOriginList       []string
	ExternalHost                string
	MaxRequestsInFlight         int
	MaxMutatingRequestsInFlight int
	RequestTimeout              time.Duration
	MinRequestTimeout           int
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	JSONPatchMaxCopyBytes int64
	// The limit on the request body size that would be accepted and
	// decoded in a write request. 0 means no limit.
	// We intentionally did not add a flag for this option. Users of the
	// apiserver library can wire it to a flag.
	MaxRequestBodyBytes       int64
	TargetRAMMB               int
	EnableInfightQuotaHandler bool
}

func NewServerRunOptions() *ServerRunOptions {
	defaults := server.NewConfig(serializer.CodecFactory{})
	return &ServerRunOptions{
		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
		RequestTimeout:              defaults.RequestTimeout,
		MinRequestTimeout:           defaults.MinRequestTimeout,
		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
	}
}

// ApplyOptions applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
	c.CorsAllowedOriginList = s.CorsAllowedOriginList
	c.ExternalAddress = s.ExternalHost
	c.MaxRequestsInFlight = s.MaxRequestsInFlight
	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
	c.RequestTimeout = s.RequestTimeout
	c.MinRequestTimeout = s.MinRequestTimeout
	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
	c.PublicAddress = s.AdvertiseAddress

	return nil
}

// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
	if secure == nil {
		return nil
	}

	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
		hostIP, err := secure.DefaultExternalAddress()
		if err != nil {
			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
		}
		s.AdvertiseAddress = hostIP
	}

	return nil
}

// Validate checks validation of ServerRunOptions
func (s *ServerRunOptions) Validate() []error {
	errors := []error{}
	if s.TargetRAMMB < 0 {
		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
	}

	if s.EnableInfightQuotaHandler {
		if !utilfeature.DefaultFeatureGate.Enabled(features.RequestManagement) {
			errors = append(errors, fmt.Errorf("--enable-inflight-quota-handler can not be set if feature "+
				"gate RequestManagement is disabled"))
		}
		if s.MaxMutatingRequestsInFlight != 0 {
			errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight=%v "+
				"can not be set if enabled inflight quota handler", s.MaxMutatingRequestsInFlight))
		}
		if s.MaxRequestsInFlight != 0 {
			errors = append(errors, fmt.Errorf("--max-requests-inflight=%v "+
				"can not be set if enabled inflight quota handler", s.MaxRequestsInFlight))
		}
	} else {
		if s.MaxRequestsInFlight < 0 {
			errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
		}
		if s.MaxMutatingRequestsInFlight < 0 {
			errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
		}
	}

	if s.RequestTimeout.Nanoseconds() < 0 {
		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
	}

	if s.MinRequestTimeout < 0 {
		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
	}

	if s.JSONPatchMaxCopyBytes < 0 {
		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
	}

	if s.MaxRequestBodyBytes < 0 {
		errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
	}

	return errors
}

// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
	// arrange these text blocks sensibly. Grrr.

	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
		"The IP address on which to advertise the apiserver to members of the cluster. This "+
		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
		"will be used. If --bind-address is unspecified, the host's default interface will "+
		"be used.")

	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")

	fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
		"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")

	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).")

	deprecatedMasterServiceNamespace := metav1.NamespaceDefault
	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
		"DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.")

	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
		"The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
		"The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+
		"it rejects requests. Zero for no limit.")

	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
		"An optional field indicating the duration a handler must keep a request open before timing "+
		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
		"--min-request-timeout for specific types of requests.")

	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
		"An optional field indicating the minimum number of seconds a handler must keep "+
		"a request open before timing it out. Currently only honored by the watch request "+
		"handler, which picks a randomized value above this number as the connection timeout, "+
		"to spread out load.")

	fs.BoolVar(&s.EnableInfightQuotaHandler, "enable-inflight-quota-handler", s.EnableInfightQuotaHandler, ""+
		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")

	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"encoding"

	keybase1 "github.com/keybase/client/go/protocol"
	merkle "github.com/keybase/go-merkle-tree"
)

// MerkleRootVersion is the current Merkle root version.
const MerkleRootVersion = 1

// MerkleRoot represents a signed Merkle tree root.
type MerkleRoot struct {
	Version   int                   `codec:"v"`
	TreeID    keybase1.MerkleTreeID `codec:"t"`
	SeqNo     int64                 `codec:"sn"`
	Timestamp int64                 `codec:"ts"`
	Hash      merkle.Hash           `codec:"h"`
	PrevRoot  merkle.Hash           `codec:"pr"`
	EPubKey   TLFEphemeralPublicKey `codec:"epk"`
	Nonce     [24]byte              `codec:"non"`
}

// MerkleLeaf is the value of a Merkle leaf node.
type MerkleLeaf struct {
	_struct   bool `codec:",toarray"`
	Revision  MetadataRevision
	Hash      MerkleHash // hash of the signed metadata object
	Timestamp int64
}

var _ merkle.ValueConstructor = (*MerkleLeaf)(nil)

// Construct implements the go-merkle-tree.ValueConstructor interface.
func (l MerkleLeaf) Construct() interface{} {
	// In the Merkle tree leaves are simply byte slices.
	return []byte{}
}

// MerkleHash is the hash of a RootMetadataSigned block.
type MerkleHash struct {
	h Hash
}

var _ encoding.BinaryMarshaler = MerkleHash{}
var _ encoding.BinaryUnmarshaler = (*MerkleHash)(nil)

// MerkleHashFromBytes creates a new MerkleHash from the given bytes. If the
// returned error is nil, the returned MerkleHash is valid.
func MerkleHashFromBytes(data []byte) (MerkleHash, error) {
	h, err := HashFromBytes(data)
	if err != nil {
		return MerkleHash{}, err
	}
	return MerkleHash{h}, nil
}

// Bytes returns the bytes of the MerkleHash.
func (h MerkleHash) Bytes() []byte {
	return h.h.Bytes()
}

// String returns the string form of the MerkleHash.
func (h MerkleHash) String() string {
	return h.h.String()
}

// MarshalBinary implements the encoding.BinaryMarshaler interface for
// MerkleHash. Returns an error if the MerkleHash is invalid and not the
// zero MerkleHash.
func (h MerkleHash) MarshalBinary() (data []byte, err error) {
	return h.h.MarshalBinary()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
// for MerkleHash. Returns an error if the given byte array is non-empty and
// the MerkleHash is invalid.
func (h *MerkleHash) UnmarshalBinary(data []byte) error {
	return h.h.UnmarshalBinary(data)
}
<commit_msg>mdmerkle: don't serialize epk/nonce for pub roots<commit_after>package libkbfs

import (
	"encoding"

	keybase1 "github.com/keybase/client/go/protocol"
	merkle "github.com/keybase/go-merkle-tree"
)

// MerkleRootVersion is the current Merkle root version.
const MerkleRootVersion = 1

// MerkleRoot represents a signed Merkle tree root.
type MerkleRoot struct {
	Version   int                    `codec:"v"`
	TreeID    keybase1.MerkleTreeID  `codec:"t"`
	SeqNo     int64                  `codec:"sn"`
	Timestamp int64                  `codec:"ts"`
	Hash      merkle.Hash            `codec:"h"`
	PrevRoot  merkle.Hash            `codec:"pr"`
	EPubKey   *TLFEphemeralPublicKey `codec:"epk,omitempty"`
	Nonce     *[24]byte              `codec:"non,omitempty"`
}

// MerkleLeaf is the value of a Merkle leaf node.
type MerkleLeaf struct {
	_struct   bool `codec:",toarray"`
	Revision  MetadataRevision
	Hash      MerkleHash // hash of the signed metadata object
	Timestamp int64
}

var _ merkle.ValueConstructor = (*MerkleLeaf)(nil)

// Construct implements the go-merkle-tree.ValueConstructor interface.
func (l MerkleLeaf) Construct() interface{} {
	// In the Merkle tree leaves are simply byte slices.
	return []byte{}
}

// MerkleHash is the hash of a RootMetadataSigned block.
type MerkleHash struct {
	h Hash
}

var _ encoding.BinaryMarshaler = MerkleHash{}
var _ encoding.BinaryUnmarshaler = (*MerkleHash)(nil)

// MerkleHashFromBytes creates a new MerkleHash from the given bytes. If the
// returned error is nil, the returned MerkleHash is valid.
func MerkleHashFromBytes(data []byte) (MerkleHash, error) {
	h, err := HashFromBytes(data)
	if err != nil {
		return MerkleHash{}, err
	}
	return MerkleHash{h}, nil
}

// Bytes returns the bytes of the MerkleHash.
func (h MerkleHash) Bytes() []byte {
	return h.h.Bytes()
}

// String returns the string form of the MerkleHash.
func (h MerkleHash) String() string {
	return h.h.String()
}

// MarshalBinary implements the encoding.BinaryMarshaler interface for
// MerkleHash. Returns an error if the MerkleHash is invalid and not the
// zero MerkleHash.
func (h MerkleHash) MarshalBinary() (data []byte, err error) {
	return h.h.MarshalBinary()
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
// for MerkleHash. Returns an error if the given byte array is non-empty and
// the MerkleHash is invalid.
func (h *MerkleHash) UnmarshalBinary(data []byte) error {
	return h.h.UnmarshalBinary(data)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"

	"github.com/bitly/go-nsq"
)

var (
	queue    = flag.String("queue", "127.0.0.1:4161", "queue address")
	topic    = flag.String("topic", "jobs", "queue topic")
	channel  = flag.String("channel", "qmd-worker", "queue channel")
	consumer *nsq.Consumer
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-worker [-queue=<queue>] [-topic=<topic>] [-channel=<channel>]")
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	fmt.Printf("Worker with topic: %s and channel: %s\n", *topic, *channel)

	termChan := make(chan os.Signal, 1)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	// Create and configure the consumer.
	var err error
	consumer, err = nsq.NewConsumer(*topic, *channel, nsq.NewConfig())
	if err != nil {
		fmt.Println(err)
	}

	// Set the message handler.
	consumer.SetHandler(nsq.HandlerFunc(func(m *nsq.Message) error {
		cmd := strings.Split(string(m.Body), " ")
		out, err := exec.Command(string(cmd[0]), cmd[1:]...).Output()
		if err != nil {
			return err
		}
		fmt.Println(string(out))
		return nil
	}))

	// Connect the consumer.
	fmt.Printf("Connecting to %s\n", *queue)
	err = consumer.ConnectToNSQLookupd(*queue)
	if err != nil {
		fmt.Println(err)
	}

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-termChan:
			consumer.Stop()
		}
	}
}
<commit_msg>Modified qmd-worker to just print messages.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/bitly/go-nsq"
)

var (
	queue    = flag.String("queue", "127.0.0.1:4161", "queue address")
	topic    = flag.String("topic", "jobs", "queue topic")
	channel  = flag.String("channel", "qmd-worker", "queue channel")
	consumer *nsq.Consumer
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-worker [-queue=<queue>] [-topic=<topic>] [-channel=<channel>]")
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	fmt.Printf("Worker with topic: %s and channel: %s\n", *topic, *channel)

	termChan := make(chan os.Signal, 1)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	// Create and configure the consumer.
	var err error
	consumer, err = nsq.NewConsumer(*topic, *channel, nsq.NewConfig())
	if err != nil {
		fmt.Println(err)
	}

	// Set the message handler.
	consumer.SetHandler(nsq.HandlerFunc(func(m *nsq.Message) error {
		//		cmd := strings.Split(string(m.Body), " ")
		//		out, err := exec.Command(string(cmd[0]), cmd[1:]...).Output()
		//		if err != nil {
		//			return err
		//		}
		//		fmt.Println(string(out))
		fmt.Println(string(m.Body))
		return nil
	}))

	// Connect the consumer.
	fmt.Printf("Connecting to %s\n", *queue)
	err = consumer.ConnectToNSQLookupd(*queue)
	if err != nil {
		fmt.Println(err)
	}

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-termChan:
			consumer.Stop()
		}
	}
}
<|endoftext|>


<commit_before>package kegg

import (
    "log"
    "github.com/fjukstad/gocache"
    "github.com/fjukstad/gographer"
    "io/ioutil"
    "encoding/xml"
    "strconv"
    "io"
    "fmt"
    "encoding/csv"
    "strings"
    "math/rand"
    "time"
    "net/http"
    "image/png"

)

type Pathway struct {
    Id string
    Name string
    Class string
    Pathway_Map string
    Diseases [] string
    Drugs []string
    DBLinks [] string
    Organism string
    Genes []string
    Compounds [] string
}

type KeggPathway struct {
    XMLName xml.Name            `xml:"pathway"`
    Name string                 `xml:"name,attr"`
    Org string                  `xml:"org,attr"`
    Number string               `xml:"number,attr"`
    Title string                `xml:"title,attr"`
    Image string                `xml:"image,attr"`
    Link string                 `xml:"link,attr"`
    Entries []KeggEntry         `xml:"entry"`
    Relations []KeggRelation    `xml:"relation"`
}

type KeggEntry struct {
    Id string               `xml:"id,attr"`
    Name string             `xml:"name,attr"`
    Type string             `xml:"type,attr"`
    Link string             `xml:"link,attr"`
    Graphics KeggGraphics   `xml:"graphics"`

}

type KeggRelation struct {
    Entry1 string           `xml:"entry1,attr"`
    Entry2 string           `xml:"entry2,attr"`
    Type string             `xml:"type,attr"`
    Subtypes [] KeggSubtype `xml:"relation>subtype"`
}

type KeggGraphics struct {
    Name string         `xml:"name,attr"`
    Fgcolor string      `xml:"fgcolor,attr"`
    Bgcolor string      `xml:"bgcolor,attr"`
    Type string         `xml:"type,attr"`
    X string            `xml:"x,attr"`
    Y string            `xml:"y,attr"`
    Width string        `xml:"width,attr"`
    Height string       `xml:"height,attr"`

}

type KeggSubtype struct {
    Name string     `xml:"name,attr"`
    Value string    `xml:"value,attr"`
}

func ReadablePathwayNames(ids [] string) [] string {

    pathways := make([] string, len(ids))

    for i, id := range ids {
        pw := GetPathway(id)
        pathways[i] = pw.Name
    }

    return pathways

}

func getMap(url string) ([]byte) {
    response, err := gocache.Get(url)
    if err != nil {
        log.Panic("Could not download pathway kgml:",err)
    }
    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        log.Panic("KGML body could not be read:", err)
    }
    return body
}

func (pathway *KeggPathway) Print() {
    log.Println("XMLName:", pathway.XMLName)
    log.Println("Name:", pathway.Name)
    log.Println("Org:" ,pathway.Org)
    log.Println("Title:", pathway.Title)
    log.Println("Image:", pathway.Image)
    log.Println("Link", pathway.Link)
    log.Println("Entries:" , pathway.Entries)
    log.Println("Relations:", pathway.Relations)
}

func createPathwayGraph(keggId string, inputGraph *gographer.Graph){

    baseURL := "http://rest.kegg.jp/get/"+keggId
    url := baseURL + "/kgml"

    // url := "http://localhost:8000/public/pathway.kgml"
    pw := getMap(url)

    pathway := new(KeggPathway)
    err := xml.Unmarshal(pw, pathway)

    if err != nil {
        log.Panic("Could not unmarshal KGML ", err)
    }

    imgurl := "http://www.genome.jp/kegg/pathway/hsa/" +keggId + ".png"

    log.Print("downloading image...")
    resp, err := http.Get(imgurl)
    if err != nil {
        log.Panic("Image could not be downloaded ", err)
    }

    log.Print("image downloaded....")
    img, err := png.Decode(resp.Body)

    if err != nil {
        log.Panic("Image could not be decoded ", err)
    }

    imgrect := img.Bounds()

    sizeX := imgrect.Max.X - imgrect.Min.X
    sizeY := imgrect.Max.Y - imgrect.Min.Y

    log.Print("Image is : ", sizeX, sizeY)
    // First create a node that will serve as a background image to the pathway
    inputGraph.AddGraphicNode(
        0,
        "bg",
        0,
        1,
        imgurl,
        "#fff",
        "#fff",
        "rectangle",
        sizeX/2,
        sizeY/2,
        sizeX,
        sizeY)

    // Generate some nodes
    for j := range(pathway.Entries) {
        ent := pathway.Entries[j]
        id, _ := strconv.Atoi(ent.Id)
        name := ent.Name
        t, _ := strconv.Atoi(ent.Type)
        size := 1
        graphics := ent.Graphics

        description := strings.Split(graphics.Name, ", ")[0]
        fgcolor := graphics.Fgcolor
        bgcolor := graphics.Bgcolor
        shape := graphics.Type
        x, _ := strconv.Atoi(graphics.X)
        y, _ := strconv.Atoi(graphics.Y)
        height, _ := strconv.Atoi(graphics.Height)
        width, _ := strconv.Atoi(graphics.Width)

        inputGraph.AddGraphicNode(id,name,t,size,description,fgcolor,bgcolor,shape,
                            x,y,height,width)
    }

    // Generate some edges
    for i := range(pathway.Relations) {
        rel := pathway.Relations[i]
        source, _ := strconv.Atoi(rel.Entry1)
        target, _ := strconv.Atoi(rel.Entry2)
        weight := 19
        inputGraph.AddEdge(source, target, i, weight)
    }

    return
}

func GetPathway(id string) Pathway {
    baseURL := "http://rest.kegg.jp/get/"
    url := baseURL + id

    log.Print(url)

    response, err := gocache.Get(url)
    if err != nil{
        log.Panic("Cannot download pathway:", err)
    }

    pathway := parsePathwayResponse(response.Body)

    return pathway

}

func (p Pathway) Print() {
    fmt.Println("\nPathway")
    fmt.Println("\tId:", p.Id)
    fmt.Println("\tName:", p.Name)
    fmt.Println("\tClass:", p.Class)
    fmt.Println("\tPathway map:", p.Pathway_Map)
    fmt.Println("\tDiseases:", p.Diseases)
    fmt.Println("\tDrugs:", p.Drugs)
    fmt.Println("\tOrganism:", p.Organism)
    fmt.Println("\tGenes:", p.Genes)
    fmt.Println("\tCompounds:", p.Compounds)
    fmt.Println("")
}

func GiveMeSomePathways() [] string {
    pw := []string{
        "hsa05200",
        "hsa04915",
        "hsa04612",
        "hsa04062",
        "hsa04660",
        "hsa04630",
        "hsa04151",
        "hsa04310",
        "hsa04662",
    }
    return pw
}

func PathwayGraphFrom(pathway string) string {
    //pw := GetPathway(pathway)

    // Set up graph
    port := getRandomPort()
    graph := gographer.NewGraphAt(":"+strconv.Itoa(port))

    // Create a graph from pathways. Will add nodes and
    // edges and all that jazz
    createPathwayGraph(pathway, graph)

    // Return address where client can retrieve graph
    // TODO: maybe the graph initialization (after wsserver setup)
    // can be done in a go routine?

    return graph.ServerInfo()
}

// TODO: Should store ports used for visualization gateways

func getRandomPort() int {
    r := rand.New(rand.NewSource(time.Now().UnixNano()))
    return 1024 + r.Intn(45000)
}

func parsePathwayResponse(response io.ReadCloser) Pathway {

    tsv := csv.NewReader(response)
    tsv.Comma = '\t'
    tsv.Comment = '#'
    tsv.LazyQuotes = true
    tsv.TrailingComma = true
    tsv.TrimLeadingSpace = false

    records, err := tsv.ReadAll()

    if err != nil {
        log.Panic("Error reading records:", err)
    }

    p := Pathway{}
    tmp := make([] string, 0)
    current := ""
    for i := range records {

        line := strings.Split(records[i][0]," ")

        switch line[0] {
        case "ENTRY":
            a := strings.Join(line[7:], " ")
            b :=  strings.Split(a, " ")[0]
            p.Id = b

        case "NAME":
            p.Name = strings.Join(line[8:], " ")

        case "CLASS":
            p.Class = strings.Join(line[8:], " ")

        case "PATHWAY_MAP":
            p.Pathway_Map = strings.Join(line[1:], " ")

        case "DISEASE":
            current = "DISEASE"

            a := strings.Join(line[5:], " ")
            tmp = append(tmp, a)

        case "DBLINKS":
            p.Diseases = tmp
            current = "DBLINKS"

            a := strings.Join(line[5:], " ")
            tmp  = make([]string, 0)
            tmp = append(tmp, a)

        case "DRUG":
            p.Diseases = tmp
            tmp  = make([]string, 0)
            current = "DRUG"
            a := strings.Join(line[8:], " ")
            tmp = append(tmp, a)

        case "ORGANISM":
            if current == "DISEASE" {
                p.Drugs = tmp
            }
            if current == "DBLINKS" {
                p.DBLinks = tmp
            }
            p.Organism = strings.Join(line[4:], " ")

        case "GENE":
            current = "GENE"
            tmp = make([]string,0)
            a := strings.Join(line[8:], " ")
            b :=  strings.Split(a, " ")[0]
            tmp = append(tmp, b)

        case "COMPOUND":
            p.Genes = tmp
            current = "COMPOUND"
            tmp = make([]string, 0)
            a := strings.Join(line[4:], " ")

            tmp = append(tmp, a)

        case "REFERENCE":
            p.Compounds = tmp
            current = "REFERENCE"
            break

        default:
            if(current == "DISEASE"){
                a := strings.Join(line[12:], " ")
                tmp = append(tmp, a)
            }
            if(current == "DRUG"){
                a := strings.Join(line[0:], " ")
                b := strings.Replace(a, "    ", "",-1)
                tmp = append(tmp, b)
            }
            if(current == "GENE"){
                a := strings.Join(line[0:], " ")
                a = strings.Replace(a, "    ", "",-1)
                b :=  strings.Split(a, " ")[0]
                tmp = append(tmp, b)

            }
            if(current == "COMPOUND"){
                a := strings.Join(line[0:], " ")
                a = strings.Replace(a, "    ", "",-1)

                tmp = append(tmp, a)

            }

        }
    }
    return p

}

<commit_msg>Readable pathway names for each pw.<commit_after>package kegg

import (
    "log"
    "github.com/fjukstad/gocache"
    "github.com/fjukstad/gographer"
    "io/ioutil"
    "encoding/xml"
    "strconv"
    "io"
    "fmt"
    "encoding/csv"
    "strings"
    "math/rand"
    "time"
    "net/http"
    "image/png"

)

type Pathway struct {
    Id string
    Name string
    Class string
    Pathway_Map string
    Diseases [] string
    Drugs []string
    DBLinks [] string
    Organism string
    Genes []string
    Compounds [] string
}

type KeggPathway struct {
    XMLName xml.Name            `xml:"pathway"`
    Name string                 `xml:"name,attr"`
    Org string                  `xml:"org,attr"`
    Number string               `xml:"number,attr"`
    Title string                `xml:"title,attr"`
    Image string                `xml:"image,attr"`
    Link string                 `xml:"link,attr"`
    Entries []KeggEntry         `xml:"entry"`
    Relations []KeggRelation    `xml:"relation"`
}

type KeggEntry struct {
    Id string               `xml:"id,attr"`
    Name string             `xml:"name,attr"`
    Type string             `xml:"type,attr"`
    Link string             `xml:"link,attr"`
    Graphics KeggGraphics   `xml:"graphics"`

}

type KeggRelation struct {
    Entry1 string           `xml:"entry1,attr"`
    Entry2 string           `xml:"entry2,attr"`
    Type string             `xml:"type,attr"`
    Subtypes [] KeggSubtype `xml:"relation>subtype"`
}

type KeggGraphics struct {
    Name string         `xml:"name,attr"`
    Fgcolor string      `xml:"fgcolor,attr"`
    Bgcolor string      `xml:"bgcolor,attr"`
    Type string         `xml:"type,attr"`
    X string            `xml:"x,attr"`
    Y string            `xml:"y,attr"`
    Width string        `xml:"width,attr"`
    Height string       `xml:"height,attr"`

}

type KeggSubtype struct {
    Name string     `xml:"name,attr"`
    Value string    `xml:"value,attr"`
}

func ReadablePathwayNames(ids [] string) [] string {

    pathways := make([] string, len(ids))

    for i, id := range ids {
        pathways[i] = ReadablePathwayName(id)
    }

    return pathways

}

func ReadablePathwayName(id string) string {
    return GetPathway(id).Name
}

func getMap(url string) ([]byte) {
    response, err := gocache.Get(url)
    if err != nil {
        log.Panic("Could not download pathway kgml:",err)
    }
    body, err := ioutil.ReadAll(response.Body)
    if err != nil {
        log.Panic("KGML body could not be read:", err)
    }
    return body
}

func (pathway *KeggPathway) Print() {
    log.Println("XMLName:", pathway.XMLName)
    log.Println("Name:", pathway.Name)
    log.Println("Org:" ,pathway.Org)
    log.Println("Title:", pathway.Title)
    log.Println("Image:", pathway.Image)
    log.Println("Link", pathway.Link)
    log.Println("Entries:" , pathway.Entries)
    log.Println("Relations:", pathway.Relations)
}

func createPathwayGraph(keggId string, inputGraph *gographer.Graph){

    baseURL := "http://rest.kegg.jp/get/"+keggId
    url := baseURL + "/kgml"

    // url := "http://localhost:8000/public/pathway.kgml"
    pw := getMap(url)

    pathway := new(KeggPathway)
    err := xml.Unmarshal(pw, pathway)

    if err != nil {
        log.Panic("Could not unmarshal KGML ", err)
    }

    imgurl := "http://www.genome.jp/kegg/pathway/hsa/" +keggId + ".png"

    log.Print("downloading image...")
    resp, err := http.Get(imgurl)
    if err != nil {
        log.Panic("Image could not be downloaded ", err)
    }

    log.Print("image downloaded....")
    img, err := png.Decode(resp.Body)

    if err != nil {
        log.Panic("Image could not be decoded ", err)
    }

    imgrect := img.Bounds()

    sizeX := imgrect.Max.X - imgrect.Min.X
    sizeY := imgrect.Max.Y - imgrect.Min.Y

    log.Print("Image is : ", sizeX, sizeY)
    // First create a node that will serve as a background image to the pathway
    inputGraph.AddGraphicNode(
        0,
        "bg",
        0,
        1,
        imgurl,
        "#fff",
        "#fff",
        "rectangle",
        sizeX/2,
        sizeY/2,
        sizeX,
        sizeY)

    // Generate some nodes
    for j := range(pathway.Entries) {
        ent := pathway.Entries[j]
        id, _ := strconv.Atoi(ent.Id)
        name := ent.Name
        t, _ := strconv.Atoi(ent.Type)
        size := 1
        graphics := ent.Graphics

        description := strings.Split(graphics.Name, ", ")[0]
        fgcolor := graphics.Fgcolor
        bgcolor := graphics.Bgcolor
        shape := graphics.Type
        x, _ := strconv.Atoi(graphics.X)
        y, _ := strconv.Atoi(graphics.Y)
        height, _ := strconv.Atoi(graphics.Height)
        width, _ := strconv.Atoi(graphics.Width)

        inputGraph.AddGraphicNode(id,name,t,size,description,fgcolor,bgcolor,shape,
                            x,y,height,width)
    }

    // Generate some edges
    for i := range(pathway.Relations) {
        rel := pathway.Relations[i]
        source, _ := strconv.Atoi(rel.Entry1)
        target, _ := strconv.Atoi(rel.Entry2)
        weight := 19
        inputGraph.AddEdge(source, target, i, weight)
    }

    return
}

func GetPathway(id string) Pathway {
    baseURL := "http://rest.kegg.jp/get/"
    url := baseURL + id

    log.Print(url)

    response, err := gocache.Get(url)
    if err != nil{
        log.Panic("Cannot download pathway:", err)
    }

    pathway := parsePathwayResponse(response.Body)

    return pathway

}

func (p Pathway) Print() {
    fmt.Println("\nPathway")
    fmt.Println("\tId:", p.Id)
    fmt.Println("\tName:", p.Name)
    fmt.Println("\tClass:", p.Class)
    fmt.Println("\tPathway map:", p.Pathway_Map)
    fmt.Println("\tDiseases:", p.Diseases)
    fmt.Println("\tDrugs:", p.Drugs)
    fmt.Println("\tOrganism:", p.Organism)
    fmt.Println("\tGenes:", p.Genes)
    fmt.Println("\tCompounds:", p.Compounds)
    fmt.Println("")
}

func GiveMeSomePathways() [] string {
    pw := []string{
        "hsa05200",
        "hsa04915",
        "hsa04612",
        "hsa04062",
        "hsa04660",
        "hsa04630",
        "hsa04151",
        "hsa04310",
        "hsa04662",
    }
    return pw
}

func PathwayGraphFrom(pathway string) string {
    //pw := GetPathway(pathway)

    // Set up graph
    port := getRandomPort()
    graph := gographer.NewGraphAt(":"+strconv.Itoa(port))

    // Create a graph from pathways. Will add nodes and
    // edges and all that jazz
    createPathwayGraph(pathway, graph)

    // Return address where client can retrieve graph
    // TODO: maybe the graph initialization (after wsserver setup)
    // can be done in a go routine?

    return graph.ServerInfo()
}

// TODO: Should store ports used for visualization gateways

func getRandomPort() int {
    r := rand.New(rand.NewSource(time.Now().UnixNano()))
    return 1024 + r.Intn(45000)
}

func parsePathwayResponse(response io.ReadCloser) Pathway {

    tsv := csv.NewReader(response)
    tsv.Comma = '\t'
    tsv.Comment = '#'
    tsv.LazyQuotes = true
    tsv.TrailingComma = true
    tsv.TrimLeadingSpace = false

    records, err := tsv.ReadAll()

    if err != nil {
        log.Panic("Error reading records:", err)
    }

    p := Pathway{}
    tmp := make([] string, 0)
    current := ""
    for i := range records {

        line := strings.Split(records[i][0]," ")

        switch line[0] {
        case "ENTRY":
            a := strings.Join(line[7:], " ")
            b :=  strings.Split(a, " ")[0]
            p.Id = b

        case "NAME":
            p.Name = strings.Join(line[8:], " ")

        case "CLASS":
            p.Class = strings.Join(line[8:], " ")

        case "PATHWAY_MAP":
            p.Pathway_Map = strings.Join(line[1:], " ")

        case "DISEASE":
            current = "DISEASE"

            a := strings.Join(line[5:], " ")
            tmp = append(tmp, a)

        case "DBLINKS":
            p.Diseases = tmp
            current = "DBLINKS"

            a := strings.Join(line[5:], " ")
            tmp  = make([]string, 0)
            tmp = append(tmp, a)

        case "DRUG":
            p.Diseases = tmp
            tmp  = make([]string, 0)
            current = "DRUG"
            a := strings.Join(line[8:], " ")
            tmp = append(tmp, a)

        case "ORGANISM":
            if current == "DISEASE" {
                p.Drugs = tmp
            }
            if current == "DBLINKS" {
                p.DBLinks = tmp
            }
            p.Organism = strings.Join(line[4:], " ")

        case "GENE":
            current = "GENE"
            tmp = make([]string,0)
            a := strings.Join(line[8:], " ")
            b :=  strings.Split(a, " ")[0]
            tmp = append(tmp, b)

        case "COMPOUND":
            p.Genes = tmp
            current = "COMPOUND"
            tmp = make([]string, 0)
            a := strings.Join(line[4:], " ")

            tmp = append(tmp, a)

        case "REFERENCE":
            p.Compounds = tmp
            current = "REFERENCE"
            break

        default:
            if(current == "DISEASE"){
                a := strings.Join(line[12:], " ")
                tmp = append(tmp, a)
            }
            if(current == "DRUG"){
                a := strings.Join(line[0:], " ")
                b := strings.Replace(a, "    ", "",-1)
                tmp = append(tmp, b)
            }
            if(current == "GENE"){
                a := strings.Join(line[0:], " ")
                a = strings.Replace(a, "    ", "",-1)
                b :=  strings.Split(a, " ")[0]
                tmp = append(tmp, b)

            }
            if(current == "COMPOUND"){
                a := strings.Join(line[0:], " ")
                a = strings.Replace(a, "    ", "",-1)

                tmp = append(tmp, a)

            }

        }
    }
    return p

}

<|endoftext|>


<commit_before>// Package tdigest provides a highly accurate mergeable data-structure
// for quantile estimation.
//
// Typical T-Digest use cases involve accumulating metrics on several
// distinct nodes of a cluster and then merging them together to get
// a system-wide quantile overview. Things such as: sensory data from
// IoT devices, quantiles over enormous document datasets (think
// ElasticSearch), performance metrics for distributed systems, etc.
//
// After you create (and configure, if desired) the digest:
//     digest := tdigest.New(tdigest.Compression(100))
//
// You can then use it for registering measurements:
//     digest.Add(number)
//
// Estimating quantiles:
//     digest.Quantile(0.99)
//
// And merging with another digest:
//     digest.Merge(otherDigest)
package tdigest

import (
	"fmt"
	"math"
)

// TDigest is a quantile approximation data structure.
type TDigest struct {
	summary     *summary
	compression float64
	count       uint64
	rng         RNG
}

// New creates a new digest.
//
// By default the digest is constructed with a configuration that
// should be useful for most use-cases. It comes with compression
// set to 100 and uses the global random number generator (same
// as using math/rand top-level functions).
func New(options ...tdigestOption) (*TDigest, error) {
	tdigest := &TDigest{
		compression: 100,
		count:       0,
		rng:         &globalRNG{},
	}

	for _, option := range options {
		err := option(tdigest)
		if err != nil {
			return nil, err
		}
	}

	tdigest.summary = newSummary(estimateCapacity(tdigest.compression))
	return tdigest, nil
}

func _quantile(index float64, previousIndex float64, nextIndex float64, previousMean float64, nextMean float64) float64 {
	delta := nextIndex - previousIndex
	previousWeight := (nextIndex - index) / delta
	nextWeight := (index - previousIndex) / delta
	return previousMean*previousWeight + nextMean*nextWeight
}

// Quantile returns the desired percentile estimation.
//
// Values of p must be between 0 and 1 (inclusive), will panic otherwise.
func (t *TDigest) Quantile(q float64) float64 {
	if q < 0 || q > 1 {
		panic("q must be between 0 and 1 (inclusive)")
	}

	if t.summary.Len() == 0 {
		return math.NaN()
	} else if t.summary.Len() == 1 {
		return t.summary.Mean(0)
	}

	index := q * float64(t.count-1)
	previousMean := math.NaN()
	previousIndex := float64(0)
	next, total := t.summary.FloorSum(index)

	if next > 0 {
		previousMean = t.summary.Mean(next - 1)
		previousIndex = total - float64(t.summary.Count(next-1)+1)/2
	}

	for {
		nextIndex := total + float64(t.summary.Count(next)-1)/2
		if nextIndex >= index {
			if math.IsNaN(previousMean) {
				// the index is before the 1st centroid
				if nextIndex == previousIndex {
					return t.summary.Mean(next)
				}
				// assume linear growth
				nextIndex2 := total + float64(t.summary.Count(next)) + float64(t.summary.Count(next+1)-1)/2
				previousMean = (nextIndex2*t.summary.Mean(next) - nextIndex*t.summary.Mean(next+1)) / (nextIndex2 - nextIndex)
			}
			// common case: two centroids found, the result in inbetween
			return _quantile(index, previousIndex, nextIndex, previousMean, t.summary.Mean(next))
		} else if next+1 == t.summary.Len() {
			// the index is after the last centroid
			nextIndex2 := float64(t.count - 1)
			nextMean2 := (t.summary.Mean(next)*(nextIndex2-previousIndex) - previousMean*(nextIndex2-nextIndex)) / (nextIndex - previousIndex)
			return _quantile(index, nextIndex, nextIndex2, t.summary.Mean(next), nextMean2)
		}
		total += float64(t.summary.Count(next))
		previousMean = t.summary.Mean(next)
		previousIndex = nextIndex
		next++
	}
	// unreachable
}

func weightedAverage(x1 float64, w1 float64, x2 float64, w2 float64) float64 {
	if x1 > x2 {
		x1, x2, w1, w2 = x2, x1, w2, w1
	}
	return x1*w1/(w1+w2) + x2*w2/(w1+w2)
}

// AddWeighted registers a new sample in the digest.
//
// It's the main entry point for the digest and very likely the only
// method to be used for collecting samples. The count parameter is for
// when you are registering a sample that occurred multiple times - the
// most common value for this is 1.
//
// This will emit an error if `value` is NaN of if `count` is zero.
func (t *TDigest) AddWeighted(value float64, count uint32) (err error) {

	if count == 0 {
		return fmt.Errorf("Illegal datapoint <value: %.4f, count: %d>", value, count)
	}

	if t.summary.Len() == 0 {
		err = t.summary.Add(value, count)
		t.count = uint64(count)
		return err
	}

	begin := t.summary.Floor(value)
	if begin == -1 {
		begin = 0
	}

	begin, end := t.findNeighbors(begin, value)

	closest := t.chooseMergeCandidate(begin, end, value, count)

	if closest == t.summary.Len() {
		err = t.summary.Add(value, count)
		if err != nil {
			return err
		}
	} else {
		c := float64(t.summary.Count(closest))
		newMean := weightedAverage(t.summary.Mean(closest), c, value, float64(count))
		t.summary.setAt(closest, newMean, uint32(c)+count)
	}
	t.count += uint64(count)

	if float64(t.summary.Len()) > 20*t.compression {
		err = t.Compress()
	}

	return err
}

// Count returns the total number of samples this digest represents
//
// The result represents how many times Add() was called on a digest
// plus how many samples the digests it has been merged with had.
// This is useful mainly for two scenarios:
//
// - Knowing if there is enough data so you can trust the quantiles
//
// - Knowing if you've registered too many samples already and
// deciding what to do about it.
//
// For the second case one approach would be to create a side empty
// digest and start registering samples on it as well as on the old
// (big) one and then discard the bigger one after a certain criterion
// is reached (say, minimum number of samples or a small relative
// error between new and old digests).
func (t TDigest) Count() uint64 {
	return t.count
}

// Add is an alias for AddWeighted(x,1)
// Read the documentation for AddWeighted for more details.
func (t *TDigest) Add(value float64) error {
	return t.AddWeighted(value, 1)
}

// Compress tries to reduce the number of individual centroids stored
// in the digest.
//
// Compression trades off accuracy for performance and happens
// automatically after a certain amount of distinct samples have been
// stored.
//
// At any point in time you may call Compress on a digest, but you
// may completely ignore this and it will compress itself automatically
// after it grows too much. If you are minimizing network traffic
// it might be a good idea to compress before serializing.
func (t *TDigest) Compress() (err error) {
	if t.summary.Len() <= 1 {
		return nil
	}

	oldTree := t.summary
	t.summary = newSummary(uint(t.summary.Len()))
	t.count = 0

	shuffle(oldTree.means, oldTree.counts, t.rng)
	oldTree.ForEach(func(mean float64, count uint32) bool {
		err = t.AddWeighted(mean, count)
		return err == nil
	})

	return err
}

// Merge joins a given digest into itself.
//
// Merging is useful when you have multiple TDigest instances running
// in separate threads and you want to compute quantiles over all the
// samples. This is particularly important on a scatter-gather/map-reduce
// scenario.
func (t *TDigest) Merge(other *TDigest) (err error) {
	if other.summary.Len() == 0 {
		return nil
	}

	// We must keep the other digest intact
	data := other.summary.Clone()
	shuffle(data.means, data.counts, t.rng)

	data.ForEach(func(mean float64, count uint32) bool {
		err = t.AddWeighted(mean, count)
		return err == nil
	})
	return err
}

// CDF computes the fraction in which all samples are less than
// or equal to the given value.
func (t *TDigest) CDF(value float64) float64 {
	if t.summary.Len() == 0 {
		return math.NaN()
	} else if t.summary.Len() == 1 {
		if value < t.summary.Mean(0) {
			return 0
		}
		return 1
	}

	// We have at least 2 centroids
	left := (t.summary.Mean(1) - t.summary.Mean(0)) / 2
	right := left
	tot := 0.0

	for i := 1; i < t.summary.Len()-1; i++ {
		prevMean := t.summary.Mean(i - 1)
		if value < prevMean+right {
			v := (tot + float64(t.summary.Count(i-1))*interpolate(value, prevMean-left, prevMean+right)) / float64(t.Count())
			if v > 0 {
				return v
			}
			return 0
		}

		tot += float64(t.summary.Count(i - 1))
		left = right
		right = (t.summary.Mean(i+1) - t.summary.Mean(i)) / 2
	}

	// last centroid
	lastMean := t.summary.Mean(t.summary.Len() - 1)
	if value < lastMean+right {
		lastCount := float64(t.summary.Count(t.summary.Len() - 1))
		return (tot + lastCount*interpolate(value, lastMean-left, lastMean+right)) / 2
	}
	return 1
}

func interpolate(x, x0, x1 float64) float64 {
	return (x - x0) / (x1 - x0)
}

// ForEachCentroid calls the specified function for each centroid.
//
// Iteration stops when the supplied function returns false, or when all
// centroids have been iterated.
func (t *TDigest) ForEachCentroid(f func(mean float64, count uint32) bool) {
	t.summary.ForEach(f)
}

func (t TDigest) findNeighbors(start int, value float64) (int, int) {
	minDistance := math.MaxFloat64
	lastNeighbor := t.summary.Len()
	for neighbor := start; neighbor < t.summary.Len(); neighbor++ {
		z := math.Abs(t.summary.Mean(neighbor) - value)
		if z < minDistance {
			start = neighbor
			minDistance = z
		} else if z > minDistance {
			lastNeighbor = neighbor
			break
		}
	}
	return start, lastNeighbor
}

func (t TDigest) chooseMergeCandidate(begin, end int, value float64, count uint32) int {
	closest := t.summary.Len()
	sum := t.summary.HeadSum(begin)
	var n float32

	for neighbor := begin; neighbor != end; neighbor++ {
		c := float64(t.summary.Count(neighbor))
		var q float64
		if t.count == 1 {
			q = 0.5
		} else {
			q = (sum + (c-1)/2) / float64(t.count-1)
		}
		k := 4 * float64(t.count) * q * (1 - q) / t.compression

		if c+float64(count) <= k {
			n++
			if t.rng.Float32() < 1/n {
				closest = neighbor
			}
		}
		sum += c
	}
	return closest
}

func shuffle(means []float64, counts []uint32, rng RNG) {
	for i := len(means) - 1; i > 1; i-- {
		j := rng.Intn(i + 1)
		means[i], means[j], counts[i], counts[j] = means[j], means[i], counts[j], counts[i]
	}
}

func estimateCapacity(compression float64) uint {
	return uint(compression) * 10
}
<commit_msg>Fix misspell<commit_after>// Package tdigest provides a highly accurate mergeable data-structure
// for quantile estimation.
//
// Typical T-Digest use cases involve accumulating metrics on several
// distinct nodes of a cluster and then merging them together to get
// a system-wide quantile overview. Things such as: sensory data from
// IoT devices, quantiles over enormous document datasets (think
// ElasticSearch), performance metrics for distributed systems, etc.
//
// After you create (and configure, if desired) the digest:
//     digest := tdigest.New(tdigest.Compression(100))
//
// You can then use it for registering measurements:
//     digest.Add(number)
//
// Estimating quantiles:
//     digest.Quantile(0.99)
//
// And merging with another digest:
//     digest.Merge(otherDigest)
package tdigest

import (
	"fmt"
	"math"
)

// TDigest is a quantile approximation data structure.
type TDigest struct {
	summary     *summary
	compression float64
	count       uint64
	rng         RNG
}

// New creates a new digest.
//
// By default the digest is constructed with a configuration that
// should be useful for most use-cases. It comes with compression
// set to 100 and uses the global random number generator (same
// as using math/rand top-level functions).
func New(options ...tdigestOption) (*TDigest, error) {
	tdigest := &TDigest{
		compression: 100,
		count:       0,
		rng:         &globalRNG{},
	}

	for _, option := range options {
		err := option(tdigest)
		if err != nil {
			return nil, err
		}
	}

	tdigest.summary = newSummary(estimateCapacity(tdigest.compression))
	return tdigest, nil
}

func _quantile(index float64, previousIndex float64, nextIndex float64, previousMean float64, nextMean float64) float64 {
	delta := nextIndex - previousIndex
	previousWeight := (nextIndex - index) / delta
	nextWeight := (index - previousIndex) / delta
	return previousMean*previousWeight + nextMean*nextWeight
}

// Quantile returns the desired percentile estimation.
//
// Values of p must be between 0 and 1 (inclusive), will panic otherwise.
func (t *TDigest) Quantile(q float64) float64 {
	if q < 0 || q > 1 {
		panic("q must be between 0 and 1 (inclusive)")
	}

	if t.summary.Len() == 0 {
		return math.NaN()
	} else if t.summary.Len() == 1 {
		return t.summary.Mean(0)
	}

	index := q * float64(t.count-1)
	previousMean := math.NaN()
	previousIndex := float64(0)
	next, total := t.summary.FloorSum(index)

	if next > 0 {
		previousMean = t.summary.Mean(next - 1)
		previousIndex = total - float64(t.summary.Count(next-1)+1)/2
	}

	for {
		nextIndex := total + float64(t.summary.Count(next)-1)/2
		if nextIndex >= index {
			if math.IsNaN(previousMean) {
				// the index is before the 1st centroid
				if nextIndex == previousIndex {
					return t.summary.Mean(next)
				}
				// assume linear growth
				nextIndex2 := total + float64(t.summary.Count(next)) + float64(t.summary.Count(next+1)-1)/2
				previousMean = (nextIndex2*t.summary.Mean(next) - nextIndex*t.summary.Mean(next+1)) / (nextIndex2 - nextIndex)
			}
			// common case: two centroids found, the result in in between
			return _quantile(index, previousIndex, nextIndex, previousMean, t.summary.Mean(next))
		} else if next+1 == t.summary.Len() {
			// the index is after the last centroid
			nextIndex2 := float64(t.count - 1)
			nextMean2 := (t.summary.Mean(next)*(nextIndex2-previousIndex) - previousMean*(nextIndex2-nextIndex)) / (nextIndex - previousIndex)
			return _quantile(index, nextIndex, nextIndex2, t.summary.Mean(next), nextMean2)
		}
		total += float64(t.summary.Count(next))
		previousMean = t.summary.Mean(next)
		previousIndex = nextIndex
		next++
	}
	// unreachable
}

func weightedAverage(x1 float64, w1 float64, x2 float64, w2 float64) float64 {
	if x1 > x2 {
		x1, x2, w1, w2 = x2, x1, w2, w1
	}
	return x1*w1/(w1+w2) + x2*w2/(w1+w2)
}

// AddWeighted registers a new sample in the digest.
//
// It's the main entry point for the digest and very likely the only
// method to be used for collecting samples. The count parameter is for
// when you are registering a sample that occurred multiple times - the
// most common value for this is 1.
//
// This will emit an error if `value` is NaN of if `count` is zero.
func (t *TDigest) AddWeighted(value float64, count uint32) (err error) {

	if count == 0 {
		return fmt.Errorf("Illegal datapoint <value: %.4f, count: %d>", value, count)
	}

	if t.summary.Len() == 0 {
		err = t.summary.Add(value, count)
		t.count = uint64(count)
		return err
	}

	begin := t.summary.Floor(value)
	if begin == -1 {
		begin = 0
	}

	begin, end := t.findNeighbors(begin, value)

	closest := t.chooseMergeCandidate(begin, end, value, count)

	if closest == t.summary.Len() {
		err = t.summary.Add(value, count)
		if err != nil {
			return err
		}
	} else {
		c := float64(t.summary.Count(closest))
		newMean := weightedAverage(t.summary.Mean(closest), c, value, float64(count))
		t.summary.setAt(closest, newMean, uint32(c)+count)
	}
	t.count += uint64(count)

	if float64(t.summary.Len()) > 20*t.compression {
		err = t.Compress()
	}

	return err
}

// Count returns the total number of samples this digest represents
//
// The result represents how many times Add() was called on a digest
// plus how many samples the digests it has been merged with had.
// This is useful mainly for two scenarios:
//
// - Knowing if there is enough data so you can trust the quantiles
//
// - Knowing if you've registered too many samples already and
// deciding what to do about it.
//
// For the second case one approach would be to create a side empty
// digest and start registering samples on it as well as on the old
// (big) one and then discard the bigger one after a certain criterion
// is reached (say, minimum number of samples or a small relative
// error between new and old digests).
func (t TDigest) Count() uint64 {
	return t.count
}

// Add is an alias for AddWeighted(x,1)
// Read the documentation for AddWeighted for more details.
func (t *TDigest) Add(value float64) error {
	return t.AddWeighted(value, 1)
}

// Compress tries to reduce the number of individual centroids stored
// in the digest.
//
// Compression trades off accuracy for performance and happens
// automatically after a certain amount of distinct samples have been
// stored.
//
// At any point in time you may call Compress on a digest, but you
// may completely ignore this and it will compress itself automatically
// after it grows too much. If you are minimizing network traffic
// it might be a good idea to compress before serializing.
func (t *TDigest) Compress() (err error) {
	if t.summary.Len() <= 1 {
		return nil
	}

	oldTree := t.summary
	t.summary = newSummary(uint(t.summary.Len()))
	t.count = 0

	shuffle(oldTree.means, oldTree.counts, t.rng)
	oldTree.ForEach(func(mean float64, count uint32) bool {
		err = t.AddWeighted(mean, count)
		return err == nil
	})

	return err
}

// Merge joins a given digest into itself.
//
// Merging is useful when you have multiple TDigest instances running
// in separate threads and you want to compute quantiles over all the
// samples. This is particularly important on a scatter-gather/map-reduce
// scenario.
func (t *TDigest) Merge(other *TDigest) (err error) {
	if other.summary.Len() == 0 {
		return nil
	}

	// We must keep the other digest intact
	data := other.summary.Clone()
	shuffle(data.means, data.counts, t.rng)

	data.ForEach(func(mean float64, count uint32) bool {
		err = t.AddWeighted(mean, count)
		return err == nil
	})
	return err
}

// CDF computes the fraction in which all samples are less than
// or equal to the given value.
func (t *TDigest) CDF(value float64) float64 {
	if t.summary.Len() == 0 {
		return math.NaN()
	} else if t.summary.Len() == 1 {
		if value < t.summary.Mean(0) {
			return 0
		}
		return 1
	}

	// We have at least 2 centroids
	left := (t.summary.Mean(1) - t.summary.Mean(0)) / 2
	right := left
	tot := 0.0

	for i := 1; i < t.summary.Len()-1; i++ {
		prevMean := t.summary.Mean(i - 1)
		if value < prevMean+right {
			v := (tot + float64(t.summary.Count(i-1))*interpolate(value, prevMean-left, prevMean+right)) / float64(t.Count())
			if v > 0 {
				return v
			}
			return 0
		}

		tot += float64(t.summary.Count(i - 1))
		left = right
		right = (t.summary.Mean(i+1) - t.summary.Mean(i)) / 2
	}

	// last centroid
	lastMean := t.summary.Mean(t.summary.Len() - 1)
	if value < lastMean+right {
		lastCount := float64(t.summary.Count(t.summary.Len() - 1))
		return (tot + lastCount*interpolate(value, lastMean-left, lastMean+right)) / 2
	}
	return 1
}

func interpolate(x, x0, x1 float64) float64 {
	return (x - x0) / (x1 - x0)
}

// ForEachCentroid calls the specified function for each centroid.
//
// Iteration stops when the supplied function returns false, or when all
// centroids have been iterated.
func (t *TDigest) ForEachCentroid(f func(mean float64, count uint32) bool) {
	t.summary.ForEach(f)
}

func (t TDigest) findNeighbors(start int, value float64) (int, int) {
	minDistance := math.MaxFloat64
	lastNeighbor := t.summary.Len()
	for neighbor := start; neighbor < t.summary.Len(); neighbor++ {
		z := math.Abs(t.summary.Mean(neighbor) - value)
		if z < minDistance {
			start = neighbor
			minDistance = z
		} else if z > minDistance {
			lastNeighbor = neighbor
			break
		}
	}
	return start, lastNeighbor
}

func (t TDigest) chooseMergeCandidate(begin, end int, value float64, count uint32) int {
	closest := t.summary.Len()
	sum := t.summary.HeadSum(begin)
	var n float32

	for neighbor := begin; neighbor != end; neighbor++ {
		c := float64(t.summary.Count(neighbor))
		var q float64
		if t.count == 1 {
			q = 0.5
		} else {
			q = (sum + (c-1)/2) / float64(t.count-1)
		}
		k := 4 * float64(t.count) * q * (1 - q) / t.compression

		if c+float64(count) <= k {
			n++
			if t.rng.Float32() < 1/n {
				closest = neighbor
			}
		}
		sum += c
	}
	return closest
}

func shuffle(means []float64, counts []uint32, rng RNG) {
	for i := len(means) - 1; i > 1; i-- {
		j := rng.Intn(i + 1)
		means[i], means[j], counts[i], counts[j] = means[j], means[i], counts[j], counts[i]
	}
}

func estimateCapacity(compression float64) uint {
	return uint(compression) * 10
}
<|endoftext|>


<commit_before>package failover

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/siddontang/go/log"
	"net"
	"strconv"
	"sync"
	"time"
)

var (
	ErrNodeDown    = errors.New("Node is down")
	ErrNodeAlive   = errors.New("Node may be still alive")
	ErrNoCandidate = errors.New("no proper candidate to be promoted to master")
	ErrNodeType    = errors.New("Node is not the expected type")
)

const (
	MasterType = "master"
	SlaveType  = "slave"
)

const (
	ConnectState    = "connect"
	ConnectingState = "connecting"
	ConnectedState  = "connected"
	SyncState       = "sync"
)

// A node represents a real redis server
type Node struct {
	// Redis address, only support tcp now
	Addr string

	// Replication offset
	Offset int64

	conn redis.Conn
}

func (n *Node) String() string {
	return fmt.Sprintf("{addr: %s offset: %d}", n.Addr, n.Offset)
}

func (n *Node) doCommand(cmd string, args ...interface{}) (interface{}, error) {
	var err error
	var v interface{}
	for i := 0; i < 3; i++ {
		if n.conn == nil {
			n.conn, err = redis.DialTimeout("tcp", n.Addr, 5*time.Second, 0, 0)
			if err != nil {
				log.Errorf("dial %s error: %v, ry again", n.Addr, err)
				continue
			}

		}

		v, err = n.conn.Do(cmd, args...)
		if err != nil {
			log.Errorf("do %s command for %s error: %v, try again", cmd, n.Addr, err)
			n.conn.Close()
			n.conn = nil
			continue
		} else {
			return v, nil
		}
	}

	// go here means do command error, maybe redis is down.
	return nil, err
}

func (n *Node) doRole() ([]interface{}, error) {
	return redis.Values(n.doCommand("ROLE"))
}

func (n *Node) ping() error {
	_, err := n.doCommand("PING")
	return err
}

func (n *Node) slaveof(host string, port string) error {
	_, err := n.doCommand("SLAVEOF", host, port)
	return err
}

func (n *Node) close() {
	if n.conn != nil {
		n.conn.Close()
		n.conn = nil
	}
}

// A group contains a Redis master and one or more slaves
// It will use role command per second to check master's alive
// and find slaves automatically.
type Group struct {
	Master *Node
	Slaves map[string]*Node

	m sync.Mutex
}

func newGroup(masterAddr string) *Group {
	g := new(Group)

	g.Master = &Node{Addr: masterAddr}
	g.Slaves = make(map[string]*Node)

	return g
}

func (g *Group) Close() {
	g.m.Lock()
	defer g.m.Unlock()

	g.Master.close()

	for _, slave := range g.Slaves {
		slave.close()
	}
}

func (g *Group) Check() error {
	g.m.Lock()
	defer g.m.Unlock()

	return g.doRole()
}

func (g *Group) doRole() error {
	v, err := g.Master.doRole()
	if err != nil {
		return ErrNodeDown
	}

	// the first line is server type
	serverType, _ := redis.String(v[0], nil)
	if serverType != MasterType {
		log.Errorf("server %s is not master now", g.Master.Addr)
		return ErrNodeType
	}

	// second is master replication offset,
	g.Master.Offset, _ = redis.Int64(v[1], nil)

	// then slave list [host, port, offset]
	slaves, _ := redis.Values(v[2], nil)
	nodes := make(map[string]*Node, len(slaves))
	for i := 0; i < len(slaves); i++ {
		ss, _ := redis.Strings(slaves[i], nil)
		var n Node
		n.Addr = fmt.Sprintf("%s:%s", ss[0], ss[1])
		n.Offset, _ = strconv.ParseInt(fmt.Sprintf("%s", ss[2]), 10, 64)
		nodes[n.Addr] = &n
	}

	// we don't care slave add or remove too much, so only log
	for addr, _ := range nodes {
		if _, ok := g.Slaves[addr]; !ok {
			log.Infof("slave %s added", addr)
		}
	}

	for addr, slave := range g.Slaves {
		if _, ok := nodes[addr]; !ok {
			log.Infof("slave %s removed", addr)
			slave.close()
		}
	}

	g.Slaves = nodes
	return nil
}

func (g *Group) Ping() error {
	g.m.Lock()
	defer g.m.Unlock()

	return g.Master.ping()
}

// Elect a best slave which has the most up-to-date data with master
func (g *Group) Elect() (string, error) {
	g.m.Lock()
	defer g.m.Unlock()

	var addr string
	var maxOffset int64 = 0

	for _, slave := range g.Slaves {
		v, err := slave.doRole()
		if err != nil {
			//skip this slave
			log.Infof("slave %s do role command err %v, skip it", slave.Addr, err)
			continue
		}

		// the first line is server type
		serverType, _ := redis.String(v[0], nil)
		if serverType != SlaveType {
			log.Errorf("server %s is not slave now, skip it", slave.Addr)
			continue
		}

		// the second and third is host and port, skip it
		// the fouth is replication state
		state, _ := redis.String(v[3], nil)
		if state == ConnectedState || state == SyncState {
			log.Infof("slave %s replication state is %s, master %s:%v may be not down???",
				slave.Addr, state, v[1], v[2])
			return "", ErrNodeAlive
		}

		// the end is the replication offset, but it is still -1 if master is down

		// we can only use last master role command information to determind the best slave
		if maxOffset < slave.Offset {
			maxOffset = slave.Offset
			addr = slave.Addr
		}
	}

	if len(addr) == 0 {
		log.Errorf("no proper candidate to be promoted")
		return "", ErrNoCandidate
	}

	return addr, nil
}

// Promote the slave to master, then let other slaves replicate from it
func (g *Group) Promote(addr string) error {
	g.m.Lock()
	defer g.m.Unlock()

	node := g.Slaves[addr]

	if err := node.slaveof("no", "one"); err != nil {
		return err
	}

	delete(g.Slaves, addr)

	g.Master = node

	host, port, _ := net.SplitHostPort(addr)
	for _, slave := range g.Slaves {
		if err := slave.slaveof(host, port); err != nil {
			// if we go here, the replication topology may be wrong
			// so use fatal level and we should fix it manually
			log.Fatalf("slaveof %s to master %s err %v", slave.Addr, addr, err)
		} else {
			log.Infof("slaveof %s to master %s ok", slave.Addr, addr)
		}
	}

	return nil
}
<commit_msg>use replication info to elect new master<commit_after>package failover

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/siddontang/go/log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	ErrNodeDown    = errors.New("Node is down")
	ErrNodeAlive   = errors.New("Node may be still alive")
	ErrNoCandidate = errors.New("no proper candidate to be promoted to master")
	ErrNodeType    = errors.New("Node is not the expected type")
)

const (
	MasterType = "master"
	SlaveType  = "slave"
)

const (
	ConnectState    = "connect"
	ConnectingState = "connecting"
	ConnectedState  = "connected"
	SyncState       = "sync"
)

// A node represents a real redis server
type Node struct {
	// Redis address, only support tcp now
	Addr string

	// Replication offset
	Offset int64

	conn redis.Conn
}

func (n *Node) String() string {
	return fmt.Sprintf("{addr: %s offset: %d}", n.Addr, n.Offset)
}

func (n *Node) doCommand(cmd string, args ...interface{}) (interface{}, error) {
	var err error
	var v interface{}
	for i := 0; i < 3; i++ {
		if n.conn == nil {
			n.conn, err = redis.DialTimeout("tcp", n.Addr, 5*time.Second, 0, 0)
			if err != nil {
				log.Errorf("dial %s error: %v, ry again", n.Addr, err)
				continue
			}

		}

		v, err = n.conn.Do(cmd, args...)
		if err != nil {
			log.Errorf("do %s command for %s error: %v, try again", cmd, n.Addr, err)
			n.conn.Close()
			n.conn = nil
			continue
		} else {
			return v, nil
		}
	}

	// go here means do command error, maybe redis is down.
	return nil, err
}

func (n *Node) doRole() ([]interface{}, error) {
	return redis.Values(n.doCommand("ROLE"))
}

func (n *Node) ping() error {
	_, err := n.doCommand("PING")
	return err
}

func (n *Node) slaveof(host string, port string) error {
	_, err := n.doCommand("SLAVEOF", host, port)
	return err
}

func (n *Node) doRelpInfo() (map[string]string, error) {
	v, err := redis.String(n.doCommand("INFO", "REPLICATION"))
	if err != nil {
		return nil, err
	}

	seps := strings.Split(v, "\r\n")
	// skip first line, is # Replication
	seps = seps[1:]

	m := make(map[string]string, len(seps))
	for _, s := range seps {
		kv := strings.SplitN(s, ":", 2)
		if len(kv) == 2 {
			m[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
		}
	}
	return m, nil
}

func (n *Node) close() {
	if n.conn != nil {
		n.conn.Close()
		n.conn = nil
	}
}

// A group contains a Redis master and one or more slaves
// It will use role command per second to check master's alive
// and find slaves automatically.
type Group struct {
	Master *Node
	Slaves map[string]*Node

	m sync.Mutex
}

func newGroup(masterAddr string) *Group {
	g := new(Group)

	g.Master = &Node{Addr: masterAddr}
	g.Slaves = make(map[string]*Node)

	return g
}

func (g *Group) Close() {
	g.m.Lock()
	defer g.m.Unlock()

	g.Master.close()

	for _, slave := range g.Slaves {
		slave.close()
	}
}

func (g *Group) Check() error {
	g.m.Lock()
	defer g.m.Unlock()

	return g.doRole()
}

func (g *Group) doRole() error {
	v, err := g.Master.doRole()
	if err != nil {
		return ErrNodeDown
	}

	// the first line is server type
	serverType, _ := redis.String(v[0], nil)
	if serverType != MasterType {
		log.Errorf("server %s is not master now", g.Master.Addr)
		return ErrNodeType
	}

	// second is master replication offset,
	g.Master.Offset, _ = redis.Int64(v[1], nil)

	// then slave list [host, port, offset]
	slaves, _ := redis.Values(v[2], nil)
	nodes := make(map[string]*Node, len(slaves))
	for i := 0; i < len(slaves); i++ {
		ss, _ := redis.Strings(slaves[i], nil)
		var n Node
		n.Addr = fmt.Sprintf("%s:%s", ss[0], ss[1])
		n.Offset, _ = strconv.ParseInt(fmt.Sprintf("%s", ss[2]), 10, 64)
		nodes[n.Addr] = &n
	}

	// we don't care slave add or remove too much, so only log
	for addr, _ := range nodes {
		if _, ok := g.Slaves[addr]; !ok {
			log.Infof("slave %s added", addr)
		}
	}

	for addr, slave := range g.Slaves {
		if _, ok := nodes[addr]; !ok {
			log.Infof("slave %s removed", addr)
			slave.close()
		}
	}

	g.Slaves = nodes
	return nil
}

func (g *Group) Ping() error {
	g.m.Lock()
	defer g.m.Unlock()

	return g.Master.ping()
}

// Elect a best slave which has the most up-to-date data with master
func (g *Group) Elect() (string, error) {
	g.m.Lock()
	defer g.m.Unlock()

	var addr string
	var checkOffset int64 = 0
	var checkPriority int = 0

	for _, slave := range g.Slaves {
		m, err := slave.doRelpInfo()
		if err != nil {
			log.Infof("slave %s get replication info err %v, skip it", slave.Addr, err)
			continue
		}

		if m["slave"] == MasterType {
			log.Errorf("server %s is not slave now, skip it", slave.Addr)
			continue
		}

		if m["master_link_status"] == "up" {
			log.Infof("slave %s master_link_status is up, master %s:%s may be not down???",
				slave.Addr, m["master_host"], m["master_port"])
			return "", ErrNodeAlive
		}

		priority, _ := strconv.Atoi(m["slave_priority"])
		replOffset, _ := strconv.ParseInt(m["slave_repl_offset"], 10, 64)

		used := false
		// like redis-sentinel, first check priority, then salve repl offset
		if checkPriority < priority {
			used = true
		} else if checkPriority == priority {
			if checkOffset < replOffset {
				used = true
			}
		}

		if used {
			addr = slave.Addr
			checkPriority = priority
			checkOffset = replOffset
		}
	}

	if len(addr) == 0 {
		log.Errorf("no proper candidate to be promoted")
		return "", ErrNoCandidate
	}

	log.Infof("select slave %s as new master, priority:%d, repl_offset:%d", addr, checkPriority, checkOffset)

	return addr, nil
}

// Promote the slave to master, then let other slaves replicate from it
func (g *Group) Promote(addr string) error {
	g.m.Lock()
	defer g.m.Unlock()

	node := g.Slaves[addr]

	if err := node.slaveof("no", "one"); err != nil {
		return err
	}

	delete(g.Slaves, addr)

	g.Master = node

	host, port, _ := net.SplitHostPort(addr)
	for _, slave := range g.Slaves {
		if err := slave.slaveof(host, port); err != nil {
			// if we go here, the replication topology may be wrong
			// so use fatal level and we should fix it manually
			log.Fatalf("slaveof %s to master %s err %v", slave.Addr, addr, err)
		} else {
			log.Infof("slaveof %s to master %s ok", slave.Addr, addr)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    //"os"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var insertoffset = registeredTaskSearch(
            apq.v[apq.head:apq.tail],
            func(t RegisteredTask) bool {
                return t.Task().(PrioritizedTask).Key() < key
            })
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > apq.head+insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head < len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    var i, j int
    j = 0
    for i = apq.head ; i < apq.head+insertoffset ; i++ {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
    }
    //fmt.Fprintf(os.Stderr, "Length %d index %d\n", len(newv), j)
    newv[insertoffset] = task
    j++
    for ; i < apq.tail ; i++ {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
    }
    apq.v = newv
    apq.head = 0
    apq.tail = j
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<commit_msg>Fix indexing error.<commit_after>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    //"os"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var insertoffset = registeredTaskSearch(
            apq.v[apq.head:apq.tail],
            func(t RegisteredTask) bool {
                return t.Task().(PrioritizedTask).Key() < key
            })
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > apq.head+insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset-1] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head < len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    var i, j int
    j = 0
    i = apq.head
    for j < insertoffset {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
        i++
    }
    //fmt.Fprintf(os.Stderr, "Length %d index %d\n", len(newv), j)
    newv[insertoffset] = task
    j++
    for i < apq.tail  {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
        i++
    }
    apq.v = newv
    apq.head = 0
    apq.tail = j
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<|endoftext|>


<commit_before>package config

// DeviceNamed contains the name of a device and its config.
type DeviceNamed struct {
	Name   string
	Config Device
}

// DevicesSortable is a sortable slice of device names and config.
type DevicesSortable []DeviceNamed

func (devices DevicesSortable) Len() int {
	return len(devices)
}

func (devices DevicesSortable) Less(i, j int) bool {
	a := devices[i]
	b := devices[j]

	// First sort by types.
	if a.Config["type"] != b.Config["type"] {
		// In VMs, network interface names are derived from PCI
		// location. As a result of that, we must ensure that nic devices will
		// always show up at the same spot regardless of what other devices may be
		// added. Easiest way to do this is to always have them show up first.
		if a.Config["type"] == "nic" {
			return true
		}

		if b.Config["type"] == "nic" {
			return false
		}

		// Start disks before other non-nic devices so that any unmounts triggered by deferred resizes
		// specified in volatile "apply_quota" key can occur first and the rest of the devices can rely on
		// the instance's root disk being mounted.
		if a.Config["type"] == "disk" {
			return true
		}

		if b.Config["type"] == "disk" {
			return false
		}

		// Otherwise start devices of same type together.
		return a.Config["type"] > b.Config["type"]
	}

	// Start disk devices in path order.
	if a.Config["type"] == "disk" && b.Config["type"] == "disk" {
		if a.Config["path"] != b.Config["path"] {
			return a.Config["path"] < b.Config["path"]
		}
	}

	// Fallback to sorting by names.
	return a.Name < b.Name
}

func (devices DevicesSortable) Swap(i, j int) {
	devices[i], devices[j] = devices[j], devices[i]
}
<commit_msg>lxd/device/config: Put root device first<commit_after>package config

// DeviceNamed contains the name of a device and its config.
type DeviceNamed struct {
	Name   string
	Config Device
}

// DevicesSortable is a sortable slice of device names and config.
type DevicesSortable []DeviceNamed

func (devices DevicesSortable) Len() int {
	return len(devices)
}

func (devices DevicesSortable) Less(i, j int) bool {
	a := devices[i]
	b := devices[j]

	// First sort by types.
	if a.Config["type"] != b.Config["type"] {
		// In VMs, network interface names are derived from PCI
		// location. As a result of that, we must ensure that nic devices will
		// always show up at the same spot regardless of what other devices may be
		// added. Easiest way to do this is to always have them show up first.
		if a.Config["type"] == "nic" {
			return true
		}

		if b.Config["type"] == "nic" {
			return false
		}

		// Start disks before other non-nic devices so that any unmounts triggered by deferred resizes
		// specified in volatile "apply_quota" key can occur first and the rest of the devices can rely on
		// the instance's root disk being mounted.
		if a.Config["type"] == "disk" {
			return true
		}

		if b.Config["type"] == "disk" {
			return false
		}

		// Otherwise start devices of same type together.
		return a.Config["type"] > b.Config["type"]
	}

	// Start disk devices in path order.
	if a.Config["type"] == "disk" && b.Config["type"] == "disk" {
		if a.Config["path"] != b.Config["path"] {
			// The root device always goes first.
			if a.Config["path"] == "/" {
				return true
			}

			if b.Config["path"] == "/" {
				return false
			}

			return a.Config["path"] < b.Config["path"]
		}
	}

	// Fallback to sorting by names.
	return a.Name < b.Name
}

func (devices DevicesSortable) Swap(i, j int) {
	devices[i], devices[j] = devices[j], devices[i]
}
<|endoftext|>


<commit_before>package bdiscord

import (
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/bwmarrin/discordgo"
)

func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) {
	rmsg := config.Message{Account: b.Account, ID: m.ID, Event: config.EventMsgDelete, Text: config.EventMsgDelete}
	rmsg.Channel = b.getChannelName(m.ChannelID)
	if b.useChannelID {
		rmsg.Channel = "ID:" + m.ChannelID
	}
	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)
	b.Remote <- rmsg
}

func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) {
	if b.GetBool("EditDisable") {
		return
	}
	// only when message is actually edited
	if m.Message.EditedTimestamp != "" {
		b.Log.Debugf("Sending edit message")
		m.Content += b.GetString("EditSuffix")
		b.messageCreate(s, (*discordgo.MessageCreate)(m))
	}
}

func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	var err error

	// not relay our own messages
	if m.Author.Username == b.nick {
		return
	}
	// if using webhooks, do not relay if it's ours
	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
		return
	}

	// add the url of the attachments to content
	if len(m.Attachments) > 0 {
		for _, attach := range m.Attachments {
			m.Content = m.Content + "\n" + attach.URL
		}
	}

	rmsg := config.Message{Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg", UserID: m.Author.ID, ID: m.ID}

	if m.Content != "" {
		b.Log.Debugf("== Receiving event %#v", m.Message)
		m.Message.Content = b.stripCustomoji(m.Message.Content)
		m.Message.Content = b.replaceChannelMentions(m.Message.Content)
		rmsg.Text, err = m.ContentWithMoreMentionsReplaced(b.c)
		if err != nil {
			b.Log.Errorf("ContentWithMoreMentionsReplaced failed: %s", err)
			rmsg.Text = m.ContentWithMentionsReplaced()
		}
	}

	// set channel name
	rmsg.Channel = b.getChannelName(m.ChannelID)
	if b.useChannelID {
		rmsg.Channel = "ID:" + m.ChannelID
	}

	// set username
	if !b.GetBool("UseUserName") {
		rmsg.Username = b.getNick(m.Author)
	} else {
		rmsg.Username = m.Author.Username
	}

	// if we have embedded content add it to text
	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
		for _, embed := range m.Message.Embeds {
			rmsg.Text = rmsg.Text + "embed: " + embed.Title + " - " + embed.Description + " - " + embed.URL + "\n"
		}
	}

	// no empty messages
	if rmsg.Text == "" {
		return
	}

	// do we have a /me action
	var ok bool
	rmsg.Text, ok = b.replaceAction(rmsg.Text)
	if ok {
		rmsg.Event = config.EventUserAction
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)
	b.Remote <- rmsg
}

func (b *Bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
	if m.Member == nil {
		b.Log.Warnf("Received member update with no member information: %#v", m)
	}

	b.membersMutex.Lock()
	defer b.membersMutex.Unlock()

	if currMember, ok := b.userMemberMap[m.Member.User.ID]; ok {
		b.Log.Debugf(
			"%s: memberupdate: user %s (nick %s) changes nick to %s",
			b.Account,
			m.Member.User.Username,
			b.userMemberMap[m.Member.User.ID].Nick,
			m.Member.Nick,
		)
		delete(b.nickMemberMap, currMember.User.Username)
		delete(b.nickMemberMap, currMember.Nick)
		delete(b.userMemberMap, m.Member.User.ID)
	}
	b.userMemberMap[m.Member.User.ID] = m.Member
	b.nickMemberMap[m.Member.User.Username] = m.Member
	if m.Member.Nick != "" {
		b.nickMemberMap[m.Member.Nick] = m.Member
	}
}
<commit_msg>Disable some unparam checks (discord)<commit_after>package bdiscord

import (
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/bwmarrin/discordgo"
)

func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) { //nolint:unparam
	rmsg := config.Message{Account: b.Account, ID: m.ID, Event: config.EventMsgDelete, Text: config.EventMsgDelete}
	rmsg.Channel = b.getChannelName(m.ChannelID)
	if b.useChannelID {
		rmsg.Channel = "ID:" + m.ChannelID
	}
	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)
	b.Remote <- rmsg
}

func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) { //nolint:unparam
	if b.GetBool("EditDisable") {
		return
	}
	// only when message is actually edited
	if m.Message.EditedTimestamp != "" {
		b.Log.Debugf("Sending edit message")
		m.Content += b.GetString("EditSuffix")
		b.messageCreate(s, (*discordgo.MessageCreate)(m))
	}
}

func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) { //nolint:unparam
	var err error

	// not relay our own messages
	if m.Author.Username == b.nick {
		return
	}
	// if using webhooks, do not relay if it's ours
	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
		return
	}

	// add the url of the attachments to content
	if len(m.Attachments) > 0 {
		for _, attach := range m.Attachments {
			m.Content = m.Content + "\n" + attach.URL
		}
	}

	rmsg := config.Message{Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg", UserID: m.Author.ID, ID: m.ID}

	if m.Content != "" {
		b.Log.Debugf("== Receiving event %#v", m.Message)
		m.Message.Content = b.stripCustomoji(m.Message.Content)
		m.Message.Content = b.replaceChannelMentions(m.Message.Content)
		rmsg.Text, err = m.ContentWithMoreMentionsReplaced(b.c)
		if err != nil {
			b.Log.Errorf("ContentWithMoreMentionsReplaced failed: %s", err)
			rmsg.Text = m.ContentWithMentionsReplaced()
		}
	}

	// set channel name
	rmsg.Channel = b.getChannelName(m.ChannelID)
	if b.useChannelID {
		rmsg.Channel = "ID:" + m.ChannelID
	}

	// set username
	if !b.GetBool("UseUserName") {
		rmsg.Username = b.getNick(m.Author)
	} else {
		rmsg.Username = m.Author.Username
	}

	// if we have embedded content add it to text
	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
		for _, embed := range m.Message.Embeds {
			rmsg.Text = rmsg.Text + "embed: " + embed.Title + " - " + embed.Description + " - " + embed.URL + "\n"
		}
	}

	// no empty messages
	if rmsg.Text == "" {
		return
	}

	// do we have a /me action
	var ok bool
	rmsg.Text, ok = b.replaceAction(rmsg.Text)
	if ok {
		rmsg.Event = config.EventUserAction
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)
	b.Remote <- rmsg
}

func (b *Bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
	if m.Member == nil {
		b.Log.Warnf("Received member update with no member information: %#v", m)
	}

	b.membersMutex.Lock()
	defer b.membersMutex.Unlock()

	if currMember, ok := b.userMemberMap[m.Member.User.ID]; ok {
		b.Log.Debugf(
			"%s: memberupdate: user %s (nick %s) changes nick to %s",
			b.Account,
			m.Member.User.Username,
			b.userMemberMap[m.Member.User.ID].Nick,
			m.Member.Nick,
		)
		delete(b.nickMemberMap, currMember.User.Username)
		delete(b.nickMemberMap, currMember.Nick)
		delete(b.userMemberMap, m.Member.User.ID)
	}
	b.userMemberMap[m.Member.User.ID] = m.Member
	b.nickMemberMap[m.Member.User.Username] = m.Member
	if m.Member.Nick != "" {
		b.nickMemberMap[m.Member.Nick] = m.Member
	}
}
<|endoftext|>